% Template for SLT-2016 paper; to be used with:
%          spconf.sty  - ICASSP/ICIP LaTeX style file, and
%          IEEEbib.bst - IEEE bibliography style file.
% --------------------------------------------------------------------------
\documentclass{article}
\usepackage{spconf,amsmath,graphicx, multirow, standalone}
\usepackage{csquotes}
\usepackage{tikz}
\usepackage{listings}
\usetikzlibrary{positioning,arrows,calc}

\newcommand{\bs}[1]{\boldsymbol{#1}}

% Title.
% ------
\title{Building an end-to-end speech recognizer}

\name{{\em Moritz Wolter$^1$}}
\address{$^1$Department Electrical Engineering-ESAT, Katholieke Universiteit Leuven\\
Kasteelpark Arenberg 10, Bus 2441, B-3001 Leuven Belgium}
\begin{document}

\maketitle
%
\begin{abstract}
Attention mechanisms are an interesting new trend in machine learning. Recently an attention based speech transducer called Listen, Attend and Spell (LAS) \cite{Chan2015} has been developed. LAS is an end-to-end system, which can be trained jointly using gradient descent. In contrast to previous systems LAS decouples input and decoding time, producing an output sequence directly without requiring collapsing functions or independence assumptions. Here we present our own re-implementation of this algorithm.
\end{abstract}
%
\begin{keywords}
Neural attention, data alignment, speech recognition, end-to-end networks.
\end{keywords}
%
\section{Introduction}
\label{sec:intro}
Automatic speech recognition is concerned with finding ways to enable computers
to recognize spoken language and transcribe it to text. In order to solve this task using machine learning algorithms, it must be determined, which parts of a recording are interesting.
Neural attention is a good way to do that. It is the most important part of the LAS architecture, which consists of two parts, a listener and a speller.
The listener functions as an encoder computing a compressed high level representation of the input. It consists of a bidirectional Long Short Term Memory (BLSTM) layer followed by pyramidal LSTM layers (PLSTM). Each pyramidal layer halves the time dimension of the input.
At the hart of the speller is an attend and spell cell. This augmented recurrent neural network cell contains the attention and spelling functions which produce the desired output sequence when unrolled.

\section{Long Short Term Memory}
\label{sec:LSTM}
Long short term memory blocks can be thought of as differentiable versions of computer memory chips \cite{Graves2012}. Differentiability is important, because it allows systems containing these memory cells to be trained using gradient descent. LSTMs can be accessed trough input output and forget gates, which let the optimization algorithm control, what inputs are stored, which parts of the internal memory are erased and which state elements turn into outputs.
The gates $\mathbf{i}_t, \mathbf{f}_t, \mathbf{o}_t$ as well as the state $\mathbf{s}_t$ and the output  $\mathbf{h}_t$ are defined by \cite{Graves2013}\footnote{ Various versions of LSTM cells exist. This one is commonly referred to as the \textquotedblleft peephole\textquotedblright $ \: $ variant. } :
\begin{align}
\mathbf{i}_t &= \sigma (\mathbf{W}_i [\mathbf{x}_t \; \mathbf{h}_{t-1} \; \mathbf{s}_{t-1}]^T + \mathbf{b}_i), \\
\mathbf{f}_t &= \sigma (\mathbf{W}_f [\mathbf{x}_t \; \mathbf{h}_{t-1} \; \mathbf{s}_{t-1}]^T + \mathbf{b}_f), \\
\mathbf{s}_t &= \mathbf{f}_t \mathbf{s}_{t-1} + \mathbf{i}_t \tanh( \mathbf{W}_s [\mathbf{x}_t \; \mathbf{h}_{t-1}]^T + \mathbf{b}_s ), \\
\mathbf{o}_t &= \sigma (\mathbf{W}_o [\mathbf{x}_t \; \mathbf{h}_{t-1} \; \mathbf{s}_t]^T + \mathbf{b}_o ), \\
\mathbf{h}_t &= \mathbf{o}_t \tanh(\mathbf{s}_t).
\end{align}
The most important equation is the one for the state $\mathbf{s}_t$, the state functions as cell memory, and it's equation contains all possible operations, which can be applied to it. The first summand is the result of multiplying the forget gate's output element wise with the state from the previous time-step. One entries in $\mathbf{f}_t$ mean \enquote{Keep this value}, zeros mean \enquote{trow it away}. The second summand allows adding input values to the state. The values between zero and one in $\mathbf{i}_t$ express which of the input elements should be remembered. Finally $\mathbf{o}_t$ determines, which elements of the internal memory will be shared with the outside world. In contrast to simple recurrent cells LSTMs do not suffer from the vanishing gradient problem. Which is why they form a the bedrock of the LAS system.

\section{Listen attend and Spell}
The Listen Attend and Spell \cite{Chan2015} architecture is a deep neural network, designed to jointly learn to align and transcribe speech data. It consists of two parts.
The first part also called the listener accepts filter bank spectra $\mathbf{x}_n$ as inputs and produces compressed high level output features $\mathbf{h}_m$. Compression reduces the computational load during feature processing later.
The speller in turn accepts the features as input and outputs distributions over character sequences $\mathbf{y}_p$. Due to the uncoupling of the input and decoding time step, it requires a state, as well as an attention mechanism. The state provides a memory of what happened in the decoder in the past. The attention function determines, which listener-features are relevant at a given decoding time step. Combining attention and state makes it possible to label the input data.
\begin{figure}
\centering
\includegraphics[width=0.4\textwidth]{../tikz/lasArcBottomUp.pdf}
\caption{The LAS architecture \cite{Chan2015}. BLSTM blocks are shown in red. LSTM blocks in blue and attention nets in green.}
\label{fig:las}
\end{figure}
An overview of the las-achrcitecture is given in figure~\ref{fig:las}.

\subsection{The Listener}
\label{subsec:Listener}
The listener, consists of Long Short Term Memory blocks. These blocks are arranged in layers. The inputs are first fed into a BLSTM layer. This choice gives the system access to future data, therefore only fully recorded data can be analyzed. When going up in figure~\ref{fig:las}, pyramidal layers (pBLSTM) follow the initial BLSTM layer. The pyramidal structure concatenates the hidden values computed previously, such that their time dimension is halved:
\begin{equation}
\mathbf{h}_{t}^n = \text{BLSTM}(\mathbf{h}_{t-1}^{n}, [\mathbf{h}_{2t}^{n-1}, \mathbf{h}_{2t+1}^{n-1}]).
\end{equation}
Technically instead of two, three or more previously computed feature vectors could be concatenated, which increases the compression factor per pyramidal layer.
This operation reduces the length $U$ of the high level features $\mathbf{H}$. Without this compression the following attend and spell operation has a hard time extracting the relevant information, because a longer time span has to be considered to decode a single character. Additionally the compression reduces the problem complexity, which speeds up the training process.

\subsection{Attend and spell}
\label{subsec:AttendAndSpell}
The speller takes the features and produces a distribution over Latin character sequences as output. The computation of this output involves the context vector $\mathbf{c}_i$, the decoder state $\mathbf{s}_i$, the features $\mathbf{H}$ and the previous output $\mathbf{y}_i$. The index $i$ denotes decoding time, $i-1$ is used to refer to results from the last time step. The last decoding step $I$, at which the system terminates is a learned quantity. While the last input step $U$, depends on the input features, lower case $u$ denotes the input step.\\
During operation the Attend and spell functions keep track of previous output labels and previously important features, which are contained in the context. This information is stored in the state $\mathbf{s}_i$. To function the network must determine, which part of the computed features $\mathbf{H}$ are relevant at any given decoding time step $i$. The context vector $\mathbf{c}_i$ contains a linear combination of relevant features, weighted according to their importance. The AttentionContext function determines these weights based on the state. Finally the speller function finds a probability over possible labels using the relevant features in the context and the state. The computing steps are therefore \cite{Chan2015}:
\begin{align}
 \mathbf{s}_i &= \text{RNN}(\mathbf{s}_{i-1}, \mathbf{y}_{i-1}, \mathbf{c}_{i-1}), \\
 \mathbf{c}_i &= \text{AttentionContext}(\mathbf{s}_i,\mathbf{H}), \\
  P(\mathbf{y}_i|\mathbf{x}, \mathbf{y}_{<i}) &= \text{CharacterDistribution}(\mathbf{s}_i,\textbf{c}_i).
\end{align}
The state follows from a recurrent multilayer LSTM. In contrast to the listener the speller is causal, meaning that it makes decisions based only on information computed during previous decoding steps. LSTMs are necessary here, because past states must be remembered. The attention mechanism, called AttentionContext above, computes a new context vector once every time step.
This computation starts with the determination of the scalar energy $e_{i,u}$, which will be used as weight for it's corresponding feature vector  $h_u$. The computation starts with two feedforward neural networks or multilayer perceptrons (MLP), $\phi$ and $\psi$ \cite{Chan2015}:
\begin{align}
e_{i,u} = \phi(\mathbf{s}_i)^T \psi(\mathbf{h_u}), \\
\alpha_{i,u} = \frac{ \exp(e_{i,u})}{ \sum\limits_{u} \exp(e_{i,u})}, \\
\label{eq:alphas}
\mathbf{c}_i = \sum\limits_{u} \alpha_{i,u} \mathbf{h}_u.
\end{align}
$\alpha$ is produced by running $\mathbf{e}$ trough a softmax function, which scales $\mathbf{e}$ such that all elements are within $(0,1)$ and add up to one. These scaled weights, can then be used to form the context vector $\mathbf{c}_i$. When the training process converges the $\alpha_i$s typically follow a distribution with sharp edges\cite{Chan2015}. Thus it is justified to think of the alphas as a sliding window. This window contains only the currently relevant parts of the condensed input data set.

\subsection{Training}
For end-to-end speech recognition all networks must be trained jointly. The objective is to maximize the logarithmic probability:
\begin{equation}
\max\limits_\theta \sum\limits_{i} \log P(y_i | \mathbf{x}, y_{<i};\theta).
\end{equation}
Here $y_i$ denotes the current output distribution, $x$ the input, $\theta$ the various network parameters and finally $y_{<i}$ the ground truth, which is the known true desired output. In practice the objective is minimized by working with a cross entropy loss function.
Using the known output during training creates a situation, where the past outputs are always right. In practice however the situation will be different, as the network is going to make mistakes. As it is desired to create a robust model it is necessary to sometimes include the character distribution generated by the networks being trained.
Which leads to the objective \cite{Chan2015}:
\begin{align}
\hat{\mathbf{y}}_{i} = \text{CharacterDistribution}(s_i,\textbf{c}_i), \\
\max_{\theta} \sum\limits_{i} \log P(y_i|\mathbf{x}, \hat{y}_{<i};\theta).
\end{align}
The novelty in comparison to the previous expression is that $\hat{y}_{<i}$ is sometimes taken from the past network outputs instead of the ground truth. An idea which Chan et al. found in \cite{Bengio2015}.

\subsection{Decoding with beam search}
\label{sec:beamsearch}
In order to generate a readable text, it is necessary to choose characters from the generated character distributions. One way to do this is to simply pick the most likely letter from each distribution. This approach is called greedy decoding. It ignores the possibility of generating better results by also considering less likely options. It is reasonable to expect better results, when considering more then just the most likely label, because the attend and spell decoder takes past labels into account. Consequently a broader search trough the most likely options should be performed. Unfortunately memory limitations make it impossible to search trough all possible combinations. Therefore only the $n$ most likely options are explored and the rest is ignored. This approach is referred to as beam search. Taking into account the most likely options for each label produces a tree of possible transcriptions. The different routes along this tree are called hypotheses. A score for each hypothesis can be computed, by multiplication of the probability values the las-network assigned to each branch along its path. In order to account for different hypotheses lengths the total probability must be divided by the hypothesis length. In general text data is far more abundantly available than speech data. To take advantage of these large text corpora a language model trained on these can be used to make a more informed decision when choosing a beam-hypothesis. A selection can then be made according to \cite[page 6]{Chan2015}:
\begin{align}
s(\mathbf{y}|\mathbf{x}) = \frac{\log P(\mathbf{y}|\mathbf{x})}{ |\mathbf{y}|_c} + \lambda \log P_{LM}(\mathbf{y}).
\end{align}
Here $P_{LM}$ denotes the weight the language model assigns to each hypothesis. And $\lambda$ is a weight factor, which determines the language model importance. The formula above describes beam selection using a language model to re-score the attend and spell probabilities with a language model.

\section{Results}
New will present two experiments, which indicate that the re-implemented LAS system works.
\label{sec:results}
\subsection{Greedy Decoding}
We verified the attention mechanism experimentally. In a first experiment all listener LSTM states sizes were set to 64. One pyramidal layer was used in the listener. The speller's state was given a dimension of 128, and all feedforward networks in the speller where chosen to have 64 units per layer with two layers each. During training the network output reuse probability was set to $0.6$, and normal distributed noise with a standard deviation of 0.65 was added to the inputs for regularization purposes. Using momentum gradient descent with a learning rate of $10^{-3}$ and a decay term of $0.9$ as well as greedy decoding we observed a phoneme error rate of 0.55 on the timit test set after training for 40 epochs.
A plot of the attention as well at human alignments for timit utterance \texttt{fmld0\_sx295} is shown in figure~\ref{fig:fullAttention}.

\begin{figure}
\centering
\includegraphics[width=0.245\textwidth]{../tikz/alpha.pdf}
\includegraphics[width=0.21\textwidth]{../tikz/align.pdf}
\caption{Plot of the alignment vectors computed by the network for all 45 labels assigned to timit utterance \texttt{fmld0\_sx295} (left), and alignments assigned by a human listener (right).}
\label{fig:fullAttention}
\end{figure}

\subsection{Beam search and dropout}
Two measures have been taken to increase system performance. First a beam of las labels and states is kept, in order to explore several labeling hypotheses as earlier. However no language model rescoring was used.

Second we removed input noise regularization and added dropout. For all networks input neurons are dropped with a probability of $0.2$, hidden and output neurons are dropped with a probability of $0.5$. The rationale behind using dropout regularization is to allow larger networks to be trained longer without running into over-fitting problems. In comparison to the previous experiment all network parameters have been doubled. Which means that all BLSTM layers used 128 units, all LSTM layers used 265 and all feedforward layers again 128 units. The network output reuse probability during training was set 0.6.
\begin{figure}
\centering
\includegraphics[width=0.22\textwidth]{../tikz/LAS_dropout0805_in00_p06_e120_double_loss.pdf}
\includegraphics[width=0.22\textwidth]{../tikz/LAS_dropout0805_in00_p06_e120_double_error.pdf}
\caption{Dropout LAS learning curves with a listener LSTM using 128 units per direction and one PLSTM. The speller LSTM state size was set to 256. All feed-forward networks in the speller had 128 units per layer and two layers overall.}
\label{fig:dropBeamRes}
\end{figure}
The settings described above led to the learning curves shown in figure~\ref{fig:dropBeamRes}. The validation error drops consistently under 50 percent, without large outliers.

Considering once more utterance \texttt{fmld0\_sx295}.
\begin{lstlisting}[caption={Targets}]
<sos>  sil ih f sil k eh r l sil
k ah m z sil t ah m aa r ah hh
ae v er r ey n jh f er m
iy dx iy ng ih
sil t uw sil <eos>
\end{lstlisting}
The network produces:
\begin{lstlisting}[caption={Network output}]
<sos> sil hh ih f sil k ih r ow sil
k ah m sil sil t ah m aa aa hh
hh v v er ey n n sil f f er m
iy iy iy iy sil
sil t uw sil sil <eos>
\end{lstlisting}
Which bears some resemblance with the human labeling. The levenstein distance over target length ratio is 0.36 for this utterance. However over the entire test set 0.45 has been measured. This result is considerably better than the 0.55 which where observed in the previous experiment using a much simpler model with greedy decoding and input noise regularization.

\section{Conclusion}
\label{sec:conc}
Based on the experimental evidence gathered we conclude that the implemented listen attend and spell model is able to recognize speech and determine relevant parts of input recordings. The attention weights computed by the model as well as the actual labeling do resemble results obtained by human listeners. This result suggests that \cite{Chan2015} correctly claims that neural attention mechanisms can be used to align text to speech data. In terms of network regularization dropout \cite{Srivastava2014} has been found to be a useful tool. Working network parameters turned out to be scaled down versions of the ones stated in the original las paper \cite{Chan2015}. For the feedforward parameters which are not mentioned it has been found that 64 or 128 units and one hidden layer leads to convergence on timit. However the author is confident that further tuning and exploring deeper configurations will lead to additional accuracy improvements.
\bibliographystyle{IEEEbib}
%\bibliographystyle{abbrv}
\bibliography{../chapters/references}

\end{document}


\chapter{The charm-tagged analysis}
    \label{app:CharmTaggedAnalysis}

This appendix reports the main results of the charm-tagged analysis, based on $\unit[20.3]{fb^{-1}}$ of $\pp$ collisions at $\unit[8]{TeV}$.
This analysis has been published in Phys. Rev. D. 90, 052008 (2014) \cite{Aad:2014nra}, together with some of the interpretations of the monojet analysis.
The charm-tagged analysis is presented here for completeness, although it is not strictly part of this Thesis.
This analysis is interpreted in terms of stop pair production with each $\stopone$ decaying as $\stoptocharm$, and complements the exclusion limits of the monojet analysis.

Events are selected with the preselection criteria described in Section~\ref{sec:EventSelection}, with at least four reconstructed jets with $\pt>\unit[30]{GeV}$ and $|\eta|<2.5$.
The kinematics of the charm jets from the stop decays depend mainly on the $\Delta m = m_{\stopone} - m_{\ninoone}$.
As $\Delta m$ decreases, the $\pt$ of the charm jets become softer and it is more likely that other jets from initial state radiation have a higher transverse momentum than the charm jets.
As a consequence, the stop signal is expected to have relatively large jet multiplicities and a jet coming from the hadronization of a charm quark ($c$-tagged) can be found among any of the subleading jets.

Jets are $c$-tagged via a dedicated algorithm using multivariate techniques.
It combines information from the impact parameters of displaced tracks and topological properties of secondary and tertiary decay vertices reconstructed within the jet.
The algorithm provides three probabilities: one targeted for light-flavor quarks and gluon jets ($P_u$), one for charm jets ($P_c$) and one for $b$-quark jets ($P_b$).
From these three probabilities, anti-$b$ and anti-$u$ discriminators are calculated:

\begin{equation}
\begin{split}
\text{anti-}b &\equiv \log{\left(\frac{P_c}{P_b}\right)} \\
\text{anti-}u &\equiv \log{\left(\frac{P_c}{P_u}\right)}.
\end{split}
\label{eq:charmTagDiscriminators}
\end{equation}

These discriminators are then used for the selected jets in the final state.
Figure~\ref{fig:CharmTaggedDiscriminants} shows the distributions of the anti-$b$ and anti-light discriminators for the first- and third-leading jets (ordered in decreasing jet $\pt$), respectively.
The data are compared to MC simulations for the different SM processes, separated by jet flavor, and the multijet processes are estimated following the method described in Appendix~\ref{app:JetSmearingMethod}.
They include the signal preselection defined in Table~\ref{tab:SignalRegionCuts} without applying the tagging requirements.
Good agreement is observed between data and MC simulations.
Two operating points specific to $c$-tagging are used.
The \emph{medium} operating point ($\log{(P_c/P_b)}>-0.9$, $\log{(P_c/P_u)}>0.95$) has a $c$-tagging efficiency of $\approx 20\%$, and a rejection factor of $\approx 8$ for $b$-jets, $\approx 200$ for light-jets and $\approx 10$ for $\tau$-jets.
The \emph{loose} operating point is ($\log{(P_c/P_b)}>-0.9$) has a $c$-tagging efficiency of $\approx 95\%$, with a factor 2.5 rejection of $b$-jets, but with no significant rejection to light or $\tau$ jets.

\begin{figure}[!ht]
  \begin{center}
    \mbox{
      \includegraphics[width=0.495\textwidth]{Appendix_CharmTagged/Figures/can_PreselSRT_jet1logPcPb_final_jetflav.eps}
      \includegraphics[width=0.495\textwidth]{Appendix_CharmTagged/Figures/can_PreselSRT_jet3logPcPu_final_jetflav.eps}
    }
  \end{center}
  \caption[Distribution of the discriminator against $b$-jets, $\log{(P_c/P_b)}$, for the first-leading jet and against light-jets, $\log{(P_c/P_u)}$, for the third-leading jet.]
{Distribution of the discriminator against $b$-jets, $\log{(P_c/P_b)}$, for the first-leading jet and against light-jets, $\log{(P_c/P_u)}$, for the third-leading jet.
  The data are compared to MC simulations for the different SM processes, separated by jet flavor and include the same signal preselection defined in Table~\ref{tab:SignalRegionCuts} without applying the tagging requirements, which are indicated by the arrows.
  The bottom panels show the ratio between data and MC predictions. The error bands in the ratios include the statistical and experimental uncertainties in the predictions.
  For illustration purposes, the distribution of two different SUSY scenarios for stop pair production are included.
  In the SUSY signal, the first-leading jet mostly originates from ISR and the third-leading jet is expected to contain a large fraction of $c$-jets.}
  \label{fig:CharmTaggedDiscriminants}
\end{figure}

A veto against $b$-jets is applied to the selected jets in the event by using a loose $c$-tag requirement.
In addition, at least one of the three subleading jets is required to be $c$-tagged using the medium criteria.
The leading jet is required to have $\pt>\unit[290]{GeV}$ and two separate signal regions, C1 and C2, are defined with $\met>\unit[250]{GeV}$ and $\met>\unit[350]{GeV}$, respectively.
The tighter requirements on $\met$ for C2 targets models with larger stop and neutralino masses.
Table~\ref{tab:CharmTaggedSignalRegionCuts} summarizes the C1 and C2 signal region cuts.

\begin{table}[!t]
  \renewcommand{\baselinestretch}{1}
  \begin{center}
    \begin{small}
      \begin{tabular*}{\textwidth}{@{\extracolsep{\fill}}lcc}\hline\hline
        \multicolumn{3}{c}{\small{\textbf{Selection criteria}}} \\\hline
        %%% --- PRESELECTION
        \multicolumn{3}{c}{{\small{Preselection}} } \\\hline
        \multicolumn{3}{l}{Primary vertex}\\
        \multicolumn{3}{l}{$\met > 150$~GeV }\\
        \multicolumn{3}{l}{At least one jet with $\pt >150$~GeV and $|\eta|< 2.8$}\\
        \multicolumn{3}{l}{Jet quality requirements}   \\
        \multicolumn{3}{l}{Lepton vetoes}\\ \hline
        %%%% --- MONOJET
        %\multicolumn{7}{c}{\small{Monojet-like selection}}\\\hline
        %\multicolumn{7}{l}{At most a total of three jets with $\pt > 30$~GeV and $|\eta|<2.8$}\\
        %\multicolumn{7}{l}{$\Delta\phi(\text{jet}, \mathbf{p_{T}}^\text{miss}) > 0.4$}\\\hline
        %Signal region                   & M1  & M2  & M3  & M4  & M5  & M6  \\
        %Minimum leading jet $\pt$ (GeV) & 280 & 340 & 450 & 450 & 550 & 600 \\
        %Minimum $\met$ (GeV)            & 220 & 340 & 450 & 340 & 550 & 600 \\ \hline
        %%% --- C-TAGGED
        \multicolumn{3}{c}{\small{$c$-tagged selection}}\\\hline
        \multicolumn{3}{l}{At least a total of four jets with $\pt > 30$~GeV and $|\eta|<2.5$}\\
        \multicolumn{3}{l}{$\Delta\phi(\text{jet}, \mathbf{p_{T}}^\text{miss}) > 0.4$}\\
        \multicolumn{3}{l}{All four jets must pass loose tag requirements ($b$-jet vetoes)}\\
        \multicolumn{3}{l}{At least one medium charm tag in the three subleading jets}\\\hline
        Signal region        & C1 & C2   \\
        Minimum leading jet $\pt$ [GeV] & 290 & 290   \\
        Minimum $\met$ [GeV] & 250 & 350  \\ \hline\hline
      \end{tabular*}
    \end{small}
  \end{center}
  \caption[Event selection criteria applied for the signal regions of the charm-tagged analysis.]{Event selection criteria applied for the signal regions of the charm-tagged analysis.}
  \label{tab:CharmTaggedSignalRegionCuts}
\end{table}

The expected SM background for this analysis is dominated by $\znn$, $\ttbar$ and $\wln$ production, including smaller contributions from $\zll$, single-top, diboson ($WW$, $WZ$, $ZZ$) and multijet processes.

The $W/Z$ + jets processes are estimated in control regions, defined with close cuts as those from the monojet approach, with differences motivated by the background composition and the contribution from heavy-flavor jets.
A tighter cut of $\unit[81]{GeV} < m_{\mu\mu} < \unit[101]{GeV}$ is used to define the $\zmm$ + jets control sample, to further reject $\ttbar$ contamination.
This region is complemented with $\zee$+jets control sample, with the same requirements in the invariant mass, with the calorimeter clusters associated to electrons removed from the $\met$.
As in the monojet case, control samples for $\wen$+jets and $\wmn$+jets are also defined.
Due to the reduction in the statistics as a consequence of the application of the $c$-tagging, the $\met$ and leading jet $\pt$ requirements are lowered to $\unit[150]{GeV}$.

The $\ttbar$ process is estimated with a dedicated control region selecting two opposite-charge leptons ($ee$, $\mu\mu$ or $e\mu$ configurations), the same selection criteria for jet multiplicity and $c$-tagging as in the signal region, and relaxed $\met>\unit[150]{GeV}$ and leading jet $\pt>\unit[150]{GeV}$.

\begin{figure}[!ht]
  \begin{center}
    \mbox{
      \includegraphics[width=0.495\textwidth]{Appendix_CharmTagged/Figures/can_VR_Wenu_C1_met_final.eps}
      \includegraphics[width=0.495\textwidth]{Appendix_CharmTagged/Figures/can_VR_Wenu_C1_jet1PtWithEle_final.eps}
    }
    \mbox{
      \includegraphics[width=0.495\textwidth]{Appendix_CharmTagged/Figures/can_VR_Wmunu_C1_metnomu_final.eps}
      \includegraphics[width=0.495\textwidth]{Appendix_CharmTagged/Figures/can_VR_Wmunu_C1_jet1Pt_final.eps}
    }
    \mbox{
      \includegraphics[width=0.495\textwidth]{Appendix_CharmTagged/Figures/can_VR_Zll_C1_metnolep_final.eps}
      \includegraphics[width=0.495\textwidth]{Appendix_CharmTagged/Figures/can_VR_Zll_C1_jet1Pt_final.eps}
    }
  \end{center}
  \caption[$\met$ and leading jet $\pt$ distributions in the $\wen$+jets, $\wmn$+jets and $\zll$+jets control regions, after the normalization factors extracted from the fit have been applied.]{The measured $\met$ and leading jet $\pt$ distributions in the $\wen$+jets, $\wmn$+jets and $\zll$+jets control regions compared to the background predictions. The latter include the global normalization factors extracted from the fit. The error bands in the ratios include the statistical and experimental uncertainties on the background predictions.}
  \label{fig:Plot_CharmTagged_EW_Jetkinematics}
\end{figure}

\begin{figure}[!ht]
  \begin{center}
    \mbox{
      \includegraphics[width=0.495\textwidth]{Appendix_CharmTagged/Figures/can_VR_TTBarll_C1_metnomu_final.eps}
      \includegraphics[width=0.495\textwidth]{Appendix_CharmTagged/Figures/can_VR_TTBarll_C1_jet1PtWithEle_final.eps}
    }
  \end{center}
  \caption[$\met$ and leading jet $\pt$ distributions in the $\ttbar$ control region for the charm-tagged selection, after the normalization factors extracted from the fit have been applied.]{The measured $\met$ and leading jet $\pt$ distributions in the $\ttbar$ control region compared to the background predictions. The latter include the global normalization factors extracted from the fit. The error bands in the ratios include the statistical and experimental uncertainties on the background predictions.}
  \label{fig:Plot_CharmTagged_ttbar_Jetkinematics}
\end{figure}

A simultaneous likelihood fit to the $\wen$+jets, $\wmn$+jets, $\zll$+jets and $\ttbar$ control samples is performed following the techniques described in Section~\ref{sec:Fit}.
After the fit, the $W/Z$+jets backgrounds receive a multiplicative correction between 0.8 and 0.9 while the $\ttbar$ is normalized with a scale factor of 1.1.

The data and the expected background predictions for the signal regions C1 and C2 are summarized in Table~\ref{tab:CharmTaggedSignalRegions}.
Good agreement is observed between data and MC predictions.
These predictions are determined with a total uncertainty of 10\% and 14\%, respectively.
Figure~\ref{fig:Plot_CharmTagged_SR_Jetkinematics} shows the measured leading jet $\pt$ and $\met$ distributions compared to the background predictions.
For illustration purposes, the distribution of two different SUSY scenarios for stop pair production in the $\stoptocharm$ decay channel with stop masses of $\unit[200]{GeV}$ and neutralino masses of $\unit[125]{GeV}$ and $\unit[195]{GeV}$ are included.

\begin{table}[!ht]
\begin{center}
\begin{small}
\begin{tabular*}{\textwidth}{@{\extracolsep{\fill}}lrr}\hline
{\bf  Signal Region}  & \textbf{C1} & \textbf{C2}  \\
    Observed events  (20.3 fb${}^{-1}$)&  $208$  & $71$  \\ \hline

    SM prediction &  $210 \pm 21$  & $75 \pm 11$ \\ \hline

    $\wen$        &  $11 \pm 2$ &  $3.0 \pm 0.7$ \\

    $\wmn$        &  $8 \pm 2$ &  $3.0 \pm 0.7$  \\

    $\wtn$        &  $42 \pm 9$  & $14 \pm 3$ \\

    $\zee$        &  $-$   &  $-$ \\

    $\zmm$        &  $0.07 \pm 0.01$  & $0.04 \pm 0.01$  \\

    $\ztt$        &  $0.7 \pm 0.1$    & $0.15 \pm 0.03$ \\

    $\znn$        &  $62 \pm 9$ & $27 \pm 3$  \\

    $\ttbar$, single top          &     $63 \pm 13$   &  $18 \pm 4$ \\

    Dibosons      &  $21 \pm 13$  & $10 \pm 9$  \\

    Higgs         &  $0.16 \pm 0.03$  & $0.07 \pm 0.01$  \\

    Multijets     &  $2 \pm 2$  & $0.1 \pm 0.1$  \\ \hline \hline

    \end{tabular*}
    \end{small}

    \end{center}
    \caption[Data and background predictions in the signal regions C1 and C2.]{Data and background predictions in the signal regions C1 and C2.
      For the SM predictions both statistical and systematic uncertainties are included.
        Note that in each case the individual
        uncertainties can be correlated, and do not necessarily add up quadratically to the total background uncertainty.
    }
\label{tab:CharmTaggedSignalRegions}
\end{table}

\begin{figure}[!ht]
  \begin{center}
    \mbox{
      \includegraphics[width=0.495\textwidth]{Appendix_CharmTagged/Figures/can_SR_C1NoMet_met_final.eps}
      \includegraphics[width=0.495\textwidth]{Appendix_CharmTagged/Figures/can_SR_C1NoJetPt_jet1Pt_final.eps}
    }
    \mbox{
      \includegraphics[width=0.495\textwidth]{Appendix_CharmTagged/Figures/can_SR_C2NoJetPt_jet1Pt_final.eps}
      \includegraphics[width=0.495\textwidth]{Appendix_CharmTagged/Figures/can_VR_SRC2_nJet30_final.eps}
    }
  \end{center}
  \caption[Several kinematic distributions in the C1 and C2 signal regions for the charm-tagged analysis.]{(top) Measured $\met$ and leading jet $\pt$ distributions for the C1 selection before the cut in the variable shown (as indicated by the vertical arrows) is applied. In the case of the $\met$ distribution, the cuts corresponding to C1 and C2 selections are both indicated.
  (bottom) Measured leading jet $\pt$ and jet multiplicity for the C2 selection. The data are compared to the SM predictions. For illustration purposes, the distribution of two different SUSY scenarios are included. The error bands in the ratios include both the statistical and systematic uncertainties on the background predictions.}
\label{fig:Plot_CharmTagged_SR_Jetkinematics}
\end{figure}

These results are translated to exclusion limits on the pair production of top squarks with $\stoptocharm$ (BR=100\%) as a function of the stop mass for different neutralino masses.
Expected and observed exclusion limits are extracted following the $CL_s$ technique described in Chapter~\ref{chapter:StatisticalModel}, for which a simultaneous fit to the signal and control regions is performed including statistical and systematic uncertainties.

Figure~\ref{fig:ExclusionStoptocharmCharmTagged} shows the 95\% CL exclusion limits for the (best expected) combination of the signal regions C1 and C2.
The 95\% CL observed limits corresponding to the $\pm 1 \sigma$ variations on the SUSY theoretical cross sections are also added.
The charm-tagged analysis excludes the masses of the stop up to 270~GeV when $\Delta m$ is large.
The sensitivity of the analysis reduces when the $\Delta m$ decreases, due to the c-tagging requirements in the selection.
For compressed mass configurations, the charm jets are too soft to be identified and therefore, only stop masses up to 180~GeV can be excluded.

\begin{figure}[!ht]
\begin{center}
\mbox{
\includegraphics[width=0.795\textwidth]{Appendix_CharmTagged/Figures/limitPlotStop_Stop_combined_C1_C2_.eps}
}
\end{center}
\caption[Exclusion plane at 95\% CL for stop pair production with $\stoptocharm$ as a function of the $m_{\stop}$ and $m_{\ninoone}$]{Exclusion plane at 95\% CL as a function of stop and neutralino masses. The observed (red line) and expected (blue line) upper limits from this analysis are compared to previous results from Tevatron experiments~\cite{Abazov:2008rc}, and from LEP experiments~\cite{Aaltonen:2012tq} at CERN with squark mixing angle $\theta=0^{\circ}$. The dotted lines around the observed limit indicate the range of observed limits corresponding to $\pm 1 \sigma$ variations on the NLO SUSY cross section predictions. The shaded area around the expected limit indicates the expected $\pm 1 \sigma$ ranges of limits in the absence of a signal. A band for $m_{\stopone} - m_{\ninoone}< \unit[2]{GeV}$ indicates the region in the phase space for which the stop can become long-lived \protect\cite{Aad:2014nra}.}
\label{fig:ExclusionStoptocharmCharmTagged}
\end{figure}


Our work broadly falls under the category of semi-supervised sequence learning for natural language.
There are four recent trends to pay attention to: deep contextual embeddings, pre-trained language models, a reduction in task-specific architectures, and task- and domain-specific fine-tuning.

Because of their ability to transform words from a discrete representation to a continuous one, Word2vec~\citep{mikolov2013efficient} was a standard component in many NLP systems.
However, their context-independent nature limited their usefulness.
For example, an embedding for the word ‘play’ is identical when used to describe a sporting event as for actors putting on a dramatic work.
% Maybe also add a comment on how the method is linear (shallow), not deep.

One of the first methods to encode context in words was proposed by~\cite{dai2015semisupervised}.
Using unlabeled data, they pre-trained a conventional language model using a next-word prediction objective.
After fine-tuning the model on multiple sentiment classification tasks, they achieved best in class results.
The method had limitations, though.
By using a long short-term memory (LSTM) architecture, gradients became unstable for long documents.

~\cite{peters2018deep} built upon this approach.
Using a similar pre-training objective, they pre-train a large language model on a dataset of 30 million sentences.
However, they did not fine-tune the language mode.
Instead they used the resulting model as a feature-extractor for task-specific architectures.
After pre-training, they feed sentences into the language model.
The output is a deep-contextual character embedding.
They now discard the language model.
Next, the character-level features are passed into a task-specific architecture.
This design allowed them to achieve state-of-the-art results on an array of natural language tasks (SQuAD, SNLI, SRL, Coref, NER, SST-5) by solely swapping out Word2vec embeddings for ELMo embeddings.

It wasn’t until GPT-1 where task-specific architectures begin to be discarded.
Using a transformer architecture,~\cite{radford2018improving} pre-trained a language model.
Their use of transformers allowed the model to learn longer range dependencies in the text.
They then use the entire pre-trained model as a base for supervised models.
They showed that a pre-trained model, with little adaptation, can transfer learned representations to a multitude of NLP tasks.
GPT-1 improved state-of-the art across 9 of 12 NLP benchmarks.
These tasks covered 4 broad categories: natural language inference, question answering, semantic similarity, and text classification.
However, the language model struggled on tasks such as part-of-speech tagging.
This is because of the left-to-right masking and next-word-prediction pre-training objective.
It did solidify the approach of using task-agnostic architecture in contrast to ELMo’s feature extraction method.

~\cite{DBLP:journals/corr/abs-1810-04805} addresses GPT-1’s left-to-right limitation by introducing a new pre-training objective.
Instead of predicting the next word, they train their BERT model on with a Masked Language Modeling (MLM) objective.
This allows the language model to benefit from both left-context, right-context, or a combination of both.
They also train the model on the largest dataset to date.
By combining bidirectional pre-training and an internet-scale dataset, they achieved state-of-the-art on 11 NLP tasks.
This was the strongest demonstration to date that language models are compute-bound.
BERT combines the best of previous approaches: a large pre-trained language model that learned deep contextual embeddings, and fine-tuning a general purpose model.

%In one of the early methods to...
%One of the early papers to leverage semi-supervised sequence learning

%Big trends
%- Deep contextual embeddings
%- Pre-trained language models
%- Reduction in task-specific architectures
%- Task and domain specific fine-tuning.

%Over the last few years, researchers have ...(demonstrated?)

%Recent approaches have investigated...

%\subsection{Deep Learning Approaches to Aspect-Based Sentiment Analysis}\label{subsec:deep-learning-approaches-to-aspect-based-sentiment-analysis}

%... is a special case of ...

%Early works ...

%In recent work ...

%Research into aspect based sentiment analysis .... following methodolgies (approaches).

%The datasets typically used for Aspect-Based Sentiment Analysis are ...

%(Early work) Early works on ABSA ....

%(Recent DL improvements) Recent advances in deep contextual embeddings.

%Trend towards pre-trained models. with fine-tuning.

%Since the early works on ABSA [18], [19], [20], several methods have been put forward to address the problem. In this section, we review some of the works which have utilized deep learning techniques.

%(How recent leverages) Recent methods leverage this approach.


\newpage
\section{Dynamic network analysis}

  \subsection{Discretisation of data}

    Because the messages are posted on this forum virtually every minute the information about them might be considered as being continuous in time. In order to study the evolution of the network in time, there is a need to define discrete time intervals that will split the network in parts. Networks that are dynamic are called longitudinal networks.
    \\\\
    The time interval using which the data had to be split had to be determined empirically with a network to be kept in a size that would be representational. The choice was between a \emph{week-long}, \emph{two weeks}-long and a \emph{month}-long interval. Eventually, a \emph{$30$ day}-long time interval was selected, main reason being the fact that the date span of downloaded messages was almost 12 years long which created 141 data buckets. Additionally, message threads are often more than a week long, so we did want to avoid the situation where most threads are split into several buckets. It also allowed different kind of distributions stand out more because they were created with more data.

    The biggest downside was that the network sizes were still very big, thus all computations that had to be performed on a whole network took more time.

  \subsection{Estimating users' opinions about products} \label{sec:opinion_estimation}

    In order to estimate the opinions different users have about different products I had to prepare a script traversing the posts saved in my database and saving a value representing opinion. I had to create a separate table in my database for that, which structure is shown in table \ref{tab:db_opinions}.
    \begin{table}[H]
      \begin{tabularx}{\textwidth}{|L{0.3}|L{1.7}|} \hline
        \rowcolor[gray]{0.75} \textbf{Field} & \textbf{Description} \\\hline
        user\_id & Unique user reference; found in users table as id. \\
        word\_id & Unique word reference; found in words table as id. \\
        opinion & A real value representing opinion. \\
        bucket & Bucket number from which the opinion value was calculated. \\\hline
      \end{tabularx}
      \caption{Opinions table structure.}
      \label{tab:db_opinions}
    \end{table}

    The script was based on algorithm \ref{alg:estimate_user_keyword}. I have run it against every bucket, first gathering the list of users and keywords in that bucket.
    The algorithm was ran for every user and keyword. It fetched all messages written by the user in a bucket it was ran for. Then for every post it has checked whether the keyword was used within the post and returned position of each occurrence of that word. The position was established to be the $n$th \textquote{token} in the message, where \textquote{tokens} were obtained in the exact same way as words in section \ref{sec:text_segmentation}. Then the algorithm extracted all words marked as opinions from the message body and for each of these opinions it has checked their position as it did with keywords. Next, it has checked whether the opinion was \emph{positive} or \emph{negative} and applied a measure on a keyword based on squared distance between itself and the opinion biased positively or negatively.
    \begin{algorithm}[H]
      \begin{algorithmic}[1]
        \Procedure{EstimateOpinion}{user, keyword, bucket}
          \State estimate $\gets 0$
          \State posts $\gets$ \textsc{Posts}(user, bucket)
          \ForAll{post \textbf{in} posts}
            \State keywordPositions $\gets$ \textsc{KeywordPositions}(keyword, post)
            \ForAll{keywordPosition \textbf{in} keywordPositions}
              \State opinions $\gets$ \textsc{Opinions}(post)
              \ForAll{opinion \textbf{in} opinions}
                \State opinionSign $\gets$ \textsc{OpinionSign}(opinion)
                \State opinionPosition $\gets$ \textsc{OpinionPosition}(opinion, post)
                \State distance $\gets$ |keywordPosition - opinionPosition|
                \State estimate $\gets$ estimate + (opinionSign / distance$^2$)
              \EndFor
            \EndFor
          \EndFor
          \State \textbf{return} estimate
        \EndProcedure
      \end{algorithmic}
      \caption{Estimating user's opinion about product.}
      \label{alg:estimate_user_keyword}
    \end{algorithm}

    \subsubsection{Measure}

      Let $e_o$ be an opinion estimate obtained using a method described above. Its value should be found in a range described by the following sum
      \begin{equation}
        \frac{1}{1^2} + \frac{1}{2^2} + \frac{1}{3^2} + \frac{1}{4^2} + \ldots = \sum_{n=1}^{\infty} \frac{1}{n^2} = \frac{\pi^2}{6} \mbox{,} \label{eqn:estimate}
      \end{equation}
      and as each keyword can have opinions written before and after itself, we have two multiply the sum in equation \ref{eqn:estimate} by a factor of $2$. Hence,
      \begin{equation}
        -\frac{\pi^2}{3} \leq e_o \leq \frac{\pi^2}{3} \mbox{.} \label{eqn:estimate_limits}
      \end{equation}

      Application of squared distance as a base for the measure favours opinions that are very close to keywords. This was extremely important especially for long posts so that keywords mentioned at the end of the message did not get influenced by an opinion mentioned at the beginning of it. With this approach it is expected for the most opinions to be found in a range of $[\sfrac{1}{4};\sfrac{1}{4}]$---most opinions would be generally no further away than 2 words from a keyword.

      Because measuring data in such way is not perfect and may create results contradictory to the real data I have decided to introduce a \emph{neutral} opinion using the middle range of the numbers. Hence, opinions were assigned in a following way:
      \begin{itemize}
        \item \emph{positive:} $L \leq e_o \leq \frac{\pi^2}{3}$,
        \item \emph{neutral:} $-L < e_o < L$,
        \item \emph{negative:} $-\frac{\pi^2}{3} \leq e_o \leq L$,
      \end{itemize}
      where $L = 0.01$ is an empirically found constant allowing for fairly even division of opinion types. Table \ref{tab:opinion_number} shows the number of opinions found, divided by the type of the opinion.
      \begin{table}[H]
        \centering
        \begin{tabularx}{\textwidth}{|L{1}|L{1}|L{1}|L{1}|} \hline
          \rowcolor[gray]{0.8} \textbf{Positive} & \textbf{Neutral} & \textbf{Negative} & \textbf{Total} \\\hline
          $4,219,548\;(42.99\%)$ & $3,398,282\;(34.62\%)$ & $2,197,055\;(22.39\%)$ & $9,814,885\;(100\%)$ \\\hline
        \end{tabularx}
        \caption{Number of opinions about products found, by the opinion type.}
        \label{tab:opinion_number}
      \end{table}

      \begin{figure}[H]
        \centering
        \includegraphics[width=0.75\textwidth]{chapters/03_implementation/opidist}
        \caption{Distribution of opinions.}
        \label{fig:opinion_distribution_reg}
      \end{figure}
      The maximum and minimum value of the estimated opinion were found to be approximately $2.924$ and $-2.825$ respectively, so they lay within the limits shown in equation \ref{eqn:estimate_limits}. Figure \ref{fig:opinion_distribution_reg} shows distribution of estimated opinion values. As expected, most of the opinions fall in the $[\sfrac{1}{4};\sfrac{1}{4}]$ range.

      There are also visibly more opinions on the positive side of the scale than on its negative counterpart. The most dense is the neutral region but it is the consequence of having so much opinions in only $0.02$ wide scale. \textquote{Harmonic}-like peaks are conspicuous at $-1$, $-\sfrac{1}{4}$, $\sfrac{1}{4}$ and $1$, which one can see closer on figure \ref{fig:opinion_distribution_log} using a logarithmic scale for $Y$ axis. Additionally we can see peaks at $-2$ and $2$ which were not distinguishable on a linear scale.
      \begin{figure}[H]
        \centering
        \includegraphics[width=0.75\textwidth]{chapters/03_implementation/opidist_log}
        \caption{Distribution of opinions with $Y$ axis in logarithmic scale.}
        \label{fig:opinion_distribution_log}
      \end{figure}
      The visible peaks are a clear evidence that the majority of messages with opinions are short enough for only several opinions influencing a keyword. Otherwise, the distribution would have had a more smooth, cone-like shape. The perfect cone-like shape would be achieved if posts were consisting of one keyword only with a lot of opinions written on both sides of it. Also, the advantage of positive opinions is more evident using this scale.

    \subsubsection{Problems}

      Checking \emph{opinion sign} is not as easy as determining whether the opinion is positive or negative. The problem is that there exist modifier words that completely change the meaning of word. Examples of words like that include \emph{not} or \emph{don't}.

      In order to mitigate this problem and to bias the opinions correctly it was necessary to check for existence of such modifier words in a proximity of opinion words.

      Following expressions illustrate the problem: \textquote{\emph{not} \textbf{good},} \textquote{\emph{not} sufficient \textbf{good},} \textquote{\emph{not} at all \textbf{good}.}
      One can clearly see that it is not necessary to check just one word before an opinion. I have decided to cover all three cases like that even though there may be more edge cases. Hence, modifier words were only checked as far as two words from an opinion and were not taken into consideration if there were another opinion or keyword in between them, for example in messages like \textquote{\emph{not} \textbf{bad}, \textbf{good} even,} where not stopping after one opinion have been found would cause the modifier to act on both opinions instead.

  \subsection{Average path length}

    Figure \ref{fig:avg_path} shows changes of average path length calculated for each bucket of data. The general trend shows that the average length of paths in the network has been decreasing. This may be due to the fact that most of the users did not leave the forum and continued to write messages while retaining their connections. Nevertheless the path lengths are much higher, in a range between $3.27$ and $3.55$, than the one calculated for the whole network which was calculated at $2.68$. This is clearly because of the whole network being significantly larger (having more edges), thus having higher probability for connections to exist.
    \begin{figure}[H]
      \centering
      \includegraphics[width=\textwidth]{chapters/03_implementation/avg_path}
      \caption{Changes of average path length over time.}
      \label{fig:avg_path}
    \end{figure}

  \subsection{Collaborative Similarity} \label{sec:cs}

    Ming-Sheng Shang \textit{et al.} have proposed a new index called \emph{collaborative similarity}, to quantify the diversity of tastes based on the collaborative selection\cite{Shang2010}. This newly proposed index sounds promising for the data under analysis in this thesis. It may be a good idea to explore and look at the \emph{user-message network} from the other angle and if we can find that the user selection is highly clustered---i.e. not that much diverse---then we will be able to easily identify main groups of users.

    \subsubsection{The concept}

      Figure \ref{fig:cs_network} illustrates a small bipartite network that consists of six users and eight objects. The degree of user $i$, denoted by $k_i$, is defined as the number of objects connected to $i$. Analogously, the degree of object $\alpha$, denoted by $d_\alpha$, is the number of users connected to $\alpha$. For example, as shown in figure \ref{fig:cs_network}, $k_i = d_\alpha = 3$. The density function, $p(k)$, is the probability that a randomly selected user is of degree $k$, while the cumulative function, $P(k)$, denotes the probability that a randomly selected user is of degree no less than $k$.
      \begin{figure}[h]
        \centering
        \includegraphics[width=0.6\textwidth]{chapters/03_implementation/cs_network}
        \caption{Small user-message bipartite network.}
        \label{fig:cs_network}
      \end{figure}

      The \emph{nearest neighbors’ degree} for user $i$, denoted by $d_{nn}(i)$, is defined as the average degree over all the objects connected to $i$. For example, as shown in figure \ref{fig:cs_network},
      \begin{equation}
        d_{nn}(i) = \frac{d_\alpha + d_\beta + d_\gamma}{3} = \frac{7}{3}\mbox{.}
      \end{equation}

      The degree-dependent nearest neighbors’ degree, $d_{nn}(k)$ is the average nearest neighbors’ degree over all the users of degree $k$, that is,
      \begin{equation}
        d_{nn}(k) = \langle d_{nn}(i) \rangle_{k_i=k}\mbox{.}
      \end{equation}

      Corresponding definitions for objects, say $p(d)$, $P(d)$, $k_{nn}(\alpha)$ and $k_{nn}(d)$, are similar and thus omitted here.

      The traditional clustering coefficient\cite{WattsStrogatz1998} cannot be used to quantify the clustering pattern of a bipartite network since it always give a zero value. Lind \textit{et al.}\cite{LindGonzalezHerrmann2005} proposed a variant counting the rectangular relations instead of triadic clustering, which can be applied to general bipartite networks. However, this letter aims at a special class of bipartite networks, and thus we propose a new index to characterise the clustering selections\footnote{Here the term \textquote{clustering} describes the fact that a user's selections are usually very similar to each other, and may belong to a few clusters or communities according to the standard clustering analysis or community detection.} resulted from the collaborative interests of users. A standard measure of object similarity according to the collaborative selection is the \emph{Jaccard similarity}\cite{Jaccard1901}.
      \begin{equation}
        s_{\alpha\beta} = \frac{\Gamma_\alpha \cap \Gamma_\beta}{\Gamma_\alpha \cup \Gamma_\beta}\mbox{,}
      \end{equation}
      where $\Gamma_\alpha$ and $\Gamma_\beta$ are the sets of neighbouring nodes of $\alpha$ and $\beta$, respectively. Obviously, $s_{\alpha\beta} = s_{\beta\alpha}$ and $0 \leq s_{\alpha\beta} \leq 1$ for any $\alpha$ and $\beta$. For example, as shown in figure \ref{fig:cs_network}, $s_{\alpha\beta} = s_{\beta\gamma} = \sfrac{1}{3}$ and $s_{\alpha\gamma} = \sfrac{1}{2}$. The collaborative similarity of user $i$ is then defined as the average similarity between $i$'s selected objects:
      \begin{equation}
        C_u(i) = \frac{1}{k_i(k_i-1)} \sum_{\alpha\neq\beta} s_{\alpha\beta}\mbox{,}
      \end{equation}
      where $\alpha$ and $\beta$ run over all $i$'s neighbouring objects. For example, as shown in figure \ref{fig:cs_network}, the collaborative similarity of user $i$ is $C_u(i) = \sfrac{7}{18}$. According to the definition, a user whose collections are very similar to each other will have high collaborative similarity. For example, a user who only watches science fiction movies is probably of higher collaborative similarity than the one who has very diverse interests of movies. The user collaborative similarity of the whole network is defined as
      \begin{equation}
        C_u = \frac{1}{N^\prime} \sum_{i} C_u(i)\mbox{,}
      \end{equation}
      where $i$ runs over all users with degrees larger than $1$ and $N^\prime$ denotes the number of these users. The degree-dependent collaborative similarity, $C_u(k)$, is defined as the average collaborative similarity over all the $k$-degree users. Corresponding definitions for objects are as following:
      \begin{equation}
        C_o(\alpha) = \frac{1}{d_\alpha(d_\alpha-1)} \sum_{i\neq j} s_{ij}\mbox{,}
      \end{equation}
      where $s_{ij} = \left| \frac{\Gamma_i \cap \Gamma_j}{\Gamma_i \cup \Gamma_j} \right|$ is the Jaccard similarity between users $i$ and $j$ and:
      \begin{equation}
        C_o = \frac{1}{M^\prime} \sum_\alpha C_o(\alpha)\mbox{,}
      \end{equation}
      where $M^\prime$ denotes the number of objects with degrees larger than $1$. $C_o(d)$ is the average collaborative similarity over all the $d$-degree objects.

    \subsubsection{Results}

      \begin{table}[h]
        \begin{tabularx}{\textwidth}{|C{1.1}|C{0.95}|C{0.95}|C{1}|C{1}|C{1}|C{1}|C{1}|C{1}|C{1}|} \hline
          \rowcolor[gray]{0.75} \textbf{Bucket} & $N$ & $M$ & $E$ & $\langle k \rangle$ & $\langle d \rangle$ & $C_u$ & $\bar s_m$ & $C_m$ & $\bar s_u$ \\\hline
$1$ & $193$ & $2616$ & $29560$ & $78.02$ & $5.76$ & $0.1608$ & $0.1046$ & $0.1278$ & $0.1453$ \\\hline
$11$ & $417$ & $3647$ & $55201$ & $68.54$ & $7.84$ & $0.1376$ & $0.0955$ & $0.1182$ & $0.1148$ \\\hline
$21$ & $609$ & $3787$ & $73964$ & $62.79$ & $10.10$ & $0.1253$ & $0.0936$ & $0.1167$ & $0.1028$ \\\hline
$31$ & $718$ & $4027$ & $89545$ & $63.74$ & $11.37$ & $0.1235$ & $0.0926$ & $0.1166$ & $0.0986$ \\\hline
$41$ & $942$ & $4247$ & $93741$ & $55.60$ & $12.33$ & $0.1070$ & $0.0860$ & $0.1047$ & $0.0857$ \\\hline
$51$ & $1318$ & $4622$ & $118121$ & $51.71$ & $14.75$ & $0.0975$ & $0.0812$ & $0.0995$ & $0.0768$ \\\hline
$61$ & $1966$ & $5232$ & $191560$ & $54.71$ & $20.56$ & $0.0988$ & $0.0849$ & $0.1033$ & $0.0747$ \\\hline
$71$ & $1701$ & $5095$ & $89352$ & $52.53$ & $17.54$ & $0.0975$ & $0.0800$ & $0.0996$ & $0.0741$ \\\hline
$81$ & $1905$ & $5090$ & $91795$ & $48.19$ & $18.03$ & $0.0932$ & $0.0787$ & $0.0985$ & $0.0711$ \\\hline
$91$ & $1680$ & $4774$ & $77094$ & $45.89$ & $16.15$ & $0.0900$ & $0.0775$ & $0.0935$ & $0.0736$ \\\hline
$101$ & $1817$ & $5544$ & $105478$ & $58.05$ & $19.03$ & $0.1096$ & $0.0867$ & $0.1103$ & $0.0799$ \\\hline
$111$ & $1743$ & $4989$ & $81130$ & $46.55$ & $16.26$ & $0.0918$ & $0.0777$ & $0.0961$ & $0.0716$ \\\hline
$121$ & $1706$ & $4900$ & $81357$ & $47.69$ & $16.60$ & $0.0968$ & $0.0784$ & $0.1000$ & $0.0755$ \\\hline
$131$ & $1994$ & $5562$ & $104444$ & $52.38$ & $18.78$ & $0.0986$ & $0.0796$ & $0.1011$ & $0.0732$ \\\hline
$141$ & $2080$ & $5250$ & $94994$ & $45.67$ & $18.09$ & $0.0915$ & $0.0757$ & $0.0962$ & $0.0712$ \\\hline        \end{tabularx}
        \caption{The basic properties of every 10th data bucket.}
        \label{tab:cs_properties}
      \end{table}

      Table \ref{tab:cs_properties} summarises basic statistics of selected data buckets. $N$, $M$ and $E$ denote the number of users, messages and edges connecting them, respectively. $\langle k \rangle$ and $\langle d \rangle$ are the average user degree and average message degree. $C_u$ and $C_m$ are the average collaborative similarity for users and messages, and for comparison, $\bar s_m$ and $\bar s_u$ are the average similarities over all message pairs and over all user pairs, respectively. The user selection is considered to be poorly clustered (i.e., more diverse) because the condition for that is $C_u \gg \bar s_m$ whereas $C_u$ is the same order of magnitude as $\bar s_m$. Unfortunately, this basically means that it will not be easy to identify groups of users that may have formed in this message board and I had to abandon this somewhat interesting index. Figure \ref{fig:cs_cusm} show the way $C_u$ and $\bar s_m$ were changing over time. In no bucket, $C_u$ was much greater than $\bar s_m$ and even in bucket 84 they were pretty close to each other: $0.0746$ vs $0.0683$ respectively.
      This does not however disqualify that method completely. The results only mean that it will not be easy to find any anomalies. There are $52605$ users in the database so I could just run over a $C_u(i)$ of every user and look for suspicious changes. However strenuous it sounds, it may be worth it. What we are looking for are users whose $C_u(i)$ is significantly different than it was before, incidating possible \emph{marketing campaign}.
      \begin{figure}[H]
        \includegraphics[width=0.9\textwidth]{chapters/03_implementation/C_us_m}
        \caption{Changes of $C_u$ and $\bar s_m$ over time.}
        \label{fig:cs_cusm}
      \end{figure}

      Figure \ref{fig:cs_dist_full} represents distribution of $C_u(i)$ across all users. Where $C_u(i)$ was in a range $[0, 1]$ it was stretched to $[0, 100]$ to create $101$ distinct buckets so that the distribution may have been calculated. Large number of nodes in \textquote{$0$} bucket constitutes with rather big amount of users with the degree less than or equal to $1$, where $C_u(i)$ is calculated as $0$ for such degree. Figure \ref{fig:cs_dist_tail} shows the tail of the distribution from figure \ref{fig:cs_dist_full} as it is not visible due to the scale used.
      \begin{figure}[H]
        \centering
        \begin{subfigure}[b]{0.49\textwidth}
          \centering
          \includegraphics[width=\textwidth]{chapters/03_implementation/cs_dist}
          \caption{Full scale.}
          \label{fig:cs_dist_full}
        \end{subfigure}
        \begin{subfigure}[b]{0.49\textwidth}
          \centering
          \includegraphics[width=\textwidth]{chapters/03_implementation/cs_dist_tail}
          \caption{The tail.}
          \label{fig:cs_dist_tail}
        \end{subfigure}
        \caption{Distribution of $C_u(i)$ across all users.}
      \end{figure}

      Figure \ref{fig:cu_12195} shows changes of $C_u(i)$ for user $i=12195$. One can clearly tell that the collaborative similarity for that user suddenly peaks three times as it was before. The next step would involve looking at the individual messages posted by that user during the time the peak occurred (bucket number 107).
      \begin{figure}[h!]
        \centering
        \includegraphics[width=0.8\textwidth]{chapters/03_implementation/u12195}
        \caption{Changes of $C_u(i)$ for $i=12195$.}
        \label{fig:cu_12195}
      \end{figure}

      After reading the posts, it looks like one of the forum users started a \textquote{headphone brand elimination game!} topic on July 27, 2009 that was active until August 25, 2009 (buckets 106-107). The rules of the game were simple:
      \begin{enumerate}
        \item The topic started with brand names written down along with initial 10 health points.
        \item Users were supposed to post a message containing one brand they want to hurt or heal, adding or removing one health point respectively.
        \item Health points did not stack above 20.
        \item A brand would lose---thus removed from the list effectively---once it was left with 0 health points.
        \item Once four brands were left on the battlefield, their health was restored according to some other rules and the battle followed until one winner has left standing.
      \end{enumerate}
      The battle struggled between brands \emph{Sennheiser} and \emph{STAX} for a longer period, with the former eventually winning 20-0. I think that the analysis of this topic alone would bring some interesting results. The only problem is that, because of this topic, I had to completely skip the results for buckets 106 and 107 due to the fact that the results would be distorted.

    \subsubsection{$C_u(i)$ fluctuation patterns found} \label{sec:cs_fluctuations}

      Figure \ref{fig:cs_fluctuations} shows various types of patters of $C_u(i)$ fluctuations that seem to repeat throughout individual users. I will not try to interpret these as it would require further, thorough examination which I did not perform due to time constraints.

      \begin{figure}[H]
        \centering
        \begin{subfigure}[b]{0.32\textwidth}
          \centering
          \includegraphics[width=\textwidth]{chapters/03_implementation/u1}
        \end{subfigure}
        \begin{subfigure}[b]{0.32\textwidth}
          \centering
          \includegraphics[width=\textwidth]{chapters/03_implementation/u217}
        \end{subfigure}
        \begin{subfigure}[b]{0.32\textwidth}
          \centering
          \includegraphics[width=\textwidth]{chapters/03_implementation/u455}
        \end{subfigure}
        \begin{subfigure}[b]{0.32\textwidth}
          \centering
          \includegraphics[width=\textwidth]{chapters/03_implementation/u33338}
        \end{subfigure}
        \begin{subfigure}[b]{0.32\textwidth}
          \centering
          \includegraphics[width=\textwidth]{chapters/03_implementation/u61222}
        \end{subfigure}
        \begin{subfigure}[b]{0.32\textwidth}
          \centering
          \includegraphics[width=\textwidth]{chapters/03_implementation/u72553}
        \end{subfigure}
        \begin{subfigure}[b]{0.32\textwidth}
          \centering
          \includegraphics[width=\textwidth]{chapters/03_implementation/u112071}
        \end{subfigure}
        \begin{subfigure}[b]{0.32\textwidth}
          \centering
          \includegraphics[width=\textwidth]{chapters/03_implementation/u113353}
        \end{subfigure}
        \begin{subfigure}[b]{0.32\textwidth}
          \centering
          \includegraphics[width=\textwidth]{chapters/03_implementation/u137702}
        \end{subfigure}
        \caption{Patterns of $C_u(i)$ fluctuations.}
        \label{fig:cs_fluctuations}
      \end{figure}

  \subsection{Motifs}

    \subsubsection{Introduction}

      One of the methods that I was sure will bring up some groups of users was the motif discovery within the network. It was found that real world network often feature recurrent sub-graphs or patterns. \emph{Motifs} are sub-graphs that repeat themselves in the network or even in networks of the same type. Indeed, motifs are of notable importance largely because they may reflect functional properties. They have recently gathered much attention as a useful concept to uncover structural design principles of complex networks\cite{MasoudiSchreiberKashani2012}.

      Network motifs were at first defined systematically in \emph{Escherichia coli}, where they were detected as patterns that occurred in the transcription network much more often than would be expected in random networks\cite{MiloAlon2002}. The exact same motifs have been later found in other organisms, ranging from other bacteria\cite{ManganZaslaverAlon2003,Eichenberger2004} and yeast\cite{MiloAlon2002, Lee2002}, to plants\cite{Saddic2006} and animals\cite{Boyer2005}. Motif discovery have been the most successful in biological networks, most likely because they were the most studied at the time---especially the transctiption networks. I will not explain many details about the motifs, because this is a complicated matter and already has been discussed and reviewed thoroughly\cite{Alon2007}.

    \subsubsection{Motif discovery}

      Although network motifs may provide a deep insight into the network's functional abilities, their detection is computationally challenging. Because of that, I have decided to only concentrate on the most basic, triangle motifs, that I have discovered using not very sophisticated algorithm conjoining two users sharing similar opinions about particular brands, hoping that this approach will lead to some interesting opportunities. This also allowed me to map the bipartite network I was working on so far to a regular network built on a set of users exchanging opinions only. Because I have already joined users with brands they were discussing based on the opinion being positive, negative or neutral, the only step required to obtain these triangles was to link users with matching opinions. Data required for motif discovery has been obtained in section \ref{sec:opinion_estimation}.

      First, the algorithm builds a bipartite network with users and opinions about the brands. Edges (solid lines) represent the opinion the user has about the product. It is not possible to have more than one edge between the nodes, because all opinions about the brand users may have are averaged and only this average number is taken into account during network creation. When the network is ready, algorithm finds user nodes having edges to the same brand nodes and links them together (dashed lines). The final step involves deletion of initial edges along with brand nodes resulting in non-bipartite network consisting of user nodes only. Figure \ref{fig:motifs_alg} illustrates steps taken by the algorithm.
      \begin{figure}[h]
        \centering
        \begin{subfigure}[b]{0.3\textwidth}
          \includegraphics[width=\textwidth]{chapters/03_implementation/cs_motif_1}
          \caption{Step 1.}
          \label{fig:motifs_alg_1}
        \end{subfigure}
        \quad
        \begin{subfigure}[b]{0.3\textwidth}
          \includegraphics[width=\textwidth]{chapters/03_implementation/cs_motif_2}
          \caption{Step 2.}
          \label{fig:motifs_alg_2}
        \end{subfigure}
        \quad
        \begin{subfigure}[b]{0.3\textwidth}
          \includegraphics[width=\textwidth]{chapters/03_implementation/cs_motif_3}
          \caption{Step 3.}
          \label{fig:motifs_alg_3}
        \end{subfigure}
        \caption{Algorithms steps.}
        \label{fig:motifs_alg}
      \end{figure}
      Graphs in figures \ref{fig:motifs_alg_1} and \ref{fig:motifs_alg_2} are bipartite and in figure \ref{fig:motifs_alg_3} the graph is a non-bipartite graph projected from bipartite network. Circled nodes represent users and squared nodes represent words (brand names). Solid edges show which words were used by which users and dashed edges serve as an opinion shared between two users about a brand.

    \subsubsection{Substructures derived from motif analysis}

      Figure \ref{fig:motif_network} shows a fragment of network created with the above method from bucket 106 which was excluded before in section \ref{sec:cs_results}. The network is a multigraph with nodes representing the users and edges representing an opinion about a brand shared between two users.
      \begin{figure}[b!]
        \centering
        \includegraphics[width=0.66\textwidth]{chapters/03_implementation/motif_network}
        \caption{Network obtained with motif discovery.}
        \label{fig:motif_network}
      \end{figure}

      The network was constructed using opinions about 10 brands presented in table \ref{tab:motif_brands} ordered by the number of times they were used in that bucket. Even though \emph{Stax} won the game mentioned in section \ref{sec:cs_results} it was not discussed between the users in an essential manner. The number of opinions about that brand was so low that it did not matter in the whole network. The size of each node was determined from it's betweenness centrality to stress the importance of certain nodes in this network despite the fact that degrees are fairly equal and distributed uniformly in a range $[100:216]$.

      Three brands were discussed in a particularly frequent manner between the users, thus the network consisted mostly of opinions about them. Hence, the classes of users presented in figure \ref{fig:motif_network} are made of three brands discussed the most: \emph{Grado}, \emph{Sennheiser} and \emph{Denon}. Grado was discussed the most (see table \ref{tab:motif_brands}), but only $18.93\%$ of shared opinions in the network are about this brand. This means that the opinions about Grado (green edges) are inconsistent, so users sharing opinion about Sennheiser or Denon does not share opinion about Grado in general.
      \begin{table}[H]
        \centering
        \begin{tabularx}{0.6\textwidth}{|L{0.6}|L{1.2}|L{1.2}|} \hline
          \rowcolor[gray]{0.8} \textbf{Position} & \textbf{Brand} & \textbf{Times used} \\\hline
          1 & Grado & $1,948$ \\\hline
          2 & Sennheiser & $1,408$ \\\hline
          3 & Denon & $881$ \\\hline
          4 & AKG & $804$ \\\hline
          5 & Stax & $560$ \\\hline
          6 & Beyerdynamic & $423$ \\\hline
          7 & Sony & $411$ \\\hline
          8 & Yamaha & $364$ \\\hline
          9 & Bose & $212$ \\\hline
          10 & Panasonic & $132$ \\\hline
        \end{tabularx}
        \caption{Brands used to create network shown in figure \ref{fig:motif_network}.}
        \label{tab:motif_brands}
      \end{table}
      While Denon was the least discussed from \textquote{the great three} it certainly indicates that users' opinions are reciprocal with over half ($53.56\%$) of the network edges (red coloured) joining users. Opinions about Sennheiser (blue links) form the other $27.51\%$ of the network. Vertices were coloured with the colour of edges that were linked to that vertex the most.

      Particularly interesting is the case of users that either do not discuss some brands (more likely) or have no common opinions about that brand (much less likely) with other users. This means that a substructure may exist where users are focused on several brands only and do not discuss others at all. The conclusions here might be too far-reaching, but this idea might be interesting to tackle.

      The average path length is $1.143$ which is extremely short and shows how efficient the network is. The average clustering coefficient for this network was calculated as $0.891$. The graph is nearly complete with a density being equal to $0.857$ meaning that almost every user in this network share a similar opinion with another one. The network was meant to be maximally dense, so I would say that the missing $0.143$ density constitute a contradictory opinions.


\chapter{Concept}
\label{ch:Concept}

\abstract{The first section of the third chapter describes the existing application this evaluation is based on. In addition the various phases of the development process are roughly illustrated.
}

\section{Overview of the Case Study \textit{streets4MPI}}
\label{sec:Concept::Overview}

As stated in~\autoref{sec:Introduction::Goals} the concept for the implementations to compare is inspired by \textit{streets4MPI}, which was implemented to evaluate Python's usefulness for ``computational intensive parallel applications''~\cite[p.3]{streets_report}. It was written by Julian Fietkau and Joachim Nitschke in scope of the module ``Parallel Programming'' in Spring 2012 and makes heavy use of the various libraries of the Python ecosystem. \autoref{fig:architecture_streets4mpi} provides a rough overview about the architecture of \textit{streets4MPI}.

\begin{figure}[htb]
    \centering
    \includegraphics[width=.75\textwidth]{img/architecture_streets4mpi.png}
    \caption{Architecture overview: Streets4MPI~\cite[p. 9]{streets_report}}
    \label{fig:architecture_streets4mpi}
\end{figure}

The \textit{GraphBuilder} class parses \gls{osm} input data and builds a directed graph which is stored in the \textit{StreetNetwork}. The \textit{Simulation} than uses this data and repeatedly computes shortest paths for a set amount of \textit{trips} (randomly chosen node pairs from the graph). Over time it gradually modifies the graph based on results of previous iterations to emulate structural changes in the traffic network in the simulated area. The \texit{Persistence} class then optionally writes to results to a custom output format which is visualizable by an additional script~\cite{streets_report}.

\section{Differences and Limitations}
\label{sec:Concept::Differences}

Although the evaluated applications are based on the original \textit{streets4MPI}, there are some key differences in the implementation. This section gives a brief overview over the most important aspects that have been changed. The first paragraph of each subsection describes the original application's functionality while the second highlights differences and limitations in the evaluated implementations.

In the remaining part of the thesis the different applications will be referenced quite frequently. For brevity the language implementations to compare will be called by the following scheme: ``streets4<language>''. The Go version for example is called ``streets4go''.

\subsection*{Input format}
\label{subsec:Concept::Differences::Input}

The original \textit{streets4MPI} uses the somewhat dated \gls{osm} \gls{xml} format\fnote{\url{http://wiki.openstreetmap.org/wiki/OSM_XML}} as input which is parsed by \textit{imposm.parser}\fnote{\url{http://imposm.org/docs/imposm.parser/latest/}}. It then builds a directed graph via the \textit{python-graph}\fnote{\url{https://code.google.com/p/python-graph/}} library to base the simulation on~\cite{streets_report}.

The derived versions require the input to be in ``.osm.pbf'' format. This newer version of the \gls{osm} format is based on Google's \gls{protobuf} and is superior to the \gls{xml} variant in both size and speed~\cite{osm_wiki_pbf}. It also simplifies multi language development because the code performing the actual parsing is auto generated from a language independent description file. There are \gls{protobuf} backends for C, Rust and Go which can perform that generation.

\subsection*{Simulation}
\label{subsec:Concept::Differences::Simulation}

The simulation in the base application is based on randomly picked node pairs from the source graph. For these trips the shortest path is calculated by Dijkstra's \gls{sssp} algorithm as seen in \cite{cormen}. Also a random factor called ``jam tolerance'' is introduced to avoid oscillation between two high traffic routes in alternating iterations~\cite{streets_report}. Then after some time has passed in the simulation, existing streets get expanded or shut down depending on their usage to simulate road construction.

The compared implementations of this thesis also perform trip based simulation but without the added randomness and street modification. Also the edge weights are not dynamically recalculated in each iterations. Instead the street's length is calculated once from the corrdinates of the corresponding nodes and used as edge weigth directly. The concrete algorithm is a variant of the \shinline{Dijkstra-NoDec} \gls{sssp} algorithm as seen in~\cite[p. 16]{dijkstra_utcs}. It was mainly chosen because of its reduced complexity in required data structures. The algorithm is implemented separately in all three languages so it could theoretically get benchmarked standalone to get clearer results. This was not attempted in scope of the thesis because of time constraints.

\subsection*{Concurrency}
\label{subsec:Concept::Differences::Concurrency}

\textit{streets4MPI} parallelizes its calculations on multiple processes that communicate via message passing. This is achieved with the aforementioned \textit{MPI4Py} library which delegates to a native \gls{mpi} implementation installed on the system. If no supported implementation is found it falls back to a pure Python solution. Results have show that the native one should be preferred in order to achieve maximum performance~\cite{streets_report}.

Although Rust as well as Go can integrate decently with existing native code, the reimplementations will be limited to shared memory parallelization on threads. This was mostly decided to evaluate and compare the language inherent concurrency constructs rather than the quality of their foreign funtion interfaces. To achieve a fair comparison \textit{streets4c} will use \textit{OpenMP}~\fnote{\url{http://www.openmp.org}} as it is the de facto standard for simple thread parallelization in C. Of course this solution might not match the performance of hand optimized implementations parallelized with the help of \textit{pthreads} but since the focus is on simple concurrency in the context of scientific applications \textit{OpenMP} was selected as the framework of choice.

\section{Implementation Process}
\label{sec:Concept::Implementation}

The implementation process was performed iteratively. Certain milestones were defined and implemented in all three languages. The process only advanced to the next phase when the previous milestone was reached in all applications. This approach was chosen to allow for a fair comparison of the different phases of development. If the implementations would have been developed one after another to completion (or in any other arbitrary order), this might have introduced a certain bias to the evaluation because of possible knowledge about the problem aquired in a previous language translating to faster results in the next one.

\begin{figure}[htb]
    \centering
    \includegraphics[width=\textwidth]{img/milestone_timeline.png}
    \caption{Milestone overview}
    \label{fig:milestone_timeline}
\end{figure}

\autoref{fig:milestone_timeline} shows the different milestones in order of completion. For each phase various characteristics were captured and compared to highlight the languages' features and performance in the various areas. While the main development and test runs were performed on a laptop the final application was run on a high performance machine provided by the research group Scientific Computing to compare scalability beyond common desktop level processors. In the following sections each milestone is briefly described.

\subsection*{Setting up the Project}
\label{subsec:Concept::Implementation::Setup}

The first phase of development was to create project skeletons and infrastructure for the future development. The milestone was to have a working environment in place where the sample application could be built and executed. While this is certainly not the most important or even interesting part it might show the differences in comfort between the various toolchains.

\subsection*{Counting Nodes, Ways and Relations}
\label{subsec:Concept::Implementation::Counting}

The first real milestone was to read a .osm.pbf file and count all nodes, ways and relations in it. This was done to get familiar with the required libraries and the file format in general. The time recorded began from the initial project created in phase 0 and finished after the milestone was reached. As this is the most input and output intensive phase it should reveal some key differences between the candidates both in speed as well as memory consumption.

\subsection*{Building a basic Graph Representation}
\label{subsec:Concept::Implementation::Graph_Representation}

The next goal was to conceptionally build the graph and related structures the simulation would later operate on. This involved thinking about the relation between edges and nodes as well as the choice of various containers to store the objects efficiently while also keeping access simple. In addition the shortest path algorithm had to be implemented. This meant a priority queue had to be available as the algorithm relies on that data structure to store nodes which have yet to be processed. This milestone therefore tested the language's standard libraries and expressiveness in terms of typed containers.

\subsection*{Verifying Structure and Algorithm}
\label{subsec:Concept::Implementation::Verification}

After the base structure to represent graphs and calculate shortest paths was in place it was time to validate the implementations. Unfortunately the OSM data used in the first phase contained too much nodes and ways to be able to efficiently verify any computed results. Therefore a small example graph was manually populated and fed to the algorithm.

\subsection*{Benchmarking Graph Performance}
\label{subsec:Concept::Implementation::SequentialBenchmark}

The fourth milestone was preliminary benchmark of the implementations. The basic idea was to parse the \gls{osm} data used in phase one and build the representing graph. After that the shortest path algorithm is executed once for each node. The total execution time as well as the time taken for each step (building the graph and calculating shortest paths) should be measured and compared as well as the usual memory statistics from previous phases.

\subsection*{Benchmarking Parallel Execution}
\label{subsec:Concept::Implementation::ParallelBenchmark}

The fifth phase consisted of modifying the existing benchmark to operate in parallel via threading and benchmarking the results for various configurations. While all the development and previous benchmarks were performed on a personal laptop the final benchmarks were taken on a computation node of the research group to gather relevant results in high concurrency situations.

\subsection*{Cluster Preperation}
\label{subsec:Concept::Implementation::ClusterPreparation}

The final milestone was to prepare the implementations for the execution on the cluster provided by the research group. As this was a remote environment with some key differences to the development laptop the implementations had to be prepared and slightly changed.

\section{Overview of evaluated Criteria}
\label{sec:Concept::Criteria}

For the evaluation of the three languages multiple criteria have been selected. While some of them are directly quantifiable such as development time others are rated subjectively based on experiences from the implementation process. This is mostly true for the productivity metrics. It is important to note that not all statistics apply to all milestones. The following list introduces the reviewed criteria and briefly describes them.
\\

\begin{itemize}
    \item Performance
    \begin{itemize}
        \item Execution Time\\
        \textit{The time to complete the task of the milestone}
        \item Memory Footprint\\
        \textit{Total memory consumption as well as allocation and free counts}
    \end{itemize}
    \item Productivity
    \begin{itemize}
        \item \acrshort{sloc} Count\\
        \textit{Source lines of code to roughly estimate the code's complexity and maintainability. Tracked in all milestones}
        \item Development Time\\
        \textit{Time required to implementation the desired functionality. Tracked in all milestones}
        \item Resource Management
        \textit{Amout of work required to properly manage resources like memory, file handle or threads in a given language}
        \item Tooling Support\\
        \textit{Tooling support for common tasks throughout the development process. This includes the compiler, dependency management, project setup automation and many more}
        \item Library Ecosystem\\
        \textit{Available libraries for the given language considering common data structures, algorithms or mathematical functions. Includes the quality of the language's standard library}
        \item Parallelization Effort\\
        \textit{Amount of work required to parallelize an existing sequential application}
    \end{itemize}
\end{itemize}

As these statistics were tracked during the implementation itself the \hyperref[ch:Implementation]{next chapter} directly lists and evaluates intermediate results for each milestone. In contrast \autoref{ch:Evaluation} evaluates the final performance outcomes from the cluster benchmarks as well as the gathered productivity metrics.

\section{Related Work}
\label{sec:Concept::Related}

The search for new programming languages which are fit for \gls{hpc} is not a recently developing trend. There have been multiple studies and evaluations but so far none of the proposed languages have gained enough traction to receive widespread adoption. Also most reports focused on the execution performance without really considering additional software metrics or developer productivity. \cite{related_multicore} adds lines of code and development time to the equation but both of these metrics only allow for superficial conclusions about code quality and productivity.

From the candidates presented here Go in particular has been compared to traditional \gls{hpc} languages with mixed results. Although its regular execution speed is somewhat lacking \cite{related_sor_study} showed the highest speedup from parallelization amongst the evaluated languages which is very promising considering high concurrency scenarios like cluster computing. Rust on the other hand has not been seriously evaluated in the \gls{hpc} context probably due to it still being developed.


\section{Introduction}

%In connectomics, neuroscientists annotate neurons and their connectivity within 3D volumes to gain insight into the functional structure of the brain. Rapid progress in automatic sample preparation and electron microscopy (EM) acquisition techniques has made it possible to image large volumes of brain tissue at $\approx4\, nm$ per pixel to identify cells, synapses, and vesicles. For $40\, nm$ thick sections, a $1\, mm^3$ volume of brain contains $10^{15}$ voxels, or 1 petabyte of data. With so much data, manual annotation is infeasible, and automatic annotation methods are needed~\cite{jain2010,Liu2014,GALA2014,kaynig2015large}.

%Automatic annotation by segmentation and classification of brain tissue is challenging~\cite{isbi_challenge}. The state of the art uses supervised learning with convolutional neural networks~\cite{Ciresan:2012f}, or potentially even unsupervised learning~\cite{BogovicHJ13}. Typically, cell membranes are detected in 2D images, and the resulting region segmentation is grouped into geometrically-consistent cells across registered sections. Cells may also be segmented across registered sections in 3D directly. Using dynamic programming techniques~\cite{Masci:2013a} and a GPU cluster, these classifiers can segment $\approx1$ terabyte of data per hour ~\cite{kasthuri2015saturated}. This is sufficient to keep up with the 2D data capture process on state-of-the-art electron microscopes (though 3D registration is still an expensive offline operation).

In connectomics, neuroscientists annotate neurons and their connectivity within
3D volumes to gain insight into the functional structure of the brain. Rapid
progress in automatic sample preparation and electron microscopy (EM)
acquisition techniques has made it possible to image large volumes of brain
tissue at nanometer resolution. With a voxel size of
$4\times4\times40~\text{nm}^3$, a cubic millimeter volume is one petabyte of
data. With so much data, manual annotation is not feasible, and automatic
annotation methods are needed~\cite{jain2010,Liu2014,GALA2014,kaynig2015large}.

Automatic annotation by segmentation and classification of brain tissue is
challenging~\cite{isbi_challenge} and all available methods make errors, so
the results must be \emph{proofread} by humans. This crucial
task serves two purposes: 1) to correct errors in the segmentation, and 2) to
increase the body of labeled data from which to train better automatic
segmentation methods. Recent proofreading tools provide intuitive user
interfaces to browse segmentation data in 2D and 3D and to identify and manually
correct errors~\cite{markus_proofreading,raveler,mojo2,haehn_dojo_2014}. Many
kinds of errors exist, such as inaccurate boundaries, but the most common are
\emph{split errors}, where a single segment is labeled as two, and \emph{merge
errors}, where two segments are labeled as one
(Fig.~\ref{fig:merge_and_slit_errors}). With user interaction, split errors can
be joined, and the missing boundary in a merge error can be defined with
manually-seeded watersheds~\cite{haehn_dojo_2014}. However, the visual
inspection to find errors takes the majority of the time, even with
semi-automatic correction tools~\cite{proofreading_bottleneck}.

\begin{figure}[t]
\begin{center}
  \includegraphics[width=\linewidth]{gfx/merge_and_split_errors.pdf}
\end{center}
\vspace{-4mm}
   \caption{The most common proofreading corrections are fixing split errors (red arrows) and merge errors (yellow arrow). A fixed segmentation matches the cell borders.}
   \vspace{-4mm}
\label{fig:merge_and_slit_errors}
\end{figure}

Our goal is to automatically detect potential split and merge errors to reduce visual
inspection time. Further, to reduce correction time, we propose
corrections to the user to accept or reject. We call this process \textit{guided
proofreading}.

We train a classifier for split error detection with a convolutional neural network
(CNN). This takes as input patches of membrane segmentation probabilities, cell
segmentation masks, and boundary masks, and outputs a split-probability score. As we
must process large data, this classifier only operates on cell boundaries, which
reduces computation over methods that analyze every pixel. For merge errors, we
invert and reuse the split classification network, and ask it to rate a
set of generated boundaries that hypothesize a split. %We compute
%corrections for both types of errors.

Possible erroneous regions are sorted by their score, and a candidate correction is generated for each
region. Then, a user works through this list of regions and corrections. In a
forced choice setting, the user either selects a correction or skips it to
advance to the next region. In an automatic setting, errors with a high probability are automatically corrected first, given an appropriate
probability threshold, after which the user would take over. Finally, to test
the limits of performance, we create an oracle which only accepts corrections
that improve the segmentation, based on knowledge of the ground truth. This is
guided proofreading with a perfect user.

We evaluate these methods on multiple connectomics datasets. For the forced
choice setting, we perform a quantitative user study with 20 novice users who
have no previous experience of proofreading EM data. We ask participants to
proofread a small segmentation volume in a fixed time frame. In a
between-subjects design, we compare guided proofreading to the semi-automatic
\textit{focused proofreading} approach by Plaza~\cite{focused_proofreading}. In
addition, we compare against the manual interactive proofreading tool
\textit{Dojo} by Haehn~\etal~\cite{haehn_dojo_2014}. We also asked four domain
experts to use guided proofreading and focused proofreading for comparison.

This paper makes the following contributions.
%
First, we present a CNN-based boundary classifier for split errors, plus a merge
error classifier that inverts the split error classifier. This proposes merge error corrections,
which removes the need to manually draw the missing
edge. These classifiers automatically reduce error with little training data, which is expensive to collect for connectomics data.
%
Second, we develop a human-guided proofreading approach to correcting segmentation
volumes, and compare forced-choice interaction with
automatic and oracle proofreading.
%
Third, we present results of a quantitative user study assessing
guided proofreading. Our method is able to reduce segmentation
error faster than state-of-the-art semi-automatic tools for both novice and
expert users.
%
Fourth, we present the first connectomics proofreading benchmark, with image, label, and human interaction data, and evaluation code.

Guided proofreading is applicable to all existing automatic segmentation methods that produce a label map.
While we train models based on traditional CNN architectures, we propose an error correction approach with human interaction that works with many classifiers.
As such, we believe that our approach is a promising direction to proofread segmentations more efficiently and better tackle large volumes of connectomics imagery.

%Our goal is to add automatic detection of split and merge errors to proofreading tools. We design automatic classifiers that detect split and merge errors in segmentations so the user does not need to visually inspect the whole data volume to spot errors. A proofreading tool then recommends regions with a high probability of an error to the user, and suggest corrections to accept or reject. We call this process \textit{guided proofreading}.
%
%In this paper, we introduce classifiers to detect merge- and split errors based on a convolutional neural network (CNN). We believe that this is the first time that deep learning is applied to the task of proofreading. Our classifiers work on top of any existing automatic segmentation method to find potential errors and suggest corrections. Given a membrane segmentation from a fast automatic method, our classifiers operate on the boundaries of whole cell regions. Compared to techniques that must analyze every input pixel, we reduce the data analysis to the boundaries. First, we train a CNN to detect only split errors. The output of this network is a probability whether a boundary between two segments is valid or not. We then reuse the same network to also detect merge errors by generating possible boundaries within a cell and inverting the split error score. We create corrections for both types of errors which can be accepted or rejected. This reduces the proofreading operation to simple yes/no decisions.
%
%We further propose a greedy algorithm to perform proofreading. Possible erroneous regions are sorted by their score and the algorithm iteratively suggests a correction for each region. A user then works through this stream of regions and corrections. In a forced choice setting, the user either selects a correction or skips it to advance to the next region. This choice can be also performed automatically by running the algorithm until a configurable threshold is reached. In addition, if ground truth data is available, we can use a selection oracle to drive the forced choice selection. The oracle only accepts corrections which improve the automatic segmentation. This equals perfect proofreading.
%
%We evaluate our method automatically by threshold and oracle on multiple real-world connectomics datasets. To evaluate the forced choice setting, we perform a quantitative user study. The study targets non-experts with no previous experience of proofreading electron microscopy data. We ask the participants to proofread a small segmentation volume in a fixed time frame by performing yes/no decisions. The user study is designed as a between-subjects experiment and compares guided proofreading against two other methods: a recently published fully interactive proofreading tool named \textit{Dojo} by Haehn~\etal~\cite{haehn_dojo_2014} and the semi-automatic \textit{focused proofreading} approach by Plaza~\cite{focused_proofreading}. We also asked four domain experts to use guided proofreading and focused proofreading for additional comparison.
%
%Our first contribution is a classifier for split error detection based on a convolutional neural network. The classifier performs well even when trained with little amounts of training data. This is important since generating ground truth labels in connectomics requires manually labeling pixels and is very time-consuming. Our second contribution is a mechanism to identify merge-errors by re-using the split error classifier. Merge errors are usually less common than split errors in the oversegmented automatic labelings. However, they require more interaction during correction since split lines need to be manually drawn. Our method reduces this to a single click by providing the potential correction. The split and merge error identification is executed as a greedy algorithm to correct segmentation volumes, the third contribution of this paper. The algorithm can be driven automatically with a threshold, by an oracle based on ground truth and interactively in a forced choice setting. Our final contribution is our quantitative user study. We present statistically significant results showing that novice and expert users of guided proofreading are able to proofread a given dataset better and faster than with existing interactive and semi-automatic proofreading tools. As a consequence, we are able to provide tools to proofread segmentations more efficiently, and so better tackle large volumes of connectomics imagery.