\documentclass[runningheads,a4paper]{llncs}

\usepackage{amssymb}
\usepackage{amsmath}
\usepackage{algpseudocode}
\usepackage{algorithm}
\usepackage{graphicx}
\usepackage{url}
\usepackage{float}
\usepackage{caption}
\usepackage{subcaption}
\usepackage{tabularx}
\usepackage{array}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}

\setcounter{tocdepth}{3}
\floatstyle{boxed}
\restylefloat{figure}
\algrenewcommand\algorithmicrequire{\textbf{input}}
\algrenewcommand\algorithmicensure{\textbf{output}}

\newcommand{\hctimes}[2]{{#1}\!\times\!{#2}}
\newcommand{\hcrange}[2]{\overline{{#1}\colon\!\!{#2}}}
\newcommand{\hcsignalspace}{\mathbb{R}^d}
\newcommand{\hcweightspace}{\mathbb{R}^w}
\newcommand{\hcdictspace}{\mathbb{R}^{\hctimes{w}{d}}}

\urldef{\mailall}\path|{coman,barth,martinetz}@inb.uni-luebeck.de|
\newcommand{\keywords}[1]{\par\addvspace\baselineskip
\noindent\keywordname\enspace\ignorespaces#1}

\begin{document}

\mainmatter

\title{Sparse Coding Neural Gas Applied to Image Recognition}
\titlerunning{Sparse Coding Neural Gas Applied to Image Recognition}

\author{Horia Coman$^{1,2}$ \and Erhardt Barth$^{1}$ \and Thomas Martinetz$^{1}$}
\authorrunning{Sparse Coding Neural Gas Applied to Image Recognition}

\institute{$^1$Institute for Neuro-and Bioinformatics, University of L\"{u}beck\\
Ratzeburger Allee 160, 23538 L\"{u}beck, Germany\\
\url{http://www.inb.uni-luebeck.de}\\
\mailall\\
$^2$LAPI, The ``POLITEHNICA'' University of Bucure\c{s}ti\\
Splaiul Independen\c{t}ei nr. 313, 060042 Bucure\c{s}ti, Romania\\
\url{http://imag.pub.ro}}

\toctitle{Sparse Coding Neural Gas Applied to Image Recognition}
\tocauthor{Sparse Coding Neural Gas Applied to Image Recognition}
\maketitle

\begin{abstract} A generalization of the Sparse Coding Neural Gas (\textbf{SCNG}) algorithm for feature learning is proposed and is then discussed in the context of modern classifier techniques for images. Two versions are presented. The latter obtains faster convergence by exploiting the nature of particular feature coding methods. The algorithm is then used as part of a larger image classification system, which is tested on the MNIST handwritten digit dataset and the NORB object dataset, obtaining results close to state-of-the-art methods.
\keywords{Neural Gas, Sparse Coding, Sparse Coding Neural Gas, Image Recognition, Matching Pursuit}
\end{abstract}

\section{Introduction}

The task of image recognition is a complex one. Simply training a classifier on raw image data will yield poor performance. A common strategy is to look at important properties of an image, called \emph{features}. Then, given an image $\textbf{I}$, these features are used to compute a feature descriptor $\textbf{F}$. This contains, for each feature, a measure of the inclusion of that feature into the image. Then, in order to obtain an estimated class, only $\textbf{F}$ is considered. The construction of features is thus a big part of most machine learning applications. However, for best performance, specific domain knowledge must be used. This makes both the design and comparison of learning systems harder.

In the last decade, automatic feature construction, also known as unsupervised feature learning \cite{best-architecture-object-recognition,emergence-sparse-coding,sparse-coding-strategy-V1,tiny-images}, has become mainstream, surpassing hand-crafted methods on diverse problems \cite{best-architecture-object-recognition,learning-convolutional-feature-hierarchies,gradient-based-learning,convolutional-networks-vision,best-practices-cnn,simple-method-sparse-coding,sparse-features-audio-classification}. These techniques aim to build features by looking at the statistical properties of a dataset. In addition to this, a full framework for classification has been refined, based on the model of Convolutional Neural Networks (\textbf{CNNs}) \cite{gradient-based-learning}, which has parallels with the structure of the V1 area of the mammalian brain.

This paper studies a particular type of feature learning method, called Sparse Coding Neural Gas \cite{sparse-coding-neural-gas-1,sparse-coding-neural-gas-2,sparse-coding-neural-gas-3,sparse-coding-neural-gas-4}, on two tasks of image classification. The algorithm itself is an adaptation of the Neural Gas algorithm \cite{neural-gas-1,neural-gas-2}.

\section{Overview of Feature Extraction}

Given an image $\textbf{I} \in \mathbb{R}^{\hctimes{m}{n}}$, the conceptual recognition pipeline can be summarized as:

\begin{equation}
\textbf{I} \Rightarrow \textbf{F} \rightarrow \omega
\end{equation}

The $\rightarrow$ corresponds to actual classification. A simple classifier, usually a Logistic/SoftMax Regression or a Linear SVM is used. The burden falls on the feature extraction phase, denoted by $\Rightarrow$, to produce descriptors of such a nature that the simple classifiers can properly discriminate the classes.

For our purposes, a feature is a small filter of some sort. More precisely, the full feature set consists of $w$ normalized square images of size $d = \hctimes{p}{p}$ with $p < \min(m,n)$. This set is denoted by $\textbf{C} = \left[ \textbf{C}^1 \left|\right. \textbf{C}^2 \left|\right. \dots \left|\right. \textbf{C}^w \right] \in \mathbb{R}^{\hctimes{d}{w}}$. Normalization is usually imposed by the feature learning method, but is, in general, a nice property to have, because it makes coding methods more ``interpretable'' and less susceptible to favoring one feature over another because of scale differences. A feature set can be obtained from several sources. Firstly, it can be generated randomly \cite{random-weights-feature-learning}. Secondly, a well-known set can be employed, such as DCT bases or Gabor Wavelet bases \cite{simple-method-sparse-coding}. Thirdly, the set can be learned from a sample of patches extracted from the training set of the classification system \cite{emergence-sparse-coding,sparse-coding-strategy-V1,tiny-images}. \textbf{SCNG} is an example of an algorithm used for this kind of learning.

Actual extraction consists of three steps, called \emph{coding}, \emph{nonlinear}, and \emph{reduce}. The coding step accepts as input the original image $\textbf{I}$ and produces a set of $w$ images of the same size as $\textbf{I}$. In order to obtain the $(i,j)^{th}$ pixel of the $l^{th}$ image, the patch of size $\hctimes{p}{p}$ from $\textbf{I}$, centered at position $(i,j)$, is ``coded'', with regards to filter $\textbf{C}^l$. The simplest form of coding is an inner product between the two images. Considering the whole image, this corresponds to doing a convolution with the filter $\textbf{C}^l$. In fact, this is the strategy employed by CNNs. In general, the response for the $(i,j)^{th}$ pixel need not depend just on $\textbf{C}^l$, but on the whole $\textbf{C}$. Therefore, in the next section, which is dedicated to coding methods, we will consider the problem of finding the values of the pixels at position $(i,j)$ in all $w$ images at once.

The nonlinear step accepts as input the set of $w$ images produced by the coding step and produces another set of $w$ images, again of the same size as $\textbf{I}$ but with elements mapped to a restricted interval, such as $[-1,+1]$. Each pixel in each image is transformed independently by passing its value through a sigmoid-like nonlinearity, such as the logistic function. The first and second steps, together, can be viewed as a feed-forward neural network, with $mn$ input units and $wmn$ output units, a very specific weight setup and a complex feed-forward rule.

Lastly, the reduce step accepts as input the previous set of $w$ images and produces a final set of $w$ images, but this time of smaller sizes than the original. More precisely, each image is divided into non-overlapping blocks of size $\hctimes{q}{q}$ and all the values from each block are combined to form one value, according to some function. Common choices are the $\max(\left|\star\right|,\dots,\left|\star\right|)$ and $\sum_{i,j}{(\star)^2}$ functions. The output of this stage is a set of $w$ images of size $\hctimes{(m / q)}{(n / q)}$. In general, CNNs can have a more flexible reduce step, but we've found this limited form, which considers each image individually, to be useful as well. The reasons given for the reduce step are that it introduces a certain kind of resistance to small translations. Basically, anywhere in a $\hctimes{q}{q}$ block a feature is detected, the corresponding output of the reduce step should be large. Invariance to larger translations, scaling, rotation etc. is something that has to be captured by the features or by the classifier. Otherwise, enough data which cover these translations must be provided for learning.

The final feature vector $\textbf{F}$ is a linearized version of these images, that is, a $\left[(mnw) / q^2\right]$-dimensional vector. Also notice that the whole system can be viewed as one heterogeneous neural network consisting of two different stages. In general, several of these modules can be linked, each with its own set of features, tailored to the type of images it receives from the previous layer. In principle, very deep feature extraction networks can thus be built, depending on the complexity of the dataset being studied. In our experiments, only feature extractors with one layer were used. Again, if a perceptron or a two layer MLP are used as the classifier and the inner product is used as the coding method, the whole system becomes a single large neural network. Classical back-propagation can then be used at the end to \emph{fine-tune} the weights, starting from initial values assigned according  to $\textbf{C}$. In our experiments, this procedure was not used, but situations such as transfer-learning or self-taught learning \cite{self-taught-learning} can make use of this property.

\section{Coding}

This section describes in more detail how to do the coding. For simplicity, we will assume we work with $d$-dimensional signals. Thus, the $\hctimes{p}{p}$ patches previously discussed must be linearized such that $d = p^2$. Assume also that we are given a set of features $\textbf{C}$, like in the previous section. Most of the times we will have $w > d$, that is, the set of features is \emph{overcomplete}. Our goal is to approximate a signal $\textbf{x} \in \hcsignalspace$ in terms of $\textbf{C}$. The most common approach is to use a linear combination of the features. The \emph{code} is then the signal $\textbf{a} \in \hcweightspace$ and the \emph{approximation} is

\begin{equation}
\hat{\textbf{x}} = \sum_{i=1}^w {a_i \textbf{C}^i} = \textbf{C}\textbf{a}~.
\end{equation}

The quality of the code is determined by how well the reconstruction $\hat{\textbf{x}}$ matches the original signal. For audio and image processing, it has been shown that a sparse code for $\textbf{x}$, that is, one with numerous zero or close to zero components, has many desirable properties \cite{emergence-sparse-coding,sparse-coding-strategy-V1}. Many methods for sparse coding have been proposed \cite{undetermined-minimal-L1,sparse-coding-strategy-V1}. We will focus on a group of iterative methods for computing $\textbf{a}$, known as pursuits, which originate in the signal processing community. All assume $\textbf{C}$ and $\textbf{x}$ are given and run for a number of $k \leq d$ iterations. The general problem they try to solve is $\arg\min_a \|\textbf{x} - \textbf{C}\textbf{a}\|_2^2$ subject to $\|\textbf{a}\|_0 \leq k$. This an NP-complete problem. The pursuits are greedy approximations to it. Let the initial residual $\mathcal{R}^0\textbf{x} = \textbf{x}$. At iteration $t$, let $\textbf{C}^\omega$ be the most similar feature in $\textbf{C}$, relative to $\mathcal{R}^t\textbf{x}$. The updated code and residual, $\textbf{a}^{t+1}$ and $\mathcal{R}^{t+1}\textbf{x}$, are produced by decomposing $\mathcal{R}^t\textbf{x}$ in terms of $\textbf{C}^\omega$. After $k$ iterations, $\textbf{a}^k$ is returned as the code associated to $\textbf{x}$, and $\mathcal{R}^k\textbf{x}$ is returned as a measure of the ability of the algorithm to reconstruct the signal in terms of $\textbf{C}$. The difference between the several methods consists in how they find $\textbf{C}^\omega$ and how they update $\textbf{a}^{t+1}$. The general procedure is illustrated in \textbf{Algorithm \ref{algo:Pursuit}}. At the end of this algorithm we obtain $\textbf{x} = \textbf{C}\textbf{a}^k + \mathcal{R}^k\textbf{x}$. Also, the norm of the final residual $\mathcal{R}^k\textbf{x}$ tends to $0$ as $k \rightarrow +\infty$ for sensible choices of $\text{\textbf{sim}}$ and $\text{\textbf{next}}$ functions. In the limit, the equality becomes $\textbf{x} = \textbf{C}\textbf{a}^{+\infty}$.

\begin{algorithm}[h!]
\caption{The General Pursuit Method}
\label{algo:Pursuit}
\begin{algorithmic}
\Require $\textbf{C},\textbf{x},k$
\Ensure $\textbf{a}^k,\mathcal{R}^k\textbf{x}$
\State $\Lambda^0 \gets \phi$
\State $\textbf{a}^0 \gets \textbf{0}$
\State $\mathcal{R}^0\textbf{x} \gets \textbf{x}$
\State $t \gets 0$
\While {$t < k \text{~or~} \|\mathcal{R}^t\textbf{x}\|_2 \geq \delta$}
\State $\omega \gets \arg \max_{i \in \textbf{dom}} \text{\textbf{sim}}(\mathcal{R}^t\textbf{x},\textbf{C},\Lambda^t,i)$
\State $\Lambda^{t+1} \gets \Lambda^t \cup \omega$
\State $\textbf{a}^{t+1} \gets \text{\textbf{next}}(\textbf{a}^t,\mathcal{R}^t\textbf{x},\textbf{C},\Lambda^{t+1},\omega)$
\State $\mathcal{R}^{t+1}\textbf{x} \gets \mathcal{R}^t\textbf{x} - \textbf{a}^{t+1}_\omega C^\omega$
\State $t \gets t + 1$
\EndWhile
\end{algorithmic}
\end{algorithm}

The simplest pursuit method, introduced in \cite{matchingpursuit1}, is Matching Pursuit (\textbf{MP}). \textbf{Table \ref{table:PursuitParametrization}} shows what form the $\text{\textbf{sim}}$ and $\text{\textbf{next}}$ functions take in this case. An important property of this algorithm is that for every $t$, $\|\mathcal{R}^t\textbf{x}\|_2^2 \geq \|\mathcal{R}^{t+1}\textbf{x}\|_2^2$ and, furthermore, with a decay that is exponential. The two major drawbacks of this method are that the approximation at time $t$, $\textbf{C}\textbf{a}^t$, is not optimal with respect to the selection of features $\Lambda^t$; and that for the residual norm to actually reach small enough values, a $k > w$ could be necessary. However, these drawbacks are not critical for classification purposes, and, because of its simplicity and speed, we use it in our experiments.

An improvement to \textbf{MP} is Orthogonal Matching Pursuit (\textbf{OMP}) \cite{matchingpursuit2,orthopursuit,pursuitdifferences}, which addresses the two issues discussed above. Again, \textbf{Table \ref{table:PursuitParametrization}} shows the forms the $\text{\textbf{sim}}$ and $\text{\textbf{next}}$ functions take. Also, notice that at each iteration, only the features not considered before are processed. All the properties of \textbf{MP} hold here as well. At iteration $t$, the approximation computed is the closest point in $\overline{\text{span}(\textbf{C}^{\Lambda^t})}$ to $\textbf{x}$, according to the Euclidean norm. The version presented here is suboptimal from an algorithmic point of view. More sophisticated methods based on QR decomposition have been developed \cite{matchingpursuit2,pursuitdifferences}.

\renewcommand{\arraystretch}{1.5}
\begin{table}[h!]
  \caption{The different parametrization for pursuit methods}
  \label{table:PursuitParametrization}
  \begin{tabularx}{\textwidth}{|l|>{\centering}X|>{\centering}X|c|}
       \hline
        Method & $\text{\textbf{sim}}$ function & $\text{\textbf{next}}$ function & $\text{\textbf{dom}}$ domain\\ \hline \hline
        \textbf{MP} & $\left| \langle \mathcal{R}^t\textbf{x} , \textbf{C}^i \rangle \right|$ & $\textbf{a}^t + \langle \mathcal{R}^t\textbf{x} , \textbf{C}^\omega \rangle \delta_\omega$ & $\hcrange{1}{w}$ \\  \hline
        \textbf{OMP} & $\left| \langle \mathcal{R}^t\textbf{x} , \textbf{C}^i \rangle \right|$ & $\arg\min_{a} {\| \textbf{x} - \textbf{C}^{\Lambda^{t+1}}\textbf{a} \|_2^2}$ & $\hcrange{1}{w} \setminus \Lambda^t$ \\
       \hline
    \end{tabularx}
\end{table}
\renewcommand{\arraystretch}{1.0}

\section{Learning a Feature Set}

We now turn to the problem of learning the feature set $\textbf{C}$, given a coding method $\hat{\mathcal{C}}_\textbf{C}$ and a sample $\textbf{X} = \left[ \textbf{X}^1 \left|\right. \textbf{X}^2 \left|\right. \dots \left|\right. \textbf{X}^N \right] \in \mathbb{R}^{\hctimes{d}{N}}$ of linearized image patches of size $\hctimes{p}{p}$, usually extracted from either the whole training set or from a larger ``natural scenes'' dataset \cite{self-taught-learning}. As we previously mentioned, the method we employed here is the Sparse Coding Neural Gas approach, which is an adaptation of the Neural Gas algorithm introduced in the context of vector quantization. Vector quantization can be considered as a stricter version of feature learning, where the codes are $1$-sparse and only a boolean ``indicator'' of the feature most similar to the input $\textbf{x}$, as measured by the Euclidean distance, is stored.

The Neural Gas algorithm is an iterative one. It begins by initializing $\textbf{C}$ to $w$ random observations from the training sample $\textbf{X}$. Then, for a number of $T_{max}$ iterations, an adaptation process takes place, which slowly changes $\textbf{C}$ in order to best represent the distribution over the input space. More precisely, at each iteration $t$ an observation is randomly selected from $\textbf{X}$ and distances to each element of $\textbf{C}$ are computed. Each feature is then modified in a manner proportional to the distortions between it and the signal $\textbf{x}$, on the one hand, and the ranking of this distortion in the list of all distortions, on the other hand. Therefore, the update process includes a local and a global component. \textbf{Algorithm \ref{algo:NeuralGas}} gives the whole picture. Note that both a time decreasing learning factor is used as well as a time decreasing neighborhood control. The algorithm is similar to the well-known Self-Organizing Map. The difference lies in changing the weight update from one which considers a pre-defined topology, to one which looks at the neighborhood withing the input space. More complete versions of this algorithm \cite{neural-gas-1} can actually build a topological description of the input space which is useful for exploratory data analysis.

\begin{algorithm}[h!]
\caption{Neural Gas}
\label{algo:NeuralGas}
\begin{algorithmic}
\Require $\textbf{X},w,T_{max},\mu^0,\mu^{T_{max}},\lambda^0,\lambda^{T_{max}}$
\Ensure $\textbf{C}$
\State $\textbf{C} \gets \mbox{randomly select $w$ observations from $\textbf{X}$}$
\For {$t = \hcrange{1}{T_{max}}$}
\State $\mu^t \gets \mu^0 (\mu^{T_max} / \mu^0)^{t / T_{max}}$ \Comment {Current learning rate}
\State $\lambda^t \gets \lambda^0 (\lambda^{T_{max}} / \lambda^0)^{t / T_{max}}$ \Comment {Current neighborhood control}
\State $\textbf{x} \gets \text{an observation from $\textbf{X}$}$
\State $\textbf{a} \gets [ ~ \|\textbf{x} - \textbf{C}^i\|_2^2 ~ \mbox{for $i \in \hcrange{1}{w}$} ~ ]$
\State $\textbf{C} \gets \textbf{C} + [ ~ \mu^t e^{-rank_{\textbf{a}}(a_i) / \lambda^t} (\textbf{x} - \textbf{C}^i) ~ \mbox{for $i \in \hcrange{1}{w}$} ~ ]$
\EndFor
\end{algorithmic}
\end{algorithm}

The Neural Gas algorithm works in the input space rather than feature set space. Adapting the algorithm to work with features and accept any coding method gives rise to a first version of the Sparse Coding Neural Gas. The major modification is the fact that each update is done according to Oja's Rule \cite{oja-rule} instead of the simple error term of the Neural Gas. The full algorithm is described in \textbf{Algorithm \ref{algo:SparseCodingNeuralGasV1}}. Notice that the $rank_{\textbf{a}}$ function considers absolute values, so that features are updated proportional to the magnitude of the associated response.

\begin{algorithm}[h!]
\caption{Sparse Coding Neural Gas V1}
\label{algo:SparseCodingNeuralGasV1}
\begin{algorithmic}
\Require $\textbf{X},w,\mathcal{C},T_{max},\lambda^0,\lambda^{T_{max}},\mu^0,\mu^{T_{max}}$
\Ensure $\textbf{C}$
\State $\textbf{C} \gets \mbox{randomly initialize $w$ normalized features}$
\For {$t = \hcrange{1}{T_{max}}$}
\State $\mu^t \gets \mu^0 (\mu^{T_{max}} / \mu^0)^{t / T_{max}}$  \Comment {Current learning rate}
\State $\lambda^t \gets \lambda^0 (\lambda^{T_{max}} / \lambda^0)^{t / T_{max}}$ \Comment {Current neighborhood control}
\State $\textbf{x} \gets \text{an observation from $\textbf{X}$}$
\State $\textbf{a} \gets \mathcal{C}_{\textbf{C}}\{ \textbf{x} \}$
\State $\textbf{C} \gets \textbf{C} + [ ~ \mu^t e^{-rank_{\textbf{a}}(a_i) / \lambda^t} a_i (\textbf{x} - a_i \textbf{C}^i) ~ \mbox{for $i \in \hcrange{1}{w}$} ~ ]$
\State $\textbf{C} \gets \mbox{normalize each feature in $\textbf{C}$}$
\EndFor
\end{algorithmic}
\end{algorithm}

A further improvement is possible considering the fact that many coding methods are iterative and produce orderings of a subset $\hcrange{1}{w} \setminus \Lambda^t$ of the feature elements at each iteration. \textbf{MP} and \textbf{OMP} are such methods. A second version of the Sparse Coding Neural Gas is presented as \textbf{Algorithm \ref{algo:SparseCodingNeuralGasV2}}. Notice that at each iteration only the subset of previously unselected features is updated, instead of the whole set. Also, the variable $S^i$, which is a substitute for all the abstracted coding method specific information, must contain a copy of the original feature set $\textbf{C}$ at iteration $t$, before the inner-loop coding procedure. The reason for this is that $\textbf{C}$ is updated in the inner-loop and it can cause problems for the coder to change the features as time progresses.

\begin{algorithm}
\caption{Sparse Coding Neural Gas V2}
\label{algo:SparseCodingNeuralGasV2}
\begin{algorithmic}
\Require $\textbf{X},w,\mathcal{C},T_{max},\lambda^0,\lambda^{T_{max}},\mu^0,\mu^{T_{max}}$
\Ensure $\textbf{C}$
\State $C \gets \mbox{randomly initialize $w$ normalized features}$
\For {$t = \hcrange{1}{T_{max}}$}
\State $\mu^t \gets \mu^0 (\mu^{T_{max}} / \mu^0)^{t / T_{max}}$  \Comment {Current learning rate}
\State $\lambda^t \gets \lambda^0 (\lambda^{T_{max}} / \lambda^0)^{t / T_{max}}$ \Comment {Current neighborhood control}
\State $\textbf{x} \gets \text{an observation from $\textbf{X}$}$
\State $S^0 \gets \text{initialize coding method specific state}$
\For {$i = \hcrange{0}{k}$}
\State $[\alpha^i~\Lambda^i~S^{i+1}] \gets \mathcal{C}_{\textbf{C}} \{S^i,\textbf{x}\}$ \Comment {$\alpha^i$ stores similarities for features in $\hcrange{1}{w} \setminus \Lambda^i$}
\State $\textbf{C} \gets \textbf{C} + [ ~ \mu^t e^{-rank_{\alpha^i}(\alpha_j^i) / \lambda^t} \alpha_j^i (\textbf{x} - \alpha_j^i \textbf{C}^j) ~ \mbox{for $j \in \hcrange{1}{w}\setminus\Lambda^i$} ~ ]$
\State $\textbf{C} \gets \mbox{normalize each feature in $\textbf{C}$}$
\EndFor
\EndFor
\end{algorithmic}
\end{algorithm}

\section{Experiments}

In order to test the classifier system, two datasets were employed: the well known MNIST handwritten digit dataset \cite{gradient-based-learning} and the NORB object dataset \cite{learning-methods-invariance-pose-lighting}. Both of these are widely used for benchmarking classifiers. MNIST has $10$ classes corresponding to the $10$ Arabic numerals. It consists of $60000$ training images and $10000$ test images. NORB has $5$ classes, corresponding to different categories of objects (animal, human, plane, truck, car). It consists of $24300$ training images and $24300$ test images. An example of the kind of features learned from these sets can be seen in \textbf{Figure \ref{fig:LearnedFigures}}. For the NORB dataset, a pre-processing step of ``whitening'' is applied. This speeds up convergence and is achieved through the ZCA transform as described in \cite{tiny-images}.

Classification scores for the methods we used as well as other details are presented in \textbf{Table \ref{table:ResultsMNIST}} and \textbf{Table \ref{table:ResultsNORB}}. Using random patches as features and using features learned by gradient descent \cite{sparse-coding-strategy-V1} in the feature set space are also presented. The best result in the literature is also included. For both datasets, only methods which dealt with the unmodified dataset are considered. Extending the dataset through elastic distortions \cite{best-practices-cnn} has been shown to improve performance. However, such methods are not always applicable. Notice that both \textbf{SCNG} and Gradient Descent in feature set space produce better results than simply using random features, but are otherwise close in performance.

\renewcommand{\arraystretch}{1.2}
\begin{table}[h!]
  \caption{Classification results for MNIST}
  \label{table:ResultsMNIST}
  \begin{tabularx}{\textwidth}{|l|l|X|}
    \hline
    \textbf{Method} & \textbf{Error} & \textbf{Notes} \\ \hline\hline
    Baseline & 5.34 & Linear SVM on raw pixel data \\ \hline
    CNN (unsupervised pretraining) & 0.53 & Best without dataset extension. See \cite{best-architecture-object-recognition}. \\ \hline
    Our Method (SCNG) & 0.71 & MP-$11$ with $p=11$ and $q=4$ and $w=1024$ \\ \hline
    Our Method (Gradient) & 0.77 & MP-$11$ with $p=11$ and $q=4$ and $w=1024$ \\ \hline
    Our Method (Random) & 0.87 & MP-$11$ with $p=11$ and $q=4$ and $w=1024$ \\ \hline
    Our Method (SCNG) & 0.69 & MP-$25$ with $p=11$ and $q=4$ and $w=1024$ \\ \hline
    Our Method (Gradient) & 0.67 & MP-$25$ with $p=11$ and $q=4$ and $w=1024$ \\ \hline
    Our Method (Random) & 0.77 & MP-$25$ with $p=11$ and $q=4$ and $w=1024$ \\ \hline
  \end{tabularx}
\end{table}
\renewcommand{\arraystretch}{1.0}

\renewcommand{\arraystretch}{1.2}
\begin{table}[h!]
  \caption{Classification results for NORB}
  \label{table:ResultsNORB}
  \begin{tabularx}{\textwidth}{|l|l|X|}
    \hline
    \textbf{Method} & \textbf{Score} & \textbf{Notes} \\ \hline\hline
    Baseline & 25.13 & Linear SVM with standardization \\ \hline
    CNN (back-propagation) & 7.86 & Best without dataset extension. See \cite{high-performance-neural-networks-visual-classification}. \\ \hline
    Our Method (SCNG) & 11.59 & MP-$11$ with $p=17$ and $q=3$ and $w=1024$ \\ \hline
    Our Method (Gradient) & 11.51 & MP-$11$ with $p=17$ and $q=3$ and $w=1024$ \\ \hline
    Our Method (Random) & 12.46 & MP-$11$ with $p=17$ and $q=3$ and $w=1024$ \\ \hline
    Our Method (SCNG) & 10.87 & MP-$25$ with $p=17$ and $q=3$ and $w=1024$ \\ \hline
    Our Method (Gradient) & 11.01 & MP-$25$ with $p=17$ and $q=3$ and $w=1024$ \\ \hline
    Our Method (Random) & 11.81 & MP-$25$ with $p=17$ and $q=3$ and $w=1024$ \\ \hline
  \end{tabularx}
\end{table}
\renewcommand{\arraystretch}{1.0}

After features were extracted from a dataset, a classifier was trained on the features training dataset. We employed a simple Linear SVM, as implemented by LIBLINEAR \cite{liblinear}. The regularization parameter $C$ was fine-tuned through cross-validation. A subset of $20\%$ of the training set instances was put aside for this purpose. We tested $C$ with $20$ possible values, logarithmically distributed between $10^{-3}$ and $10^{-1}$. For each value, $5$ random $50/50$ splits of the dataset were performed. The classifier was trained on one half of the data and was tested on the other half. Average scores were then computed for model selection purposes. After a good $C$ was found, a classifier was built on the whole training dataset and evaluated on the provided testing dataset. We also tried Logistic Regression and Gaussian Kernel SVMs. For the former, the scores were slightly lower, while for the latter, the scores were similar. We thus preferred the Linear SVM for performance and computational reasons.

\begin{figure}
\centering
\includegraphics[width=0.49\textwidth]{WSOM_Paper_SCNG_MNIST_Basis.png}
\includegraphics[width=0.49\textwidth]{WSOM_Paper_SCNG_SmallNORB_Basis.png}
\caption{\textbf{SCNG} learned features for the MNIST (left) and NORB (right) datasets. For NORB, the ZCA pre-processing step was applied.}
\label{fig:LearnedFigures}
\end{figure}

\section{Conclusion}

In this paper we have shown a way to build a complex image classifier with the Sparse Coding Neural Gas. The classifier consists of several components (feature learning, coding system, proper classifier), and we investigated whether the Sparse Coding Neural Gas algorithm is applicable as a feature learning method. The resulting classifier was tested on the MNIST and NORB datasets and found to perform close to state-of-the-art. Better methods usually employ two or more feature extraction layers, more sophisticated classifiers, or groups of classifiers which vote on the final class. Our simple setup is promising to reach or perhaps even surpass state-of-the-art results with some further extensions along these lines.

\subsubsection*{Acknowledgments.} Horia Coman would like to thank, for supporting this work, The German Academic Exchange Service Programme ``Ostpartnerschaften'', the University of L\"{u}beck,  and The Sectoral Operational Programme Human Resources Development 2007-2013 of the Romanian Ministry of Labor, Family and Social Protection through the Financial Agreement POSDRU/86/1.2/S/61756.

\begin{thebibliography}{4}

\bibitem{best-architecture-object-recognition} Jarrett, K. and Kavukcuoglu, K. and Ranzato, M. and Keen, Y.: What is the Best Multi-Stage Architecture for Object Recognition?. Proc. International Conference on Computer Vision (ICCV'09) (2009)

\bibitem{emergence-sparse-coding} Olshausen, B. and Field, D.: Emergence of Simple-Cell Receptive Field Properties by Learning a Sparse Code for Natural Images. Nature 381, 607--609 (1996)

\bibitem{sparse-coding-strategy-V1} Olshausen, B. and Field, D.: Sparse Coding with an Overcomplete Basis Set: A Strategy Employed by V1?. Vision Research 37, 3311--3325 (1998)

\bibitem{tiny-images} Krizhevsky, A.: Learning Multiple Layers of Features from Tiny Images. (2009)

\bibitem{learning-convolutional-feature-hierarchies} Kavukcuoglu, K. and Sermanet, P. and Boureau, Y. and Gregor, K. and Mathieu, M. and LeCun, Y.: Learning Convolutional Feature Hierarchies for Visual Recognition. Advances in Neural Information Processing Systems (NIPS 2010), vol. 23 (2010)

\bibitem{gradient-based-learning} LeCun, Y. and Bottou, L. and Bengio, Y. and Haffner, P.: Gradient-Based Learning Applied to Document Recognition. Intelligent Signal Processing, 306--351. IEEE Press (2001)

\bibitem{convolutional-networks-vision} LeCun, Y. and Kavukcuoglu, K. and Farabet, C.: Convolutional Networks and Applications in Vision. Proc. International Symposium on Circuits and Signals (ISCAS'10). IEEE Press (2010)

\bibitem{best-practices-cnn} Simard, P. Y. and Steinkraus, D. and Platt, J. C.: Best Practices for Convolutional Neural Networks Applied to Visual Document Analysis. Int'l Conference on Document Analysis and Recognition, 958--963 (2003)

\bibitem{simple-method-sparse-coding} Labusch, K. and Barth, E. and Martinetz, T.: Simple Method for High-Performance Digit Recognition Based on Sparse Coding. IEEE Transactions on Neural Networks, vol. 19, no. 11, 1985--1989 (2008)

\bibitem{sparse-features-audio-classification} Henaff, M. and Jarret, K. and Kavukcuoglu, K. and LeCun, Y.: Unsupervised Learning for Scalable Audio Classification. Proceedings of International Symposium on Music Information Retrieval (ISMIR'11) (2011)

\bibitem{sparse-coding-neural-gas-1} Labusch, K. and Barth, E. and Martinetz, T.: Sparse Coding Neural Gas: Learning of Overcomplete Data Representations. Neurocomputing, vol. 72, no. 7-9, 1547--1555 (2009)

\bibitem{sparse-coding-neural-gas-2} Labusch, K. and Barth, E. and Martinetz, T.: Learning Data Representations with Sparse Coding Neural Gas. Proceedings of the 16th European Symposium on Artificial Neural Networks, 233-238 (2008)

\bibitem{sparse-coding-neural-gas-3} Labusch, K. and Barth, E. and Martinetz, T.: Demixing Jazz-Music: Sparse Coding Neural Gas for the Separation of Noisy Overcomplete Sources. Neural Network World, vol. 19, no. 5, 561-579 (2009)

\bibitem{sparse-coding-neural-gas-4} Labusch, K. and Barth, E. and Martinetz, T.: Sparse Coding Neural Gas for the Separation of Noisy Overcomplete Sources. ICANN(1), 788-789 (2008)

\bibitem{neural-gas-1} Martinetz, T. and Schulten, K.: A ``Neural-Gas'' Network Learns Toplogies. Artificial Neural Networks, vol. 1, 397--402, (1991)

\bibitem{neural-gas-2} Martinetz, T. and Berkovich, S. and Schulten. K: ``Neural-Gas'' Network for Vector Quantization and its Application to Time-Series Prediction. IEEE Transactions on Neural Networks, vol. 4, no. 4, 397--402 (1991)

\bibitem{random-weights-feature-learning} Saxe, A. and Koh, P. W. and Chen, Z. and Bahand, M. and Suresh, B. and Ng, A.: On Random Weights and Unsupervised Feature Learning. Proceedings Of the Twenty-Eight International Conference on Machine Learning (2011)

\bibitem{self-taught-learning} Raina, R. and Battle, A. and Lee, H. and Packer, B. and Ng, A.: Self-taught Learning: Transfer Learning from Unlabeled Data. Proceedings of the 24th International Conference on Machine Learning (ICML '07), 759--766 (2007)

\bibitem{undetermined-minimal-L1} Donoho, D.: For Most Large Underdetermined Systems of Linear Equations the Minimal $\mathcal{L}_1$-norm Solution is also the Sparsest Solution. Communications on Pure and Applied Mathematics, vol. 59, 797--766 (2007)

\bibitem{matchingpursuit1} Mallat, Z. and Zhang, Z.: Matching Pursuits With Time-Frequency Dictionaries. IEEE Transactions on Signal Processing, vol. 41, 3397--3451 (1993)

\bibitem{matchingpursuit2} Davis, G. and Mallat, S. and Zhang. Z.: Adaptive Time-Frequency Decomposition with Matching Pursuits. SPIE Journal of Optical Engineering, vol. 33, 2183-2191 (1994)

\bibitem{orthopursuit} Pati, Y. and Rezaiifar, R. and Krishnaprasad, P.: Orthogonal Matching Pursuit: Recursive Function Approximation with Applications to Wavelet Decomposition. Asilomar Conference on Signals, Systems and Computers (1993)

\bibitem{pursuitdifferences} Blumensath, T. and Davies, M.: On the Difference Between Orthogonal Matching Pursuit and Orthogonal Least Squares (2007)

\bibitem{oja-rule} Oja, E.: Simplified Neuron Model as a Principal Component Analyzer. Journal of Mathematical Biology, vol. 15, 267-273 (1982)

\bibitem{learning-methods-invariance-pose-lighting} LeCun, Y. and Huang, F.-J. and Bottou, L.: Learning Methods for Generic Object Recognition with Invariance to Pose and Lighting. Proceedings of CVPR'04 (2004)

\bibitem{liblinear} Fan, R.-E. and Chang, K.-W. and Hsieh, C.-J. and Wang, X.-R. and Lin, C.-J.: LIBLINEAR: A Library for Large Linear Classification. J. Mach. Learn. Res, vol. 9, 1871--1874 (2008)

\bibitem{high-performance-neural-networks-visual-classification} Cire\c{s}an, D. and Meier, U. and Masci, J. and Gambardella, L. and Schmidhuber, J.: High-Performance Neural Networks for Visual Object Classification (2011)

\end{thebibliography}

\end{document}


%\let\oldthesubsection=\thesubsection
\renewcommand{\thesubsection}{\Roman{subsection}}

%A dolgozat olyan nyelvtechnológiai előfeldolgozó algoritmusokat mutat be, melyek hatékonyan képesek szövegek elemzésére agglutináló nyelvek esetén.
%Vizsgálataimat magyar nyelvre végeztem, de a módszerek kidolgozása során törekedtem a nyelvfüggetlenségre.
%Így, a bemutatott eljárások más, hasonló struktúrájú nyelvek esetén is sikerrel alkalmazhatóak.
%%Munkám során kiemelt szerep jutott a morfológiai egyértelműsítés feladatának, mivel ennek kimenetére számtalan információkinyerő rendszer épít.
%
%Az első téziscsoportban a általános magyar nyelvű morfológiai egyértelműsítés területén elért eredményeimről számolok be.
%Ezt követően bemutatom a létrehozott annotáló eszköz egy gyakorlati alkalmazását.
%Végül, ismertetem azon új előfeldolgozó eljárásokat, melyek zajos (klinikai) szövegeket is hatékonyan képesek elemezni.
%

\subsection{Effective morphological tagging methods for morphologically rich languages} %TODO: igeidők
\label{thes:morf}

Full morphological tagging is a complex task composed of two parts.
Beside identifying morpho-syntactic tags, lemmata of words must be computed as well.
While the first task is a well-known problem of \acrlong{nlp}, the latter one is often neglected.
Results are summarized by describing  the new lemmatization method first, followed by the full tagging systems.

\begin{core}
\begin{thesis}\label{thes:morf-lemma}
I developed a new lemmatization method for agglutinative languages.
The presented algorithm is based on the output of a morphological analyzer. It can handle both known and unknown words effectively by incorporating diverse stochastic models.
Results presented show that the new system has high accuracy on Hungarian texts.
\end{thesis}

\begin{pub}
\cite{Orosz2011,Orosz2012,Orosz2012a,Orosz2013a}
\end{pub}
\end{core}

The proposed algorithm performs lemmatization in two steps.
First, it uses a morphological analyzer and a guesser component to generate lemma candidates, then disambiguation is performed using stochastic models.
The latter part is carried out calculating the score ($S$) of each lemma ($l$) for a given word ($w$) and tag ($t$) using the interpolation of two different models:
\begin{equation} %\label{lemma-interpolated}
S(l|w,t) = P(l)^{\lambda_1} P(l,t|w)^{\lambda_2}
\end{equation}

The system combines a simple unigram model with the output of a suffix-based guesser.
To calculate the lambda parameters, guesses of models are evaluated on the training data, then the better model's score gets increased while that of the worse one is decreased.

Several experiments have been presented on the Szeged Corpus showing that the proposed method has superior accuracy for Hungarian compared to other available tools.

\thesisline%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{core}
\begin{thesis}\label{thes:morf-tagging}
I designed a hybrid morphological tagging system (PurePos\footnote{The presented system is open source and is freely available at \href{https://github.com/ppke-nlpg/purepos}{https://github.com/ppke-nlpg/purepos}}) for less-resourced and agglutinative languages.
The method relies on stochastic methods incorporating the output of a morphological analyzer.
Its lemmatization component utilizes algorithms presented in Thesis \ref{thes:morf-lemma}.
Furthermore, the tool is built up in a way to be able to incorporate domain-specific rules effectively.
Experiments confirm its state-of-the-art accuracy for Hungarian and resource-scare scenarios.
\end{thesis}

\begin{pub}
\cite{Orosz2011,Orosz2012,Orosz2012a,Orosz2013a}
\end{pub}
\end{core}

\begin{figure}[ht]
  \centering
  \includegraphics[width=1\textwidth]{MorphTagging/architecture.png}
  \caption{The architecture of the full morphological tagging tool}
  \label{fig:purepos-arch_en}
\end{figure}

The architecture of PurePos (cf. Figure \ref{fig:purepos-arch_en}) is built up to allow multiple models cooperating effectively.
The disambiguation is carried out in multiple steps.
The data flow starts from a \acrshort{ma} providing word analyses as \emph{(lemma, tag)} pairs.
Next, trigram-tagging methods (see \cite{Brants2000,Halacsy2007}) are employed for selecting morpho-syntactic labels of words.
%In fact, these algorithms have been adapted to fit agglutinative languages.
Finally, lemmatization is carried out employing the methods presented in Thesis \ref{thes:morf-lemma}.

\begin{figure}[H]
  \centering
  \includegraphics[width=1\textwidth]{MorphTagging/msd_token.png}
  \caption{Learning curves of full morphological taggers on the Szeged Corpus (using Humor labels)}
  \label{fig:humor-token_en}
\end{figure}

Several experiments were carried out measuring the performance of PurePos on the Szeged Corpus \cite{Csendes2004}.
Results show that the new method yields very high (96.26\%) full tagging accuracy on Hungarian.
Moving on, I also compared existing tagging systems with the presented one on a less-resourced scenario.
These experiments showed (cf. Figure \ref{fig:humor-token_en}) that PurePos can be successfully used even when the training dataset is limited.
Finally, all the hybrid enhancements of PurePos ware evaluated one-by-one, showing that they can be used to fix several sorts of errors.

\thesisline%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Although, methods of Thesis group \ref{thes:morf-tagging} have high accuracy, it was shown that they can be improved further.
Therefore, a combination technique is presented increasing the ceiling of morphological tagging tools' performance for agglutinative languages.

\begin{core}
\begin{thesis}
I developed a methodology for combining morphological tagging systems effectively.
The system presented selects the best lemma and tag candidates separately using two different combination methods.
These components are trained with cross-validation using instance based learning.
I showed that my method can significantly reduce the number of errors of existing annotation tools.
\end{thesis}

\begin{pub}
\cite{Laki2013a,Orosz2013c,Orosz2013d}
\end{pub}
\end{core}

First of all, discrepancy of tagging systems was analyzed.
For this, I designed a new metric (Own Error Rate) which measures the differences of output of taggers.
It turned out that the most typical mistakes of HuLaPos \cite{Laki2013} and PurePos are different enough to be aggregated.

Following this, the most common combination techniques were investigated considering their applicability to full morphological tagging.
Next, a new combination method was presented involving adapted feature sets for a morphologically rich language.
It utilizes instance based learning \cite{Aha1991} and trains classifiers with cross-validation, which
can employ the whole training dataset for both the baseline tools and the level-one learners. %TODO Nóri fordítás
The novelty of the presented method is its architecture (cf. Figure \ref{fig:comb3_en}) which allows us to utilize different combiners for the lemmatization and \acrshort{pos} tagging subtasks.

\begin{figure}[H]
  \centering
  \includegraphics[scale=0.15]{MorphTagging/comb3.png}
  \caption{Combining the output of two PoS taggers and lemmatizers}
  \label{fig:comb3_en}
\end{figure}

Finally, evaluation experiments were presented indicating that the number errors of the best tagger can be decreased further.
The new algorithm could reduce the number of errors of PurePos by 28.90\%.

\subsection{Measuring morpho-syntactic complexity using morphological annotation algorithms}
\label{thes:mlu}

Measuring morpho-syntactic complexity is usually carried out calculating \acrlong{mlu}s.
This metric is often computed in words for analytical languages, while morphemes (\acrshort{mlum}) are used for morphologically complex ones.
Although automatic methods and tools exist for e.g English, other less-resourced languages lack such systems.
Therefore, \acrshort{mlum} could be only computed manually, which is a rather time-consuming task.

This thesis group presents\footnote{This research has been conducted together with Kinga Jelencsik-Mátyus.
%Manual annotation of the data were performed by both of us, while the morpheme counting principles are her work.
My contributions are the construction of the tagging chain, its adaptation and the automatization of the MLUm calculation.} methods for processing speech transcripts effectively and estimating \acrlong{mlum} automatically.
%Our solution relies on the PurePos tagger tool (cf. Thesis \ref{thes:morf-tagging}).
%Results also indicate that the labor-intense manual work can be replaced with our new method.

\begin{core}
\begin{thesis}
\label{thes:spoken-morf-tagging}
I developed a hybrid morphological tagging chain for Hungarian child-language transcripts.
My method builds on top of the results presented in Thesis \ref{thes:morf-tagging} by adapting them to the domain.
Evaluation shows that performance of the method is comparable with that of tagging methods for written corpora.
Moreover, experiments indicate that the algorithm presented is accurate enough to be used in further applications.
\end{thesis}

\begin{pub}
\cite{Matyus2014,Orosz2014c}
\end{pub}
\end{core}

The proposed method adapts the algorithms introduced in Thesis \ref{thes:morf-tagging} for spoken Hungarian.
For this, the Humor \acrlong{ma} was augmented first with analyses of words typical to the domain.
Next, the output of PurePos was adjusted utilizing domain-specific knowledge.

For this, a gold corpus of about 1,000 utterances from the \acrshort{hukilc} was created by the manual annotation of texts.
Additionally, a new tagging scheme was designed representing the characteristics of spoken language properly.
%These manually corrected texts enabled us to develop and validate the proposed methods.

The evaluation of the chain resulted in 96\% token-level precision, which is comparable with that of taggers for corpora of written language.
Therefore, my investigation showed that PurePos is an appropriate base for tagging corpora of transcribed spoken texts.

\thesisline%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{core}
\begin{thesis}
\label{thes:mlu-estimation}
I proposed a new algorithm for estimating morpho-syntactic complexity (calculating \acrlong{mlum}) in Hungarian child language transcripts.
The method uses the morphological tagging chain of Thesis \ref{thes:spoken-morf-tagging} as a base.
%In doing so, it computes lengths of utterances in its output.
Evaluation of the system indicates that the methodology presented can properly replace the time-consuming manual computation of human annotators.
\end{thesis}

\begin{pub}
\cite{Matyus2014,Orosz2014c}
\end{pub}
\end{core}

The estimation method analyzes morphological annotations of tokens.
Words known by the analyzer are decomposed by Humor, while lengths of unknown words are guessed based on their \acrshort{pos} labels.
This is followed by morpheme counting rules implementing linguistic guidelines, thus providing relevant estimates.

As regards resources, a manually checked corpus was created for the experiments.
Evaluation of the methods on this dataset shows that my results highly correlate (0.9901) with counts of human annotators.
Further on, I showed that the mean relative error of the method is only 4.49\%.
Thus, the proposed algorithm can properly replace the labor-intensive human computation.

\subsection{Effective preprocessing methods for a less-resourced noisy domain}
\label{thes:clin}

More and more electronic health records are produced in hospitals containing valuable but hidden knowledge.
Since doctors cannot spend enough time on writing their reports properly, notes often contain numerous errors.
Because of such mistakes, processing of these texts cannot be carried out using general-purpose tools.
Moreover, while several algorithms are becoming available for English, Hungarian and other morphologically rich languages are still neglected.

\begin{core}
\begin{thesis}%{II.1/b}
\label{thes:clin-segment}
I developed a new framework which segments noisy clinical records into words and sentences accurately.
The method is built on top of well-known tokenization rules (e.g. \cite{Halacsy2004}), however, it augments them with unsupervised heuristics.
Evaluations showed that the algorithm can properly identify word and sentence boundaries in noisy clinical notes.
Results also indicate that other systems available cannot handle such erroneous texts.
\end{thesis}

\begin{pub}
\cite{Orosz2013d,Orosz2014a,Orosz2014x}
\end{pub}
\end{core}

\begin{figure}[H]
  \centering
  \includegraphics[scale=0.2]{Clinical/clin_segm_arch.png}
  \caption{The architecture of the proposed method}
  \label{fig:clin-segment-arch_en}
\end{figure}

The proposed method builds on pattern-matching algorithms taken from general-purpose tokenization tools.
Even though these methods perform with high accuracy, their recall still stays low.
Therefore, this study proposes a method (see Figure \ref{fig:clin-segment-arch_en}) which improves their performance using unsupervised heuristics and a domain-specific morphologic analyzer.
First, the scaled $\log\lambda$  method \cite{kiss2006unsupervised} was adapted by introducing new scaling factors.
Next, the Humor morphological analyzer was utilized to reveal further sentence boundaries.

The evaluation of the framework was carried out on a manually segmented corpus.
Numerous metrics (such as precision, recall, F-score) were employed measuring the performance of the proposed tool.
%Thus, I could analyze both the tokenization and the sentence boundary detection accuracy of the tool.
Moreover, existing Hungarian approaches were also compared with the proposed one.

Results show that other systems available can only produce low quality segmentation.
Most of them yields F-scores less than 50\% in sentence boundary identification.
On the contrary, the method proposed can detect both token and sentence boundaries accurately, producing F-values over 90\%.

%%%%%%%%%%%%%%%%%%%%%%%
\thesisline%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%

\begin{core}
\begin{thesis}%{II.2}
\label{thes:clin-pos}
I showed that tagging methods of Thesis \ref{thes:morf-tagging} can be applied for annotating electronic health records satisfactorily.
In doing so, PurePos was adjusted with stochastic and symbolic domain adaptation techniques.
The quality of the annotation produced is comparable with that of general written tagger tools.
\end{thesis}

\begin{pub}
\cite{Orosz2013,Orosz2014b,Orosz2014x}
\end{pub}
\end{core}

First of all, an extended version of the Humor analyzer was used as a base of the tagging chain, since it was prepared\footnote{The lexicon extension was carried out by Attila Novák \cite{Orosz2014} .} for electronic health records.
Further on, the tagging chain was improved using a detailed error analysis of the baseline tagger.

For this, a manually annotated corpus was created containing texts of clinical notes.
Results on this dataset show that the improved system performs significantly better (93.73\%) than the baseline system (88.09\%).
However, future work might target the segmentation and tagging tasks with a unified framework, since both systems have the most problems with abbreviated terms.

\let\thesubsection=\oldthesubsection


\section{New Keynesian Dynamic Stochastic General Equilibrium (DGSE) models}


    \filetitle{stdscale}{Re-scale all std deviations by the same factor}{model/stdscale}

	\paragraph{Syntax}\label{syntax}

\begin{verbatim}
m = stdscale(m,factor)
\end{verbatim}

\paragraph{Input arguments}\label{input-arguments}

\begin{itemize}
\item
  \texttt{m} {[} model {]} - Model object whose std deviations will be
  re-scaled.
\item
  \texttt{factor} {[} numeric {]} - Factor by which all the model std
  deviations will be re-scaled.
\end{itemize}

\paragraph{Output arguments}\label{output-arguments}

\begin{itemize}
\itemsep1pt\parskip0pt\parsep0pt
\item
  \texttt{m} {[} model {]} - Model object with all of its std deviations
  re-scaled.
\end{itemize}

\paragraph{Description}\label{description}

\paragraph{Example}\label{example}


\subsection{The new Keynesian Phillips curve}


%\section{Setting the common components} \label{sec:common}
%------------------------------------------------------

%-------------------------------------------------------------------------------
\section{Setting Physical Constants} \label{subsec:const}
%-------------------------------------------------------------------------------

The some physical constants are able to modify in \namelist{PARAM_CONST} in the configuration file.

\editboxtwo{
\verb|&PARAM_CONST                           | & \\
\verb| CONST_RADIUS            = 6.37122D+6, | & ; Radius of the planet [m] \\
\verb| CONST_OHM               = 7.2920D-5,  | & ; Angular velocity of the planet [1/s] \\
\verb| CONST_GRAV              = 9.80665D0,  | & ; Standard acceleration of gravity [m/s2] \\
\verb| CONST_Rdry              = 287.04D0,   | & ; Specific gas constant (dry air) [J/kg/K] \\
\verb| CONST_CPdry             = 1004.64D0,  | & ; Specific heat (dry air,constant pressure) [J/kg/K] \\
\verb| CONST_LAPS              = 6.5D-3,     | & ; Lapse rate of ISA [K/m] \\
\verb| CONST_Pstd              = 101325.D0,  | & ; Standard pressure [Pa] \\
\verb| CONST_PRE00             = 100000.D0,  | & ; Pressure reference [Pa] \\
\verb| CONST_Tstd              = 288.15D0,   | & ; Standard temperature (15C) [K] \\
\verb| CONST_THERMODYN_TYPE    = 'EXACT',    | & ; Internal energy type \\
\verb| CONST_SmallPlanetFactor = 1.D0,       | & ; Factor for small planet [1] \\
\verb|/                                      | & \\
}

\noindent
When \nmitem{CONST_THERMODYN_TYPE} is 'EXACT', temperature dependencies of the latent heat are considered.
%
When \nmitem{CONST_THERMODYN_TYPE} is 'SIMPLE', specific heat of water categories are set to that of dry air,
and temperature dependencies of the latent heat are ignored.
%
The \nmitem{CONST_RADIUS} is multiplied by the \nmitem{CONST_SmallPlanetFactor}.
At the same time, \nmitem{CONST_OHM} is multiplied by the inverse of the \nmitem{CONST_SmallPlanetFactor}.

%-------------------------------------------------------------------------------
\section{Setting Calendar} \label{subsec:calendar}
%-------------------------------------------------------------------------------

The calendar type can be specified in \namelist{PARAM_CALENDAR} in the configuration file.
Gregorian calendar is used in default.

\editboxtwo{
\verb|&PARAM_CALENDAR             | & \\
\verb| CALENDAR_360DAYS = .false. | & ; Whether 12x30 days calendar is used? \\
\verb| CALENDAR_365DAYS = .false. | & ; Whether leap year is considered? \\
\verb|/                           | & \\
}

\noindent
The setting of the calendar affects calculation of solar zenith angle.
It is calculated so that length of one year and the full circle of the ecliptic match.
Note that the external data in the different calendar should not be read.

When \nmitem{CALENDAR_360DAYS} is \verb|.true.|,
the calendar that one year is 12 months and one month is 30 days, is set.
%
When \nmitem{CALENDAR_365DAYS} is \verb|.true.|,
the Gregorian calendar is used without leap year.

%-------------------------------------------------------------------------------
\section{Setting Random Number Generator} \label{subsec:random}
%-------------------------------------------------------------------------------

A parameter of the random number generator is set in \namelist{PARAM_RANDOM} in the configuration file.

\editboxtwo{
\verb|&PARAM_RANDOM         | & \\
\verb| RANDOM_FIX = .false. | & ; Random seed is fixed? \\
\verb|/                     | & \\
}

\noindent
The intrinsic function of random number generator is used in the scale library. Note that the generated number is pseudorandom.
The seed of the random number is determined by current datetime, cpu time, and the process id.
%
When \nmitem{RANDOM_FIX} is \verb|.true.|, the seed is fixed by specific number.
This option is useful to reproduce the simulation results, which use the random perturbation for the initial field.

%-------------------------------------------------------------------------------
\section{Setting Performance Profiler} \label{subsec:prof}
%-------------------------------------------------------------------------------

Parameters of the performance profiler are set in \namelist{PARAM_PROF} in the configuration file.

\editboxtwo{
\verb|&PARAM_PROF                 | & \\
\verb| PROF_rap_level   = 2       | & ; Rap output level \\
\verb| PROF_mpi_barrier = .false. | & ; Add barrier command of MPI in every rap? \\
\verb|/                           | & \\
}

\noindent
To measure the elapse time, utility functions (PROF\_rapstart,PROF\_rapend) are inserted in the source code.
These measurement sections are also used for detailed performance profiling.
%
The results of the rap time are displayed at the end of the log file.
If \nmitem{IO_LOG_ALLNODE} in  \namelist{PARAM_IO} is \verb|.true.|, the result of each process is reported to each log file individually.
If \nmitem{IO_LOG_SUPPRESS} in \namelist{PARAM_IO} is \verb|.true.|, the result is sent to the standard output.
%
Each measurement section has the output level. The section, which has the output level larger than the \nmitem{PROF_rap_level} are not measured.

When \nmitem{PROF_mpi_barrier} is \verb|.true.|, barrier command of MPI is called before and after getting current time.
This option is useful to separate the computation time and communication time.
The computation time often shows large imbalance among the processes.

%-------------------------------------------------------------------------------
\section{Setting Statistics Monitor} \label{subsec:statistics}
%-------------------------------------------------------------------------------

Parameters of the statistics monitor are set in \namelist{PARAM_STATISTICS} in the configuration file.

\editboxtwo{
\verb|&PARAM_STATISTICS                    | & \\
\verb| STATISTICS_checktotal     = .false. | & ; Calculate and report variable totals to logfile? \\
\verb| STATISTICS_use_globalcomm = .false. | & ; Calculate total with global communication? \\
\verb|/                                    | & \\
}

\noindent
When \nmitem{STATISTICS_checktotal} is \verb|.true.|, domain total of some variables are calculated and reported to the log file, for debugging.
%
When \nmitem{STATISTICS_use_globalcomm} is \verb|.true.|, global domain total is calculated by using global communication.
This option can slow the simulation time.
When the option is \verb|.false.|, the total is calculated in the spatial region allocated for each process.


\section{Modeling synchrony}
\begin{figure}[!h]
\setlength{\fboxsep}{12pt}
\begin{center}
\begin{boxedminipage}{12cm}
The functionality is available to all participants. The functionality is parametrized with variable $\tau$, a set of parties $\mathcal{P}={p_1,\dots, p_n}$, and a set $F$ of functionalities.
 For each party $p_i\in\mathcal{P}$ it manages variable $\dpi$. For each $\mathcal{F}\in F$
it manages variable $\df$

Initially, $\tau=0, \mathcal{P}=\emptyset$ and $F=\emptyset$.

\begin{itemize}
\item[-] Upon receiving $(\gupdate,\sid)$ from some party $p_i\in\mathcal{P}$ set $\dpi=1$
execute \emph{Round-Update} and forward $(\gupdate, \sid , p_i)$ to $\adv$.
\item[-] Upon receiving $(\gupdate, \sid)$ from some functionality $\mathcal{f}\in F$ set $\df=1$, execute \emph{Round-Update} and return $(\gupdate, \sid , F)$ to $F$.
\item[-] Upon receiving $(\gread, \sid)$ from any participant (including the environment, the adversary, or any ideal-shared or local-functionality) return $(\gread, \sid, \tau)$ to the requester.
\end{itemize}

Procedure \emph{Round-Update}:
If $\df=1$ for all $\mathcal{F}\in F$ and $\dpi =1$ for all honest $p_i\in\mathcal{P}$,
then set $\tau=\tau+1$ and reset $\df=0$ and $\dpi=0$ for all parties in $\mathcal{P}$.

\end{boxedminipage}
\end{center}
\caption{The functionality $\gclock$}
\label{fig:gclock}
\end{figure}