% !TEX root = manual.tex

\chapter{Uncertainty Quantification Methods and Tools}
\label{chapter:uq}

\section{Overview}
\label{sec:uqoverview}
For uncertainty quantification (UQ) and validation studies the UQ Toolkit (UQTk) library is being used.
UQTk (\url{www.sandia.gov/UQToolkit}) is a lightweight C++ library, developed in Sandia National Laboratories, California,
that primarily offers tools for surrogate model construction and uncertainty propagation with
polynomial chaos expansions (PCE), as well as model calibration and validation.

UQTk Version 2.0 is released under the GNU Lesser General Public License (LGPL).

\section{Download and compilation}
\label{sec:uqdownload}

A tar-ball with the source code, tutorials, examples and documentation can be
downloaded from \url{www.sandia.gov/UQToolkit/uqtk\_download.html}

Before compilation, one should set a file \texttt{config/config.site} with computer-specific compilation options.
Both C++ and Fortran compilers are required. The file \texttt{config/config.gnu} provides a good
starting point as a sample configuration file.

Compilation is simply invoked by
\begin{ShellCmd}
make all
\end{ShellCmd}

\section{Structure}
\label{sec:uqstruct}

Below we present parts of the UQTk structure that are relevant to UQ and validation studies on \sstmacro.

\begin{itemize}
\item \texttt{config}: configuration file location
\item \texttt{doc\_cpp}: doxygen documentation
\item \texttt{src\_cpp}: source code - all libraries reside here
\subitem \texttt{src\_cpp/lib}: location of all compiled libraries in C++ and Fortran
\subitem \texttt{src\_cpp/apps}: command line utilities for performing various UQ tasks (source codes)
\subitem \texttt{src\_cpp/bin}: command line utilities for performing various UQ tasks (executables)
\item \texttt{examples\_cpp}: example codes and scripts
\subitem \texttt{examples\_cpp/uq\_surr}: a set of scripts that automates the forward UQ task utilizing the command line \texttt{apps} above.
\end{itemize}

\section{Applications/Capabilities}
\label{sec:uqapp}

The two basic UQ tasks, enabled by UQTk, are \emph{forward UQ} and \emph{inverse UQ} as outlined in the next subsections.

\subsection{Forward UQ: uncertainty propagation and global sensitivity analysis}
\label{subsec:fuq}

The main technique we employ for forward UQ is the spectral Polynomial Chaos expansions (PCEs).
A PCE for a given model allows for a) efficient uncertainty propagation, b) very fast global sensitivity analysis, and
c) cheap surrogate model construction that can replace the original model in sampling-intensive studies such as calibration,
optimization, or, generally, inverse UQ.

The PCE library consists of a class for PC objects allowing both intrusive and non-intrusive uncertainty propagation.
For \sstmacro, only non-intrusive methods are feasible that deal with the \sstmacro model as a \emph{black-box} without the need to rewrite or reformulate it.

\subsubsection{Generic workflow:}

A set of scripts that illustrate the essential forward UQ tasks is located in  \texttt{examples\_cpp/uq\_surr}.
To enable running all scripts one should set an environment variable \texttt{UQTK\_SRC} that points to the location of UQTk.

\begin{ShellCmd}
setenv UQTK_SRC location/of/uqtk/in/your/computer
\end{ShellCmd}

A generic workflow consists of 4 steps:
\begin{enumerate}
\item Generate parameter samples to run the forward model at, for PC construction and validation, \texttt{gen\_sam.x}
\begin{ShellCmd}
gen_sam.x <domain_file> <sampling_type(Q/U)> <N_samples> <N_val>
\end{ShellCmd}
The list of arguments:
\subitem \emph{domain\_file}: A file with \bgmth d \endmth rows and 2 columns. \bgmth d \endmth is the total number of parameters being explored.
           The two columns are the lower and upper bound of the corresponding parameter.
\subitem \emph{sampling\_type}: Q (Quadrature) or U (uniformly random). Note that currently in this script set, only Q is supported for further PCE generation.
\subitem \emph{N\_samples}: Number of samples used for training, i.e. for building PCE.
\subitem \emph{N\_val}:  Number of random samples generated for PCE validation. This can be set to 0 to skip validation.

\item Run the black-box model, \texttt{model.x}
\begin{ShellCmd}
model.x
\end{ShellCmd}
This is a black-box model that takes no arguments. However, it expects two input files mparam.dat and minput.dat, and returns the function evaluations in the output file moutput.dat.
\subitem \emph{mparam.dat}: a single column $d\times$1 of the parameters of interest, where d is the number of parameters
\subitem \emph{minput.dat}: controllable input in a matrix form, $N$$\times$$k$, where $k$ is the number of controllable parameters and $N$ is the number of values or model observables.
\subitem \emph{moutput.dat}: the model output in a column format $N$$\times$1.

This is the file that needs to be modified/provided by the user according to the model under study. Currently, a simple function $y=Ae^{Bx}+2Bx$ is implemented, where $A$ and $B$ are the parameters (mparam.dat) and $x$ is the single controllable input parameter (minput.dat). A user-created \texttt{model.x} should accept input files minput.dat and mparam.dat as described above, and it should produce an output file moutput.dat with the formats described above,

\item Obtain PCE for the model, \texttt{uq\_surr.x}
\begin{ShellCmd}
uq_surr.x <domain_file> <sampling_type(Q/U)> <N_samples> <N_val> \
		<P_order> <moutput_surr_filename> <moutput_val_filename>
\end{ShellCmd}

The first four arguments coincide with those from \texttt{gen\_sam.x}, the rest of the arguments are:
\subitem \emph{P\_order}: the PCE surrogate order. Polynomial series is truncated according to the total order.
\subitem \emph{moutput\_surr\_filename}: model output file resulting from running \texttt{model.x} on training samples.
\subitem \emph{moutput\_val\_filename}: model output file resulting from running \texttt{model.x} on validation samples.

\item Postprocess, e.g. global sensitivity analysis, \texttt{pp\_sens.x}
\begin{ShellCmd}
pp_sens.x
\end{ShellCmd}

The current implementation of the post processing script \texttt{pp\_sens.x} takes no arguments. It expects the presence of a PCE information file pccf\_all.dat, which is one of the outcomes of \texttt{uq\_surr.x}. The final global sensitivity results are saved in the \inlineshell{allsens.dat} file of dimensions $N$$\times$$d$, where each row corresponds to a single value for the controllable input (number of controllable inputs = N), and each column corresponds to the sensitivity index of a parameter (number of parameters = d).\\

\medskip

Note that one can run the model \texttt{model.x} in an \emph{online} regime by using the keyword "M" instead of filenames in
\begin{ShellCmd}
uq_surr.x <domain_file> <sampling_type(Q/U)> <N_samples> <N_val> <P_order> M M
\end{ShellCmd}
that effectively incorporates the steps 1-3 above.

\subsubsection{Simple Example:}

The script \texttt{example.x} incorporates the full workflow above for a test function $y=Ae^{Bx}+2Bx$. Try
\begin{ShellCmd}
example.x online
\end{ShellCmd}
which produces, in a newly created folder \texttt{test}, the sensitivity file \inlineshell{allsens.dat} with sensitivity indices with respect to $A$ and $B$ for all values of controllable input $x$.

\end{enumerate}

\subsubsection{How Forward UQ applies to \sstmacro}

In the workflow described above, \sstmacro would replace the black-box simple model given by the equation, and \sstmacro parameters of interest would replace $A$ and $B$.   Note that while
we could explore the entire \sstmacro parameter space using this workflow, the higher the dimensionality of the space the more time process takes, so we would also want to narrow it down to the most important parameters, usually by reasoning, inspection, or simple experimentation.
Also note that running an \sstmacro simulation can be considerably more expensive than evaluating an
analytical model, which is why we typically construct a \textit{surrogate}, or an arbitrarily-complex polynomial that can approximate performance given by \sstmacro.

While the above workflow must currently be manually applied to \sstmacro data, we are working on automating this process and coming up with tutorials explaining how it is done.

\subsection{Inverse UQ: parameter calibration and model validation}
\label{subsec:iuq}

UQTk relies on Bayesian inference methods for model parameter calibration. Model validation is a direct result of calibration
postprocessing. Indeed, a model is considered validated if the calibrated model parameters and the associated uncertainties can explain/predict available data well.
The library \texttt{uqtkmcmc} allows implementation of Bayesian calibration using Markov chain Monte Carlo (MCMC) methods. The MCMC technique essentially
searches the parameter space and compares model results with available data. Note that each parameter sample invokes a model evaluation, and often MCMC requires
many samples for properly estimating the uncertainties. In such cases, the full model, as a black-box, will be replaced by its surrogate, again as a black-box model, that is constructed by the forward UQ techniques described in Subsection~\ref{subsec:fuq}.


\section{Architecture}
\label{chap:prior:sec:arch}
In this section, we first briefly review of neural networks in NLP in Section \ref{chap:prior:sec:arch:nn} so as to obtain insight into how system architecture can effect downstream tasks. Then, in Section \ref{chap:prior:sec:arch:lstm} we introduce Long Short Term Memory (LSTM) \cite{Hochreiter1997LongSM} and describe how it works. In Section \ref{chap:prior:sec:arch:transformer} we introduce the Transformer \cite{Vaswani2017AttentionIA} and briefly discuss how it differs from LSTM.
\subsection{Neural Network in NLP}
\label{chap:prior:sec:arch:nn}
In the last few decades, neural networks have provided language researchers with an effective method of building models that represent the complex phenomena associated with language. In NLP, Recurrent Neural Networks (RNN) are popular building blocks because they can retrain contextual information from previous states. While useful, RNNs are limited in their ability to represent a full sequence by their short-term memory. When presented with long sequences of input, the amount of information needed to represent the total input and how individual components relate grows. Since this state is limited in size as sequences get more prolonged, less context about particular parts of the sequence is kept in the hidden state. The slow loss of prior input's information can be incredibly impactful for for nuances of language like agreement on verges and coreference, which can be many sentences apart. Besides issues in representing inputs with large context windows, RNNs suffer from what is known as the vanishing gradient problem \cite{Hochreiter1998TheVG}. The vanishing gradient problem focuses on how, when training RNNs, we must update the weights of previous hidden states, but as we do so, their gradients get smaller and smaller. Eventually, the gradient on earlier weights is so small that it practically vanishes, and older weights are not updated. To address this shortcoming and improve modeling long term dependencies architectures like LSTM and the Transformer were introduced.
\subsection{ Long  Short  Term  Memory (LSTM)}
\label{chap:prior:sec:arch:lstm}
The LSTM was proposed by Hochreiter et al.,  1997 \cite{Hochreiter1997LongSM} as a neural method that was better suited to avoid vanishing gradients. LSTMs have a gating mechanism that regulates the flow of information from previous states into the current state. What makes the LSTM unique is that it has forget gates that are trainable parameters which choose what information from earlier parts of the input is forgotten. Using these gates, LSTMs can retain relevant information even with large input sizes. Since LSTMs excel at data where their long term dependencies, they have revolutionized the world of time-series prediction \cite{Sagheer2019TimeSF}, speech recognition \cite{Han2017ESEES}, and neural machine translation (NMT) \cite{Jean2015OnUV} to name a few.
\subsection{Transformer}
\label{chap:prior:sec:arch:transformer}
The Transformer is a neural system first proposed by Vaswani et al., 2017  \cite{Vaswani2017AttentionIA} as a system for NMT. The transformer block introduced a way to move from RNNs to an architecture that focuses on the interaction between units in the input data. Attention is a method that selects what portions of the input are related to each other. Instead of keeping a global state like an RNN, Transformers focus on distilling what input items are attended to by other parts of the data. Attention is a learned weight mechanism that represents how much one item in a list is related to others. In NLP, this method has been extremely successful as it allows a model to learn the relative salience of items with regard to every other item in the input. In its initial formulation, the implementation had an encoder and a decoder, consisting of 6 stacked transformer layers. All encoders are identical (but do not share weights) and have two sub-layers: self-attention and a feed-forward network. All decoders are identical (and do not share weights) and consist of 3 sub-layers: self-attention, encoder-decoder attention, and a feed-forward layer. \\
Vaswani et al., 2017 use multi-head attention to produce a linear projection used to attend to different parts of the input. Since this method does not rely on recurrence or convolution, a positional encoding is introduced to allow the Transformer to understand where in a sentence a word is. For more details about the mechanics of the Transformer, we recommend reading The Illustrated Transformer\footnote{https://jalammar.github.io/illustrated-transformer/}. In the original implementation,  stacked layers of transformers train translation systems for English to French and English to German translation, which results in new SOTA models with $\frac{1}{4}$ the training time. \\
The Transformer's ability to model long-term dependencies well and efficiently has made it a natural fit for most NLP tasks. As we will cover in succeeding sections, researchers have used this method to produce robust and usable language representations.


% !TEX root = main.tex
% !TEX spellcheck = en-US

\begin{figure}
	\centering
		\begin{pcvstack}[center,boxed]
		\begin{pchstack}
			\procedure{$\simOH (x)$}
			{
			\pcif H[x] = \bot \pcthen \\
			\pcind H[x] \sample \mathsf{Im}(\ro) \\
		%	Q_\ro \gets \{(x,H[x])\}\\
			\pcreturn H[x]
			  }
			%
			\pchspace
			%
			\procedure{$\simulator\oracleo.\prog(x, h)$}
			{
				\pcif H[x] = \bot \pcthen \\
				\pcind H[x] \gets h \\
				\pcind \Qprog \gets \Qprog \cup \{x\}\\
		%		Q_\ro \gets \{(x,H[x])\}\\
				\pcreturn H[x]
			}
			\pchspace
			%
			\procedure{$\boxed{\simOP(\inp, \wit)} \ \simOP'(\inp)$}
			{
				\boxed{\pcassert (\inp,\wit)\in \REL} \\
				\pi \gets \simulator^{\simOH,\simulator\oracleo.\prog}(\srs,\inp)\\
				Q \gets Q \cup \{(\inp,\pi)\}\\
				\pcreturn \pi
			}
		\end{pchstack}
	\end{pcvstack}
	\caption{Simulation oracles: $\srs$ is the finalized SRS, only $\simOP'$ allows for simulation of false statements}
	\label{fig:real_simulator_oracles}
\end{figure}

\subsection{Trapdoor-Less Zero-Knowledge (TLZK)}

%We model $\simulator$ as a stateful
%algorithm that runs in two modes. The first mode,
%$(h, st') \gets \simulator (1, st, \srs, q)$ answers random oracle calls to $\ro$ on input
%$q$. The second mode $(\zkproof, st') \gets \simulator (2, st, \srs, \inp)$ simulates the
%actual argument for instance $\inp$.

% \begin{itemize}
% \item $\simOH (\srs,q)$ denotes an oracle that returns the first output of
%   $\simulator (1, st, \srs, q)$;
% \item $\simOP (\srs,\inp, \wit)$ denotes an oracle that returns the first output of
%   $\simulator (2, st, \srs, \inp)$ if $(\inp, \wit) \in \REL$, and returns $\bot$ otherwise;
% \end{itemize}

% We call a proof system $\proofsystem$ \emph{zero-knowledge} if for any
% $\REL \in \RELGEN(\secparam)$, and adversary $\adv$ there exists a $\ppt$ simulator
% $\simulator$ such that for any $(\inp, \wit) \in \REL$, $\eps_0 \approx \eps_1$,
% where
% \[
%   \eps_b = \condprob{\adv^{\oracleo_b} (\srs)}{\srs \gets \kgen (\REL)}\,,
% \]
% \changedm{where $\oracleo_0$ on input $(1, q)$ responds with $h$ such that
%   $(h, st') \gets \simOH (1, st, q)$, where $st$ and $st'$ are old and new
%   states of the simulator $\simulator$ and on input $(2, \inp, \wit)$ returns
%   $\zkproof \gets \simOP (2, st, \srs, \inp)$. Alternatively, $\oracleo_1$ on
%   input $(1, q)$ responds with $h \gets \ro (q)$ and on input $(2, \inp, \wit)$
%   returns $\zkproof \gets \prover (\srs, \inp, \wit)$.}
% 	%
% We call zero knowledge \emph{perfect} if the distributions are equal and
% \emph{computational} if they are indistinguishable for any $\ppt$ distinguisher.
\iffalse
We call a proof system $\proofsystem$ \emph{zero-knowledge} if for any
$\REL \in \RELGEN(\secparam)$, and adversary $\adv$ there exists a $\ppt$ simulator
$\simulator$ with oracles $(\simOH,\simOP)$ such that for any $(\inp, \wit) \in \REL$, $\eps_0 \approx \eps_1$,
where
\[
  \eps_0(\secpar) = \condprob{\adv^{\simOH,\simOP} (\srs)}{\srs \gets \kgen
    (\REL)},\,  \eps_1 (\secpar) = \condprob{\adv^{\ro,\prover} (\srs)}{\srs \gets \kgen (\REL)},
\]

We call zero knowledge \emph{perfect} if the distributions are equal and
\emph{computational} if they are indistinguishable for any $\ppt$ distinguisher.

% \end{description}
Alternatively, zero-knowledge can be defined by allowing the simulator to use the
trapdoor $\td$ that is generated along the $\srs$. In this paper we distinguish
simulators that requires a trapdoor to simulate and those that do not. We call the
former \emph{SRS-simulators}. We say that a protocol is \emph{trapdoor-less
  zero-knowledge} (TLZK) if its simulator does not require the trapdoor, cf.~\cref{def:TLZK}.

  \markulf{22.04}{We don't consider SRS-simulators in this paper. I would simplify and move the above after the Updatable SRS scheme section, or maybe the start of Section 3.}
\fi

%In this paper we distinguish between simulators that require a trapdoor to simulate and those that do not.
We call a protocol \emph{trapdoor-less
 zero-knowledge} (TLZK) if there exists a simulator does not require the trapdoor, and works by programming the random oracle.
 Moreover, the simulator may only be allowed to program the random oracle on point $\tzkproof[0,k]$, that is the simulator can only program the challenges that come after the $k$-th prover message. We call protocols which allow for such a simulation $k$-\emph{programmable trapdoor-less zero-knowledge}. %, and define this in~\cref{def:TLZK}.

 Our definition of zero-knowledge for non-interactive
arguments is in programmable ROM.
%the explicitly programmable random oracle model where the simulator $\simulator$ can program the random oracle.
We model this using the oracles from Fig.~\ref{fig:real_simulator_oracles} that provide a stateful wrapper around $\simulator$.
$\simOH (x)$ simulates $\ro$ using lazy sampling, $\simulator\oracleo.\prog(x, h)$ allows for programming the simulated $\ro$ and is available only to $\simulator$. $\simOP(x, w)$ and $\simOP'(x)$ call the simulator. The former is used in the zero-knowledge definition and requires the statement and witness to be in the relation, the latter is used in the simulation extraction definition and does not require a witness input.

\begin{definition}[Updatable k-Programmable Trapdoor-Less Zero-Knowledge]
  \label{def:TLZK}
  Let
  $\psfs = (\SRScer, \prover, \verifier, \simulator)$ be a $(2\mu + 1)$-message FS-transformed NIZK proof system with an updatable SRS setup. %Let $\ro$ be the random oracle.
  %$\simulator_{\fs}$ takes as input $\srs$ and instance $\inp$, programs $\ro$, and outputs a proof $\zkproof_\simulator$.
  We call $\psfs$ \emph{trapdoor-less zero-knowledge} with security $\epszk$ if for any
  adversary $\adv$, $\abs{\eps_0(\secpar) - \eps_1(\secpar)} \leq \epszk(\secpar)$, where
  \begin{align*}
    \eps_0 (\secpar) = \Pr\left[ \adv^{\initU, \ro, \prover} (\secparam) \right],\,
    \eps_1 (\secpar)=  \Pr \left[\adv^{\initU, \simOH, \simOP} (\secparam) \right].
  \end{align*}

  If $\epszk(\secpar)$ is negligible, we say $\ps_{\fs}$ is trapdoor-less zero-knowledge. Additionally, we say that $\ps_{\fs}$ is $k$-programmable, if  $\simulator$ before returning a proof $\pi$ only calls $\simulator\oracleo.\prog$ on $(\tzkproof[0..k],h)$. That is, it only programs the $k$-th message.
  \end{definition}

\begin{remark}[TLZK vs HVZK]
  We note that TLZK notion is closely related to honest-verifier zero-knowledge in the
  standard model. That is, if we consider an interactive proof system $\proofsystem$
  that is HVZK in the standard model then $\proofsystem_\fs$ is TLZK. This comes as the simulator $\simulator$ in
  $\proofsystem$ produces a valid simulated proof by picking verifier's challenges
  according to a predefined distribution and $\proofsystem_\fs$'s simulator
  $\simulator_\fs$ produces its proofs similarly by picking the challenges and
  additionally programming the random oracle to return the picked
  challenges. Importantly, in both $\proofsystem$ and $\proofsystem_\fs$ success of
  the simulator does not depend on access to an SRS trapdoor.
\end{remark}

We note that $\plonk$ is $3$-programmable TLZK, and $\sonic$ and $\marlin$ are $2$-programmable TLZK. This follows directly from the proofs of
their standard model zero-knowledge property in
\cref{lem:plonk_tlzk,lem:sonic_hvzk,lem:marlin_hvzk}.

\subsection{Updatable Simulation Extractability (USE)}
We note that the zero-knowledge property is only guaranteed for statements in the
language.
%; for simulator $\simulator = (\simOH, \simOP)$, $\simOP$
%answers with simulated proofs only for true statements.  This is, however, not sufficient
For \emph{simulation extractability} where the simulator
should be able to provide simulated proofs for false statements as well, we thus use the oracle $\simOP'$
\footnote{Note,
  that simulation extractability property where the simulator is required to give
  simulated proofs for true statements only is called \emph{true simulation
    extractability.}}.

%Therefore, we introduce a wrapper oracle around the simulator
%called $\simOP'$ that on input $(\srs, \inp)$ always returns the first output of
%$\simulator (2, st, \srs, \inp)$, regardless of whether $\inp$ is in the language. We
%define \emph{simulation-extractability} with respect to oracle $\simOP'$; that is,
%simulation-extractability is with respect to a simulator
%$\simulator' = (\simOH, \simOP')$.
%

\begin{definition}[Updatable Simulation Extractability]
	\label{def:updsimext}
  \label{def:simext}
	Let $\psni = (\SRScer, \prover, \verifier, \simulator)$ be a NIZK proof system with an updatable SRS setup.
	We say that
  $\psni$ is \emph{updatable simulation-extractable} with security loss $\epsse(\secpar,\accProb, q)$ if for
  any $\ppt$ adversary $\adv$ that is given oracle access to setup oracle
  $\initU$ and simulation oracle $\simulator\oracleo$ and that produces an accepting
  proof for $\psni$ with probability $\accProb$, where
	\[
	\accProb = \condprob{
	\begin{matrix}
	  \verifier(\srs, \inp, \zkproof) = 1  \\
	  \wedge
	(\inp, \zkproof) \not\in Q
	\end{matrix}
}{
	\begin{matrix}
	  r \sample \RND{\advse}\\
	(\inp, \zkproof) \gets \advse^{\initU, \simOH, \simOP'
		} (1^\secpar; r)
	\end{matrix}
}
	\]
	there exists an expected PPT extractor $\extse$ such that
	\[
	 \condprob{
	\begin{matrix}
  \verifier(\srs, \inp, \zkproof) = 1, \\
   (\inp, \zkproof) \not\in Q,  \\
	   \REL(\inp, \wit) = 0
	\end{matrix}
}{
	\begin{aligned}
	& r \sample \RND{\advse},
	(\inp, \zkproof) \gets \advse^{\initU, \simOH, \simOP'
		} (1^\secpar; r) \\
	& \wit \gets \ext_\se (\srs, \advse, r,
	\Qsrs,\Qro,\Qsim )
	\end{aligned}
} \leq \epsse(\secpar,\accProb, q)
	\]
	%is at least
	%\[
	%\extProb \geq \frac{1}{\poly} (\accProb - \nu)^d - \eps(\secpar)\,,
	%\]
	%for some polynomial $\poly$, constant $d$ and negligible $\eps(\secpar)$ whenever
	%$\accProb \geq \nu$.
	Here, $\srs$ is the finalized SRS. List $\Qsrs$ contains all $(\srs, \rho)$ of update SRSs and their proofs, list $\Qro$ contains all $\advse$'s
	queries to $\simOH$ and the (simulated) random oracle's answers, $\abs{\Qro}\leq q$, and list $Q$ contains all $(\inp, \zkproof)$ pairs where
	$\inp$ is an instance queried to $\simOP'$ by the adversary and
	$\zkproof$ is the simulator's answer .
  % \hamid{17.10}{shouldn't be "List $Q_\ro$ contains all $\advse$'s queries to $\simOH$"? Also, I think we need to remove $\ro$ from the statement of the definition!}
\end{definition}

\subsection{Unique Response (UR) Protocols}
A technical hurdle identified by Faust et al.~\cite{INDOCRYPT:FKMV12} for proving
simulation extraction via the Fiat--Shamir transformation is that the transformed
proof system satisfies a unique response property. The original formulation by Fischlin, although suitable for applications presented in
\cite{C:Fischlin05,INDOCRYPT:FKMV12}, does not suffice in our case. First, the
property assumes that the protocol has three messages, with the second being the
challenge from the verifier. That is not the case we consider here. Second, it is not
entirely clear how to generalize the property. Should one require that after the
first challenge from the verifier, the prover's responses are fixed?  That does not
work since the prover needs to answer differently on different verifier's challenges,
as otherwise the protocol could have fewer messages.  Another problem is that the
protocol could have a message, beyond the first prover's message, which is
randomized. Unique response cannot hold in this case. Finally, the protocols we
consider here are not in the standard model, but use an SRS.

We work around these obstacles by providing a generalized notion of the unique
response property. More precisely, we say that a $(2\mu + 1)$-message protocol
has \emph{unique responses from $k$}, and call it a $\ur{k}$-protocol, if it
follows the definition below:

\begin{definition}[Updatable k-Unique Response Protocol]
Let $\psfs = (\SRScer, \prover, \verifier, \simulator)$ be a $(2\mu + 1)$-message FS-transformed NIZK proof system with an updatable SRS setup. Let $\ro$ be the random oracle.
We say that $\psfs$ has \emph{unique responses for $k$} with security $\epsur(\secpar)$ if for any $\ppt$ adversary $\advur$:
  \[
	\Pr\left[
		\left.
	\begin{aligned}
	& \zkproof \neq \zkproof', \tzkproof[0..k] = \tzkproof'[0..k],  \\
	& \verifier' (\srs, \inp, \zkproof,c) =
	\verifier' (\srs, \inp, \zkproof',c) = 1  \\
	\end{aligned}
	\,\right|\,
	\begin{aligned}
		& (\inp, \zkproof, \zkproof', c) \gets \advur^{\initU,\ro}(1^\secpar)
	\end{aligned}
	\right] \leq \epsur(\secpar)
	\]
	where $\srs$ is the finalized SRS and $\verifier'(\srs,\inp,\zkproof=(a_1, \dots, a_\mu,a_{\mu+1}))$ behaves as $\verifier (\srs, \inp, \zkproof)$ except for using c as the $k$-th challenge instead of calling $\ro(\tzkproof[0..k]) $. Thus, $\adv$ can program the $k$-th challenge.
	We say $\psfs$  is $\ur{k}$, if $\epsur(\secpar)$ is negligible.
\end{definition}

Intuitively, a protocol is $\ur{k}$ if it is infeasible for a $\ppt$ adversary to produce a pair of accepting proofs $\zkproof \neq \zkproof'$ that are the same on the first $k$ messages of the prover.
%We note that the definition above is also meaningful for protocols without an SRS. Intuitively in that case $\srs$ is the empty string.

The definition can be easily generalized to allow for programing the oracle on more than just a single point. We opted for this simplified presentation, since all the protocols analyzed in this paper require only single-point programming,

\subsection{Rewinding-Based Knowledge Soundness (RBKS)}

Before giving the definition of rewinding-based knowledge soundness for NIZK proof systems compiled via the $2\mu + 1$-message FS transformation, we first recall the notion of a tree of transcripts.
\begin{definition}[Tree of accepting transcripts, cf.~{\cite{EC:BCCGP16}}]
	\label{def:tree_of_accepting_transcripts}
%	Consider a $(2\mu + 1)$-message proof system. \markulf{24.04}{Is this meaningful for interactive or also non-interactive prove systems obtained via FS?}
	A $(n_1,
  \ldots, n_\mu)$-tree of accepting transcripts is a tree where each node on
  depth $i$, for $i \in \range{1}{\mu + 1}$, is an $i$-th prover's message in an
  accepting transcript; edges between the nodes are labeled with
  challenges, such that no two edges on the same depth have the same
  label; and each node on depth $i$ has $n_{i} - 1$ siblings and $n_{i +
    1}$ children. The tree consists of $N = \prod_{i = 1}^\mu n_i$
  branches, where $N$ is the number of accepting transcripts. We require $N = \poly$. We refer to a $(1, \ldots, n_k=n, 1, \ldots, 1)$-tree as a $(k,n)$-tree.
\end{definition}

\iffalse

	\begin{figure}[t]
		\centering
		\fbox{
		\procedure{$\genforking_{\zdv}^{m, m'} (y,h_1^{1}, \ldots, h_{q}^{1})$}
		{
		\rho \sample \RND{\zdv}\\
		(i, s_1) \gets \zdv(y, h_1^{1}, \ldots, h_{q}^{1}; \rho)\\
    i_1 \gets i\\
		% \pcif i = 0\ \pcreturn (0, \bot)\\
		\pcfor j \in \range{2}{m'}\\
		\pcind h_{1}^{j}, \ldots, h_{i - 1}^{j} \gets h_{1}^{j - 1}, \ldots,
		h_{i - 1}^{j - 1}\\
		\pcind h_{i}^{j}, \ldots, h_{q}^{j} \sample H\\
		\pcind (i_j, s_j) \gets \zdv(y, h_1^{j}, \ldots, h_{i - 1}^{j}, h_{i}^{j},
		\ldots, h_{q}^{j}; \rho)\\
		%\pcind \pcif i_j = 0 \lor i_j \neq i\ \pcreturn (0, \bot)\\
   % \pcif \exists (j, j') \in \range{1}{m}^2, j \neq j' : (h_{i}^{j} = h_{i}^{j'})\
    \pcif \exists (j_1, \ldots, j_m) \in \range{1}{m'}^m, \text{ s.t. }\\
    \pcind j_k \neq j_{k'} \text{ for } k \neq k' \land \\
    \pcind i_{j_k} = i_{j_k'} \land \\
    \pcind
		\pcelse \pcreturn (0, \bot)
	}}
\caption{Generalized forking algorithm $\genforking_{\zdv}^{m, m'}$
  \michals{5.11}{This forking lemma version is not as general as in Attema et al to
    be exact -- they allow tree of acceptable transcripts to branch at multiple
    places. however, that is not necessary in our case. Later, we should allow ``full
    generality'' but that would also require modification of the forking soundness
    definition (which now also relies on a fact that the tree branches at a single
    point)}}
	\label{fig:genforking_lemma}
\end{figure}

\fi

\iffalse
Note that the special soundness property (as usually defined) holds for
all---even computationally unbounded---adversaries. Unfortunately, since a
simulation trapdoor for $\plonkprot$, $\sonicprot$ and $\marlinprot$ exists, the protocols
cannot be special sound in that regard. This is because an unbounded adversary
can recover the trapdoor and build an arbitrary number of simulated proofs for a fake
statement. Hence, we provide a weaker, yet sufficient, definition that somehow lies between
computational \emph{knowledge soundness} and \emph{special soundness} for unbounded adversaries. Differently from the standard definition of special soundness, we do not require from the extractor to be able to extract the witness from \emph{any} tree of acceptable transcripts. Similar to knowledge soundness, we require that the extractor fails to reconstruct the witness with some small probability only.

More precisely, in proofs of rewinding-based knowledge soundness we show later that either the extractor extracts a witness from a tree of transcripts or the adversary who was used to produce the tree can be used to break some computationally hard problem.
\fi
The existence of simulation trapdoor for $\plonkprot$, $\sonicprot$ and $\marlinprot$ means that they are not
special sound in the standard sense. We therefore put forth the notion of rewinding-based knowledge soundness that is a computational notion.
Note that in the definition below, it is implicit that each transcript in the tree is accepting with respect to a ``local programming'' of the random oracle. However, the verification of the proof output by the adversary is with respect to a non-programmed random oracle.

\begin{definition}[Updatable Rewinding-Based Knowledge Soundness]
	Let $n_1, \ldots, n_\mu \in \NN$.
	%
	Let $\psfs = (\SRScer, \prover, \verifier, \simulator)$ be a $(2\mu + 1)$-message FS-transformed NIZK proof system with an updatable SRS setup for relation $\REL$. Let $\ro$ be the random oracle.
	%
	We require existence of an expected PPT tree builder $\tdv$ that eventually outputs a $\tree$ which is either a $(n_1, \ldots, n_\mu)$-tree of accepting transcript or $\bot$ and a PPT extractor $\extcss$. Let  adversary $\advcss$ be a PPT algorithm, that outputs a valid proof with probability at least $\accProb$,
	where
	\[
	\accProb = \condprob{
	\begin{matrix}
	  \verifier(\srs, \inp, \zkproof) = 1  \\
	  \wedge
	(\inp, \zkproof) \not\in Q
	\end{matrix}
}{
	\begin{matrix}
	  r \sample \RND{\advcss}\\
	(\inp, \zkproof) \gets \advcss^{\initU, \ro
		} (1^\secpar; r)
	\end{matrix}
}.
	\]
	We say that $\psfs$ is $(n_1, \ldots, n_\mu)$-rewinding-based knowledge sound with security loss $\epscss(\secpar, \accProb, q)$ if
	\begin{align*}
	\Pr\left[
		\begin{matrix}
			\verifier(\srs, \inp, \zkproof) = 1,  \\
			\REL(\inp, \wit) = 0
		  \end{matrix}
	\,\left|\,
	\begin{aligned}
	& 	r \sample \RND{\advcss}, \\
	& 	(\srs, \inp, \cdot) \gets \advcss^{\initU, \ro} (1^\secpar; r)\\
	&  	\tree \gets \tdv(\srs, \advcss, r, \Qsrs, \Qro),
	\wit \gets \extcss(\tree)
	\end{aligned}\right.
	\right] \leq \epscss(\secpar,\accProb, q).
	\end{align*}
	Here, $\srs$ is the finalized SRS. List $\Qsrs$ contains all $(\srs, \rho)$ of updated SRSs and their proofs, and list $\Qro$ contains all of the adversaries
	queries to $\ro$ and the random oracle's answers, $\abs{\Qro}\leq q$.
%	 We call the protocol $(n_1, \ldots, n_\mu)$ rewinding-based knowledge sound when $\tdv$ builds a $(n_1, \ldots, n_\mu)$-tree of transcripts.
% \hamid{1.5}{We still need this sentence:
%\markulf{1.5}{Cut? We call the protocol $(k, n)$-rewinding-based knowledge sound if $\tree$ is a $(k, n)$-tree of accepting transcripts.}
% }
\end{definition}

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "main"
%%% End:


\subsubsection{Model Generalization}
\label{sec:randerrC}

Although a key takeaway from Section \ref{sec:randerr} is that the \gls{MLL}
calculations are the most resilient to introduced error in the training set
features for all four prediction categories, there is another aspect of the
algorithm performance not explicitly shown in those plots: generalization.
\Gls{MLL} does not generalize to unseen data; it provides predictions based on
finding the closest training set entry to the test sample.  It (usually)
outperforms the scikit-learn methods in part because the training set is so
large. This is also true for the \textit{k}-nearest neighbor implementations
where $k=1$ (burnup and cooling time, as seen in Table \ref{tbl:exp1hypparam}).

\begin{figure}[!htb]
  \centering
  \begin{subfigure}[b]{0.495\textwidth}
    \centering
    \includegraphics[width=\textwidth]{./chapters/exp1/learncurve_nuc29_err05_BalAcc_rxtr.png}
    \caption{Balanced accuracy of reactor type classification with respect
             to training set size.}
    \label{fig:learnsA}
  \end{subfigure}
  \hfill
  \begin{subfigure}[b]{0.485\textwidth}
    \centering
    \includegraphics[width=\textwidth]{./chapters/exp1/learncurve_nuc29_err05_MAPE_burn.png}
    \caption{Negative \gls{MAPE} of burnup regression with respect to
             to training set size.}
    \label{fig:learnsB}
  \end{subfigure}
  \vskip\baselineskip
  \begin{subfigure}[b]{0.48\textwidth}
    \centering
    \includegraphics[width=\textwidth]{./chapters/exp1/learncurve_nuc29_err05_MAPE_enri.png}
    \caption{Negative \gls{MAPE} of \gls{U235} enrichment regression with
             respect to training set size.}
    \label{fig:learnsC}
  \end{subfigure}
  \hfill
  \begin{subfigure}[b]{0.50\textwidth}
    \centering
    \includegraphics[width=\textwidth]{./chapters/exp1/learncurve_nuc29_err05_MAPE_cool.png}
    \caption{Negative \gls{MAPE} of time since irradiation regression with
             respect to training set size.}
    \label{fig:learnsD}
  \end{subfigure}
  \caption[Learning curves for all four prediction parameters]
          {Learning curves for reactor type, burnup, enrichment, and time
           since irradiation with respect to increasing fraction of the
           training set, for 5\% training set random error.}
  \label{fig:learns}
\end{figure}

One way to show that an algorithm is generalizing well in comparison to others
is to view the shape of its learning curve (introduced in Section
\ref{sec:complexity}): the prediction performance with respect to training set
size.  It is crucial to have the training sets be identical for each algorithm,
so they were created in advance and the learning curves are constructed
manually rather than relying on the scikit-learn method.  Smaller training sets
were created from the original one by taking 80\%, 60\%, 40\%, and 20\% of the
entries. The training sets are all stratified so that the original fractions of
\gls{BWR}, \gls{PWR}, and \gls{PHWR} are retained. They are also built on top
of one another, so the 20\%-size training set is contained within the 40\%-size
set, and so forth.

Learning curves were constructed for all four prediction categories,
demonstrated in Figure \ref{fig:learns}. As in Figures
\ref{fig:randrxtr}--\ref{fig:randcool} , the vertical axis is always oriented
so that lower is poorer performance and higher is better performance; also, the
error bars reflect a 99\% confidence interval for Figure \ref{fig:learnsA}, and
one standard deviation of the average percentage errors for Figures
\ref{fig:learnsB}--\ref{fig:learnsD}.  These learning curves represent the 5\%
random error case in Figures \ref{fig:randrxtr}--\ref{fig:randcool}, so the
scores/errors in these figures are the data points at the 100\% training set
level in Figure \ref{fig:learns}.  Therefore, the leftmost data in Figure
\ref{fig:learns} will show the \gls{MLL} point being slightly above the
scikit-learn points for the reactor type, burnup, and enrichment predictions,
and the \textit{k}-nearest neighbor point is below the \gls{MLL} and decision
trees points for time since irradiation.

Figure \ref{fig:learnsA} shows that the balanced accuracy score of reactor type
classification for the \gls{MLL} calculations decreases more at lower training
set size than for the scikit-learn algorithms. Here, the curve crosses below
the \textit{k}-nearest neighbors curve at the lowest training set size of 20\%.
For the burnup \gls{MAPE} in Figure \ref{fig:learnsB} and the enrichment
\gls{MAPE} in Figure \ref{fig:learnsC}, the \gls{MLL} curve crosses below both
of the scikit-learn algorithm curves. This happens between 20\% and 40\%
training set size for burnup, and between 40\% and 60\% training set size for
enrichment.  Lastly, Figure \ref{fig:learnsD} shows a different arrangement,
which is to be expected from the results shown in Figure \ref{fig:randcool},
where the \textit{k}-nearest neighbors performance is significantly worse than
the other two algorithms. Because the \textit{k}-nearest neighbors curve and
error bars are so large, there is an inset showing a closeup of the other two
curves above -6\%.  The decision trees and \gls{MLL} calculations curves now
appear to follow the trend in the burnup and enrichment cases, and the
\gls{MLL} curve crosses under the decision trees curve between 20\% and 40\%
training set size.

There is a dependence on training set size for all three algorithms in Figure
\ref{fig:learns}. For the most part, the \textit{k}-nearest neighbors and
decision trees curves follow an approximately parallel path, whereas the
\gls{MLL} method shows an increased rate of degradation at low training set
sizes. Since this training set is large enough, i.e., the prediction parameters
were included in small enough steps, that \gls{MLL} has consistent performance
at the larger sizes, there is not a concern in this work about its inability to
generalize. It must be noted, however, that the \gls{MLL} approach requires a
fine grid of simulation parameters in a training database to perform better
than the simple scikit-learn algorithms.

\subsubsection{Reactor Type Prior Knowledge}
\label{sec:randerrD}

There is similar work being done to this work that focuses on similar
prediction categories but in a serial manner, i.e., first determining the
reactor type before moving forward with other predictions \cite{serial_ml}.
This work predicts reactor operation parameters while blind to the reactor
type, but it makes sense intuitively that having previous knowledge of the
reactor type would allow more accurate regression of these parameters.
Therefore, the change in regression performance from the reactor type-blind
predictions to having prior knowledge of the reactor type is discussed.

The workflow was repeated for the three regression cases where they were
trained on reactor type-specific training sets. A 5\% random error was applied
to these training sets, and the 5\% random error full training set was used as
comparison. The errors for each algorithm (\textit{k}-nearest neighbors,
decision trees, and \gls{MLL} calculations) were tallied for each regression
category (burnup, enrichment, and time since irradiation) and within that, for
each reactor type (\gls{PWR}, \gls{BWR}, \gls{PHWR}). Two sets of error were
tracked: whether the reactor type was \textit{known} or \textit{unknown} prior
to prediction.

Table \ref{tbl:knownrxtr} shows the \gls{MAPE}s for each regression category
and within that, for each reactor type (\gls{PWR}, \gls{BWR}, \gls{PHWR}).  The
columns are separated first by the algorithms and second by whether the reactor
type was known or unknown prior to prediction, denoted as \textit{K} and
\textit{U}, respectively. Most of these relative errors are quite low, and
around or under 2\%.  So, e.g., despite burnup prediction from \glspl{PHWR}
improving, it was by 0.61\%, a precision of which may not be of concern. Still,
these performance differences can be looked at in more detail.

\begin{table}[!htb]
  \centering
  \begin{tabular}{@{}llllllll@{}}
    \toprule
    &  &  \multicolumn{2}{c}{\textit{k}NN}
    &     \multicolumn{2}{c}{Dec Trees}
    &     \multicolumn{2}{c}{MLL Calcs} \\
    \toprule
    \begin{tabular}[c]{@{}l@{}}Prediction\\ Parameter\end{tabular} &
    \begin{tabular}[c]{@{}l@{}}Reactor \\ Type\end{tabular} &
    K & U &  K & U & K & U \\ \midrule
    \multirow{3}{*}{\begin{tabular}[c]{@{}l@{}}Burnup \\ \% {$[MWd/MTU]$} \end{tabular}}
     & PWR  & 0.60  & 0.66  & 0.54 & 0.75 & 0.24 & 0.25 \\
     & BWR  & 0.88  & 0.90  & 0.60 & 0.66 & 0.40 & 0.40 \\
     & PHWR & 0.66  & 1.27  & 0.14 & 0.54 & 0.28 & 0.28 \\ \hline
    \multirow{3}{*}{\begin{tabular}[c]{@{}l@{}}Enrichment \\ \% {$[\%\:{}^{235}\text{U}]$} \end{tabular}}
     & PWR  & 0.85  & 0.99  & 0.36 & 0.48 & 0.26 & 0.29 \\
     & BWR  & 1.14  & 1.16  & 0.51 & 0.54 & 0.45 & 0.46 \\
     & PHWR & 0.00  & 0.00  & 0.00 & 0.02 & 0.00 & 0.00 \\ \hline
    \multirow{3}{*}{\begin{tabular}[c]{@{}l@{}}Time Since \\ Irradiation \\ \% {$[days]$} \end{tabular}}
     & PWR  & 11.44 & 10.48 & 2.35 & 2.19 & 1.55 & 1.46 \\
     & BWR  & 15.39 & 15.48 & 2.27 & 2.28 & 2.06 & 2.05 \\
     & PHWR & 19.32 & 34.41 & 4.96 & 4.52 & 2.30 & 2.30 \\ \bottomrule
  \end{tabular}
  \caption[\acrshort{MAPE}s for three regression cases comparing known versus
           unknown reactor type prior knowledge]
          {\acrshort{MAPE}s for the three prediction cases for each algorithm
           at 5\% training set error. \textit{K} refers to \textit{known}
           reactor type and \textit{U} refers to \textit{unknown} reactor type
           prior to regression.}
  \label{tbl:knownrxtr}
\end{table}

To better see these performance differences, the percent change in prediction
\gls{MAE} for each algorithm and reactor type between the reactor type being
known versus unknown prior to prediction was calculated as: $100 \cdot
\frac{MAE_{unknown} - MAE_{known}}{MAE_{unknown}}$.  This was chosen to be
relative to the unknown error since that is the benchmark in this case.  Figure
\ref{fig:knownrxtr} is three heatmaps that show this percent change for each
prediction category, algorithm, and reactor type.  This value is reflected by a
diverging color bar as well as a positive or negative percentage in each
square.  The positive percentages indicate the error decreased/improved from
the unknown reactor type case to the known reactor type case.  The negative
percentages indicate the error increased/worsened from the unknown to the known
case.

\begin{figure}[!htb]
  \centering
  \includegraphics[width=\textwidth]{./chapters/exp1/rxtr-type_known-unknown_diff_err05.png}
  \caption[Heatmaps for three regression cases comparing known versus
           unknown reactor type prior knowledge]
          {Heatmaps for the three regression cases showing the percent
           difference in prediction error between a known reactor type
           and unknown reactor type using a 5\% training set error.}
  \label{fig:knownrxtr}
\end{figure}

For burnup prediction, most differences are within $\pm10\%$ except for three
scenarios.  The decision tree algorithm has improved burnup prediction for
\glspl{PWR} by 22.0\% and for \glspl{PHWR} by 71.1\% given a known reactor
type.  The \textit{k}-nearest neighbors algorithm has 49.5\% improved burnup
prediction for the \gls{PHWR}. For \gls{U235} enrichment, the \gls{PWR}
predictions improve by approximately 18\% for the scikit-learn algorithms.
Even though the value is within $\pm10\%$, this is the only scenario where
there is an appreciable difference in \gls{MLL} performance. The decision tree
enrichment prediction of \glspl{PHWR} also has a sizeable improvement of
100.0\%.  The time since irradiation predictions for the most part do not show
improvement outside of $\pm10\%$. Of note is some volatile behavior for the
\gls{PHWR} case with the scikit-learn algorithms.  While \textit{k}-nearest
neighbors improves by 29.8\%, the decision tree predictions were worse by
12.5\%.  Since the main concern here is showing how prediction performance
improves with prior reactor type knowledge, this reduction in performance is
odd but not worthy of further investigation.

The improvements in the \gls{PHWR} predictions are not surprising since the
generalization of the scikit-learn algorithms could lead to the unique
\gls{PHWR} cases being ignored, since they are after all only 1.5\% of the
training set.  Another interesting result is that the \gls{BWR} predictions
experience no large changes, which makes sense given that they comprise 72\% of
the training set. Also, the \gls{MLL} predictions are approximately the same,
which is expected because this algorithm does not generalize, and the
prediction comes as a set of labels and is therefore already linked to the
reactor type. Overall, it is important to be aware that the regression labels
coming from a \gls{PHWR} will be unlikely to be optimal results (except for
those from \gls{MLL} calculations).

%\begin{table}[!htb]
%  \centering
%  \begin{tabular}{@{}llllllll@{}}
%    \toprule
%    &  &  \multicolumn{2}{c}{\textit{k}NN}
%    &     \multicolumn{2}{c}{Dec Trees}
%    &     \multicolumn{2}{c}{MLL Calcs} \\
%    \toprule
%    \begin{tabular}[c]{@{}l@{}}Prediction\\ Parameter\end{tabular} &
%    \begin{tabular}[c]{@{}l@{}}Reactor \\ Type\end{tabular} &
%    K & U &  K & U & K & U \\ \midrule
%    \multirow{3}{*}{\begin{tabular}[c]{@{}l@{}}Burnup \\ \% {$[MWd/MTU]$} \end{tabular}}
%     & PWR  & 0.08  & 0.08  & 0.03 & 0.04 & 0.24 & 0.25 \\
%     & BWR  & 0.11  & 0.10  & 0.03 & 0.03 & 0.40 & 0.40 \\
%     & PHWR & 0.13  & 0.19  & 0.01 & 0.03 & 0.29 & 0.29 \\ \hline
%    \multirow{3}{*}{\begin{tabular}[c]{@{}l@{}}Enrichment \\ \% {$[\%\:{}^{235}\text{U}]$} \end{tabular}}
%     & PWR  & 0.10  & 0.11  & 0.02 & 0.02 & 0.26 & 0.29 \\
%     & BWR  & 0.12  & 0.12  & 0.02 & 0.02 & 0.46 & 0.46 \\
%     & PHWR & 0.00  & 0.00  & 0.00 & 0.00 & 0.00 & 0.00 \\ \hline
%    \multirow{3}{*}{\begin{tabular}[c]{@{}l@{}}Time Since \\ Irradiation \\ \% {$[days]$} \end{tabular}}
%     & PWR  & 6.47  & 6.13  & 1.50 & 1.43 & 1.55 & 1.46 \\
%     & BWR  & 9.18  & 9.15  & 1.56 & 1.53 & 2.07 & 2.05 \\
%     & PHWR & 13.78 & 18.62 & 4.51 & 3.71 & 2.30 & 2.30 \\ \bottomrule
%  \end{tabular}
%  \caption{\gls{MAPE}s for the three prediction cases for each algorithm at 1\%
%           training set error. \textit{K} refers to \textit{known} reactor
%           type and \textit{U} refers to \textit{unknown} reactor type prior
%           to regression.}
%  \label{tbl:knownrxtr}
%\end{table}

%\begin{figure}[!htb]
%  \centering
%  \includegraphics[width=\textwidth]{./chapters/exp1/rxtr-type_known-unknown_diff_err01.png}
%  \caption{Heatmaps for the three regression cases showing the percent
%           difference in prediction error between a known reactor type
%           and unknown reactor type.}
%  \label{fig:knownrxtr}
%\end{figure}