%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Lachaise Assignment
% LaTeX Template
% Version 1.0 (26/6/2018)
%
% This template originates from:
% http://www.LaTeXTemplates.com
%
% Authors:
% Marion Lachaise & François Févotte
% Vel (vel@LaTeXTemplates.com)
%
% License:
% CC BY-NC-SA 3.0 (http://creativecommons.org/licenses/by-nc-sa/3.0/)
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%----------------------------------------------------------------------------------------
%	PACKAGES AND OTHER DOCUMENT CONFIGURATIONS
%----------------------------------------------------------------------------------------

\documentclass{article}

\input{structure.tex} % Include the file specifying the document structure and custom commands

%----------------------------------------------------------------------------------------
%	ASSIGNMENT INFORMATION
%----------------------------------------------------------------------------------------

\title{Solutions} % Title of the assignment

\author{Mohammad Mahzoun} % Author name and email address

%\date{University of Inaba --- \today} % University, school and/or department name(s) and a date

%----------------------------------------------------------------------------------------

\begin{document}

\maketitle % Print the title

%----------------------------------------------------------------------------------------
%	INTRODUCTION
%----------------------------------------------------------------------------------------

\section{Exercise 1}
\textbf{Authenticity}: The message is signed by $A$ so $B$ can verify that the message she received is indeed sent by $A$ because in order to sign a message as $A$, adversary needs to forge the signature scheme.   \\
\textbf{Confidentiality}: Since the messages are authenticated, in order to know the $m$, one has to find $ek$ which is equivalent to breaking the security of $penc$. $mac$ can be used to check if the message is not changed in the network.

\section{Exercise 2}
let $GEN()   = randomBytes \;16$ \\ \\
let $A$ $(pk_B: pub\_key)$ $(sk_A: sig\_key)$ $(m: bytes)$ : $(C: bytes) =$ \\
let private $ek = GEN()$ in \\
let private $mk = GEN()$ in \\
let $C_1$ = $penc$ $pk_B$ $(concat\;ek\;mk)$ in \\
let $C_2$ = $sign$ $sk_A$ $C_1$ in \\
let $C_3$ = $sym\_enc$ $ek$ $m$ in \\
let $C_4$ = $mac$ $mk$ $C_3$ in \\
$concat\; C_2 \; C_4$
\\
\\
let $B$ $(pk_A: verif\_key)$ $(sk_B: priv\_key)$ $(C: bytes)$ : $(m: bytes) =$ \\
let $C_1\; C_2 = split C$ in \\
if $verif\; pk_A\; C_1 \; = 1 \;$ then \\
let $M_1 = pke\_dec \; sk_B \; C_1$ in \\
let $ek\; mk = split \; M_1$ in \\
let $Cipher\; MAC = \; split \; C_2$ in \\
let $m = sym\_dec \; ek C$ in \\
if $mac\; Cipher \; mk$ = $MAC$ then $m$

\section{Exercise 3}

\end{document}


\documentclass[modern]{aastex63}

\usepackage{acro}
\usepackage{amsmath}
\usepackage{amssymb}

\newcommand{\dd}{\mathrm{d}}
\newcommand{\diff}[2]{\frac{\dd #1}{\dd #2}}

\DeclareAcronym{BBH}{
  short = {BBH},
  long = {binary black hole}
}
\DeclareAcronym{BNS}{
  short = {BNS},
  long = {binary neutron star}
}
\DeclareAcronym{EM}{
  short = {EM},
  long = {electromagnetic}
}
\DeclareAcronym{GW}{
  short = {GW},
  long = {gravitational wave}
}
\DeclareAcronym{IFO}{
  short = {IFO},
  long = {interferometer}
}
\DeclareAcronym{SNR}{
  short = {SNR},
  long = {signal to noise ratio}
}

\begin{document}

\title{Strategies for Tuning Downtime and Duty Cycle to Optimize Gravitational
Wave Detections}
\author[0000-0003-1540-8562]{Will M. Farr}
\email{will.farr@stonybrook.edu}
\affiliation{Department of Physics and Astronomy, Stony Brook University, Stony Brook NY 11794, United States}
\affiliation{Center for Computational Astronomy, Flatiron Institute, 162 5th Ave, New York NY 10010, United States}

\begin{abstract} I give some order-of-magnitude arguments about how to optimize
the duty cycle of multiple, heterogeneous GW detectors, to maximize the number
of detections.  I consider networks that are representative of likely
configurations in the near future and mid-2020s. \end{abstract}

\section{Preliminaries}

It is almost always the right choice to optimize the number of detections,
$N_\mathrm{det}$ which scales as
%
\begin{equation}
  N_\mathrm{det} \propto V T,
\end{equation}
%
where $V$ is the sensitive volume for a particular source type, and $T$ is the
overall uptime.  In a Euclidean universe (appropriate for \ac{BNS} events, and
even a good approximation for \ac{BBH} events up until the 3G era),
%
\begin{equation}
  V \propto \rho^3
\end{equation}
%
where $\rho$ is the \ac{SNR} of a representative source at a fixed distance. Due
to the cubic scaling with \ac{SNR}, for a single detector, it is almost always
the right decision to trade a small amount of downtime for a sensitivity
improvement that boosts the \ac{SNR}.

For multiple detectors, however, the issue of scheduling of downtimes has an
effect.  In a multi-detector network, \acp{SNR} add in quadrature
%
\begin{equation}
  \rho = \sqrt{\sum_{i=1}^{N_\mathrm{IFO}} \rho_i^2},
\end{equation}
%
and the overall observation time depends on the scheduling strategy.

Here we will assume that the sensitive volume is dominated by the \ac{GW}
detection horizon, not by a simultaneous \ac{EM} horizon; in this latter case,
optimizing the uptime is the only consideration.

\section{Additional Less-Sensitive Detector}

Consider the scheduling problem for the addition of one more detector to an
existing network of $N$ \acp{IFO}, where the additional detector is a factor $f
\ll 1$ less sensitive than the typical detector in the network and operates with
a duty cycle $T_\mathrm{obs} = (1-\tau) T_\mathrm{wall}$ (e.g.\ the current VIGO
detector has $f \simeq 1/3$).  If we schedule the detector downtime
synchronously with the remainder of the network (here assumed to have comparable
duty cycle), then the total $VT$ is
%
\begin{equation}
  \label{eq:low-sens-synch}
  VT \propto (1-\tau) \left( 1 + f^2 \right)^{3/2} \simeq (1-\tau) \left( 1 + \frac{3}{2} f^2 \right).
\end{equation}
%
On the other hand, if the downtime of this additional detector is scheduled
randomly, then we have
%
\begin{equation}
  VT \propto (1-\tau)^N \tau + N (1-\tau)^{N} \tau \left( 1 - \frac{1}{N} + f^2 \right)^{3/2} + (1-\tau)^{N+1} \left( 1 + f^2 \right)^{3/2} + \mathcal{O}\left(\tau^2\right).
\end{equation}
%
(The three cases correspond to the $N$ original \acp{IFO} operating, new
\ac{IFO} down; one of the $N$ original \acp{IFO} down, new \ac{IFO} operating;
and all \acp{IFO} operating.)  This is approximately
%
\begin{equation}
  \label{eq:low-sens-random}
  VT \propto \left( 1 - \left[ \frac{3}{2} + \frac{3}{8N} \right] \tau \right) \left( 1 + \frac{3}{2} f^2\right) + \mathcal{O}\left( \tau f^2 \right)
\end{equation}
%
Comparing Eq.\ \eqref{eq:low-sens-synch} and Eq.\ \eqref{eq:low-sens-random}, we
see that the synchronous strategy dominates the random-downtime strategy in all
cases.

\section{Additional Comparable-Sensitivity Detector}

Suppose we are considering whether to add an additional detector to the network
that is comparable sensitivity to the existing network of $N$.  Then in the
synchronous case we have
%
\begin{equation}
  VT \propto (1-\tau) \left(1 + \frac{1}{N}\right)^{3/2} \simeq (1-\tau) \left( 1 + \frac{3}{2N} \right).
\end{equation}
%
In the random scheduling case we have
%
\begin{multline}
  VT \propto (1-\tau)^N \tau + N (1-\tau)^N \tau + (1-\tau)^{N+1} \left(1 + \frac{1}{N}\right)^{3/2} \\ \simeq \left( 1 - \frac{3}{2} \tau \right) \left( 1 + \frac{3}{2N} \right) + \mathcal{O}\left( \frac{\tau}{N} \right)
\end{multline}
%
Once again, we see that the random-scheduling case is dominated by synchronous
scheduling case.

\section{Conclusion}

If possible, detector downtimes should be scheduled synchronously.

\bibliography{vtoptimize}

\end{document}


    \filetitle{resample}{Resample from the model implied distribution}{model/resample}

	\paragraph{Syntax}\label{syntax}

\begin{verbatim}
Outp = resample(M,Inp,Range,NDraw,...)
Oupt = resample(M,Inp,Range,NDraw,J,...)
\end{verbatim}

\paragraph{Input arguments}\label{input-arguments}

\begin{itemize}
\item
  \texttt{M} {[} model {]} - Solved model object.
\item
  \texttt{Inp} {[} struct \textbar{} empty {]} - Input data (if needed)
  for the distributions of initial condition and/or empirical shocks; if
  not bootstrap is invovled
\item
  \texttt{Range} {[} numeric {]} - Resampling date range.
\item
  \texttt{NDraw} {[} numeric {]} - Number of draws.
\item
  \texttt{J} {[} struct \textbar{} empty {]} - Database with
  user-supplied (time-varying) tunes on std devs, corr coeffs, and/or
  means of shocks.
\end{itemize}

\paragraph{Output arguments}\label{output-arguments}

\begin{itemize}
\itemsep1pt\parskip0pt\parsep0pt
\item
  \texttt{Outp} {[} struct {]} - Output database with resampled data.
\end{itemize}

\paragraph{Options}\label{options}

\begin{itemize}
\item
  \texttt{'deviation='} {[} \texttt{true} \textbar{}
  \emph{\texttt{false}} {]} - Treat input and output data as deviations
  from balanced-growth path.
\item
  \texttt{'dtrends='} {[} \emph{\texttt{@auto}} \textbar{} \texttt{true}
  \textbar{} \texttt{false} {]} - Add deterministic trends to
  measurement variables.
\item
  \texttt{'method='} {[} \texttt{'bootstrap'} \textbar{}
  \emph{\texttt{'montecarlo'}} {]} - Method of randomising shocks and
  initial condition.
\item
  \texttt{'progress='} {[} \texttt{true} \textbar{}
  \emph{\texttt{false}} {]} - Display progress bar in the command
  window.
\item
  \texttt{'randomInitCond='} {[} \emph{\texttt{true}} \textbar{}
  \texttt{false} \textbar{} numeric {]} - Randomise initial condition; a
  number means the initial condition will be simulated using the
  specified number of extra pre-sample periods.
\item
  \texttt{'stateVector='} {[} \emph{\texttt{'alpha'}} \textbar{}
  \texttt{'x'} {]} - When resampling initial condition, use the
  transformed state vector, \texttt{alpha}, or the vector of original
  variables, \texttt{x}; this option is meant to guarantee replicability
  of results.
\item
  \texttt{'svdOnly='} {[} \texttt{true} \textbar{} \emph{\texttt{false}}
  {]} - Do not attempt Cholesky and only use SVD to factorize the
  covariance matrix when resampling initial condition; only applies when
  \texttt{'randomInitCond=' true}.
\item
  \texttt{'wild='} {[} \texttt{true} \textbar{} \emph{\texttt{false}}
  {]} - Use wild bootstrap instead of Efron bootstrap; only applies when
  \texttt{'method=' 'bootstrap'}.
\end{itemize}

\paragraph{Description}\label{description}

When you use wild bootstrap for resampling the initial condition, the
results are based on an assumption that the mean of the initial
condition is the asymptotic mean implied by the model (i.e.~the steady
state).

\paragraph{Example}\label{example}


% --- [ Recovery of Post-test Loops ] ------------------------------------------

\subsection{Recovery of Post-test Loops}
\label{sec:recovery_of_post_test_loops}

The control flow recovery results of the Hammock method, the Interval method, and for comparison the theoretical optimum when recovering \textit{post-test loops} (e.g. \texttt{do-while}-loops) from the combined test programs of Coreutils and SQLite are presented in figure \ref{fig:total_results_post_loop}.

The \textbf{Hammock method} correctly recovered $4.65\%$ of the post-test loops present in the test programs (\textit{true positives}). On average, for every $10$ post-test loops of the original source code, the Hammock method recovered $4.1$ post-test loops that were \textit{not} present of the original source code (\textit{false positives}).

The \textbf{Interval method} correctly recovered $62.77\%$ of the post-test loops present in the test programs (\textit{true positives}). On average, for every $10$ post-test loops of the original source code, the Interval method recovered $18.7$ post-test loops that were \textit{not} present of the original source code (\textit{false positives}).

\begin{figure}[htbp]
	\centering
	\includegraphics[width=\textwidth]{inc/5_results/results_post_loop.png}
	\caption{Comparison of control flow recovery results for each method when recovering \textit{post-test loops}. The data is based on the combined test programs of Coreutils and SQLite.}
	\label{fig:total_results_post_loop}
\end{figure}


\section{Concluding Remarks}
\label{sec:concluding-remarks}

In this dissertation, a new group of hybrid methods called the $\Omega$-methods
were
proposed. The $\Omega$-methods are built on the foundational work of CADIS
and FW-CADIS to generate angle-informed variance reduction parameters. The two
new methods proposed were CADIS-$\Omega$ and FW-CADIS-$\Omega$. Both methods use
the $\Omega$-flux, a form of the adjoint scalar flux calculated by weighting the
adjoint angular flux with the forward angular flux, to generate source biasing and weight window
values. By using the forward angular flux
normalization, the importance map generated for the
$\Omega$-methods is adjusted to include the directionality of
the forward and the adjoint particles without explicitly including angle
in the source biasing or weight window values.

The $\Omega$-methods were implemented in two software packages developed at Oak
Ridge National Laboratory: Exnihilo and ADVANTG. The functionality to generate
the $\Omega$-fluxes were implemented in Exnihilo, which contains the
deterministic transport solver Denovo. The infrastructure to generate variance
reduction parameters consistent with CADIS and FW-CADIS was implemented in
ADVANTG. The development of these methods now allows for any user to use the
$\Omega$-methods, should they have access to the software.

In addition to the $\Omega$-methods method proposal and
implementation, CADIS-$\Omega$ has been
characterized on a wide variety of problems with flux anisotropies. The problems
were designed to understand the method's limitations and in what parameter space
the method can and should be used. To more fully understand the method's' behavior and how
flux anisotropy affected its ability to perform, a number of anisotropy metrics
were proposed. These metrics were then used
to investigate if performance improvement could be correlated with
anisotropy in any way.

The anisotropy metrics did not show significant trends with the FOM or the
solution relative error, but their distributions did help reveal more about the
distribution of anisotropy in the problems. In particular, it was easily
observable how the distribution of anisotropy changed between energy groups for
a particular problem. Future use of these metrics may also
aid us in more fully understanding other hybrid methods' performance.

CADIS-$\Omega$ is a promising hybrid method. If used with a well-suited problem,
it has the potential to improve the FOM over traditional methods by an order of
magnitude. This offers significant time and energy savings. However, the
$\Omega$-methods are not without their drawbacks. If used in a poorly-suited
problem they can take substantially more time to transport particles in
Monte Carlo. The $\Omega$-methods' characterization and performance study
presented in this dissertation have contributed a broader understanding of these types of
hybrid methods, and have created ample pathways forward for future hybrid
methods analysis.


\lab{Python}{Profiling Python Code}{Profiling}
\objective{Learn how to identify and remove bottlenecks in code.}
\label{lab:ProfilingCode}

You have undoubtedly written code that had less than stellar performance.
It would be really useful if we could see what parts of the code spent the most time executing.
We will introduce you to a number of ways to identify bottlenecks in your code.
A bottleneck is where the execution of a small part of the code slows down the overall execution speed.
The practice of locating these slower portions of code is called \emph{profiling}.
Before profiling, ensure that your code works as expected.
After profiling, we can attempt to optimize that section of the code so that it runs more efficiently.

\section*{When to optimize}
Trivial optimizations are of course encouraged.
Examples of trivial optimizations are intelligent string concatenations, avoiding array and list copies when possible, etc.
Trivial optimizations should never be made at the expense of code readability.
If code readability is jeopardized, the optimization is not trivial.
Any nontrivial optimizations should not be done until after the code has been tested and bottlenecks identified.
Nontrivial optimizations, if performed incorrectly, may introduce errors into your program.
Donald Knuth, often referred to as the father of the analysis of algorithms, has written:
\begin{quote}
There is no doubt that the grail of efficiency leads to abuse.
Programmers waste enormous amounts of time thinking about, or worrying about, the speed of noncritical parts of their programs, and these attempts at efficiency actually have a strong negative impact when debugging and maintenance are considered.
We \emph{should} forget about small efficiencies, say about 97\% of the time: premature optimization is the root of all evil.
Yet we should not pass up our opportunities in that critical 3\%.
A good programmer will not be lulled into complacency by such reasoning, he will be wise to look carefully at the critical code; but only \emph{after} that code has been identified.
It is often a mistake to make a priori judgments about what parts of a program are really critical, since the universal experience of programmers who have been using measurement tools has been that their intuitive guesses fail.
\footnote{Knuth, Donald. \emph{Structured Programming with go to Statements}. Computing Surveys \textbf{6}(4), 1974, 261.}
\end{quote}
In other words, don't waste time making nontrivial optimizations to code that doesn't matter.
When your code runs acceptably fast, stop optimizing.

\section*{Profiling Code}
Python includes a code profiler in its standard library.
The \li{cProfile} module tracks function calls and their execution while your code runs.
Understand that running your program with a profiler will slow down your program slightly, but times are still comparable since the profiler overhead is a constant factor.
The simplest way to profile your code is directly from the command line.
\begin{lstlisting}
python -m cProfile myscript.py
\end{lstlisting}
This will print the profile results directly to the console, but these results can also be saved to a file for later viewing by adding \texttt{-o <filename>}.
You can then analyze the results using the \li{pstats} module, also in the Python standard library.

You can also import the \li{cProfile} module and use it directly.
\begin{lstlisting}
import cProfile
def fac(n):
    if n == 1:
        return 1
    else:
        return fac(n-1)*fac(n-2)

cProfile.run('fac(100)')
\end{lstlisting}

IPython provides a nice interface to \li{cProfile} via the \li{\%prun} magic function.
\begin{lstlisting}
>>> %prun fac(100)
\end{lstlisting}

\section*{What to Optimize}
There are several common optimization tricks that might help you determine where to start first.
\begin{itemize}
\item Avoid recomputing values unnecessarily
    \subitem Move expensive computations out of loops
    \subitem Avoid copying data structures
    \subitem Cache results
    \subitem Cache often used methods
\item Avoid nested loops
    \subitem Simplify inner loops
    \subitem Consider using array operations instead of looping
\item Use builtin functions or library functions implemented in C
\item Avoid excessive function calls
\item Use a more efficient algorithm
\item Use generators when possible
\item Simplify mathematical expressions where possible
\end{itemize}

Let's look at an example of optimizing a function that calculates the QR decomposition of a matrix.
\begin{lstlisting}
def qr1(X):
    Q = X.copy()
    R = np.zeros((X.shape[0], X.shape[1]))
    for i in range(X.shape[1]):
        R[i,i] = la.norm(Q[:,i])
        Q[:,i] = Q[:,i]/la.norm(Q[:,i])
        for j in range(i+1, X.shape[1]):
            R[i, j] = Q[:,j].dot(Q[:,i])
            Q[:,j] = Q[:,j] - Q[:,j].dot(Q[:,i]) * Q[:,i]
    return Q, R
\end{lstlisting}
We generate a random $100 \times 100$ 2D array for profiling.  The output of \li{cProfile} should look similar to below.
{\scriptsize
\begin{verbatim}
      11806 function calls in 0.097 seconds

Ordered by: internal time

ncalls  tottime  percall  cumtime  percall filename:lineno(function)
     1    0.070    0.070    0.097    0.097 qropt.py:48(qr1)
  9900    0.023    0.000    0.023    0.000 {method 'dot' of 'numpy.ndarray' objects}
   200    0.001    0.000    0.004    0.000 misc.py:11(norm)
   200    0.001    0.000    0.001    0.000 {method 'reduce' of 'numpy.ufunc' objects}
   200    0.001    0.000    0.003    0.000 function_base.py:527(asarray_chkfinite)
   200    0.000    0.000    0.000    0.000 {numpy.core.multiarray.array}
   200    0.000    0.000    0.001    0.000 _methods.py:28(_all)
   200    0.000    0.000    0.001    0.000 {method 'all' of 'numpy.ndarray' objects}
   200    0.000    0.000    0.000    0.000 numeric.py:252(asarray)
   101    0.000    0.000    0.000    0.000 {range}
   200    0.000    0.000    0.000    0.000 {getattr}
   200    0.000    0.000    0.000    0.000 {method 'get' of 'dict' objects}
     1    0.000    0.000    0.000    0.000 {numpy.core.multiarray.zeros}
     1    0.000    0.000    0.000    0.000 {method 'copy' of 'numpy.ndarray' objects}
     1    0.000    0.000    0.097    0.097 <string>:1(<module>)
     1    0.000    0.000    0.000    0.000 {method 'disable' of '_lsprof.Profiler' objects}
\end{verbatim}
}
Executing \texttt{qr1(a)} results in almost 12000 function calls.
We then receive a listing of each function that was called.  In the far left column, we see how many times each function was called.
Next we see the total time spent in a given function.
The cumulative time, or \texttt{cumtime}, column measures how much time was spent in a given function including all calls to any other function.

The most obvious, and trivial, optimizations might be
\begin{itemize}
\item Avoid recomputing \li{R[i,i]} in the outer loop and \li{R[i,j]} in the inner loop
\item Use \li{xrange} instead of \li{range}
\item Cache the size of the array outside the loop
\end{itemize}
\begin{lstlisting}
def qr2(X):
    Q = X.copy()
    R = np.zeros(X.shape)

    ncols = X.shape[1]
    for i in xrange(ncols):
        R[i,i] = la.norm(Q[:,i])
        Q[:,i] = Q[:,i]/R[i,i]
        for j in xrange(i+1, ncols):
            R[i,j] = Q[:,j].dot(Q[:,i])
            Q[:,j] = Q[:,j] - R[i,j] * Q[:,i]

    return Q, R
\end{lstlisting}
These small changes, especially the removing the recomputations, speed things up nicely.
{\scriptsize
\begin{verbatim}
      5855 function calls in 0.080 seconds

Ordered by: internal time

ncalls  tottime  percall  cumtime  percall filename:lineno(function)
     1    0.064    0.064    0.080    0.080 qropt.py:73(qr2)
  4950    0.013    0.000    0.013    0.000 {method 'dot' of 'numpy.ndarray' objects}
   100    0.001    0.000    0.002    0.000 misc.py:11(norm)
   100    0.001    0.000    0.001    0.000 {method 'reduce' of 'numpy.ufunc' objects}
   100    0.001    0.000    0.001    0.000 function_base.py:527(asarray_chkfinite)
   100    0.000    0.000    0.000    0.000 {numpy.core.multiarray.array}
   100    0.000    0.000    0.001    0.000 _methods.py:28(_all)
   100    0.000    0.000    0.000    0.000 numeric.py:252(asarray)
   100    0.000    0.000    0.001    0.000 {method 'all' of 'numpy.ndarray' objects}
   100    0.000    0.000    0.000    0.000 {getattr}
     1    0.000    0.000    0.000    0.000 {method 'copy' of 'numpy.ndarray' objects}
   100    0.000    0.000    0.000    0.000 {method 'get' of 'dict' objects}
     1    0.000    0.000    0.000    0.000 {numpy.core.multiarray.zeros}
     1    0.000    0.000    0.080    0.080 <string>:1(<module>)
     1    0.000    0.000    0.000    0.000 {method 'disable' of '_lsprof.Profiler' objects}
\end{verbatim}
}
Let's use a larger, $600 \times 600$, array to exaggerate the timing differences.
{\scriptsize
\begin{verbatim}
      185105 function calls in 5.612 seconds

Ordered by: internal time

ncalls  tottime  percall  cumtime  percall filename:lineno(function)
     1    4.253    4.253    5.612    5.612 qropt.py:74(qr2)
179700    1.335    0.000    1.335    0.000 {method 'dot' of 'numpy.ndarray' objects}
   600    0.009    0.000    0.022    0.000 misc.py:11(norm)
   600    0.006    0.000    0.013    0.000 function_base.py:527(asarray_chkfinite)
   600    0.004    0.000    0.004    0.000 {method 'reduce' of 'numpy.ufunc' objects}
   600    0.001    0.000    0.001    0.000 {numpy.core.multiarray.array}
   600    0.001    0.000    0.005    0.000 _methods.py:28(_all)
   600    0.001    0.000    0.002    0.000 numeric.py:252(asarray)
     1    0.001    0.001    0.001    0.001 {method 'copy' of 'numpy.ndarray' objects}
   600    0.001    0.000    0.005    0.000 {method 'all' of 'numpy.ndarray' objects}
     1    0.000    0.000    0.000    0.000 {numpy.core.multiarray.zeros}
   600    0.000    0.000    0.000    0.000 {getattr}
   600    0.000    0.000    0.000    0.000 {method 'get' of 'dict' objects}
     1    0.000    0.000    5.612    5.612 <string>:1(<module>)
     1    0.000    0.000    0.000    0.000 {method 'disable' of '_lsprof.Profiler' objects}
\end{verbatim}
}
This seems to about as fast as we can make it.  We cannot avoid the dot product or calculating the norm.
Most of the other calls are not made directly by our function.
One option could be to try using \li{np.inner} instead of \li{np.dot}, but only makes it marginally faster.

Let's look at a non-trivial optimization.  We can see that we are doing a lot of column slicing with Q.
NumPy arrays are row major which means that the array is stored in memory by row.
This makes column slicing cumbersome because a column has exactly one element in each row, thus requiring NumPy
to jump to many different memory locations.  If we could formulate our algorithm to use rows instead of columns,
we can avoid the jumping around in memory and instead read a larger contiguous block.
We accomplish this by making \li{Q} a transposed copy of \li{X} and switching all of our column slices to row slices.
Be careful when you make the switch.
{\scriptsize
\begin{verbatim}
      185105 function calls in 1.402 seconds

Ordered by: internal time

ncalls  tottime  percall  cumtime  percall filename:lineno(function)
     1    1.086    1.086    1.402    1.402 qropt.py:5(QR)
179700    0.298    0.000    0.298    0.000 {numpy.core.multiarray.inner}
   600    0.005    0.000    0.015    0.000 misc.py:11(norm)
   600    0.004    0.000    0.004    0.000 {method 'reduce' of 'numpy.ufunc' objects}
   600    0.004    0.000    0.010    0.000 function_base.py:527(asarray_chkfinite)
     1    0.002    0.002    0.002    0.002 {method 'copy' of 'numpy.ndarray' objects}
   600    0.001    0.000    0.001    0.000 {numpy.core.multiarray.array}
   600    0.001    0.000    0.001    0.000 numeric.py:252(asarray)
   600    0.001    0.000    0.004    0.000 _methods.py:28(_all)
   600    0.000    0.000    0.005    0.000 {method 'all' of 'numpy.ndarray' objects}
     1    0.000    0.000    0.000    0.000 {numpy.core.multiarray.zeros}
   600    0.000    0.000    0.000    0.000 {getattr}
   600    0.000    0.000    0.000    0.000 {method 'get' of 'dict' objects}
     1    0.000    0.000    1.402    1.402 <string>:1(<module>)
     1    0.000    0.000    0.000    0.000 {method 'disable' of '_lsprof.Profiler' objects}
\end{verbatim}
}
The difference is immediately noticeable.  Remember that all of our optimizations thus far have been
to the way this QR algorithm is implemented.  Keep in mind that it is often far better to spend time optimizing the algorithm
than to optimize its implementation.  Even algebraic simplification of an algorithm can yield huge increases in speed.
One example of optimizing an algorithm is found in calculating Voronoi diagrams.  The algorithms that were commonly used were $O(n^2)$.
The attempts to optimize the implementation only reduced computation time by a constant factor.
Then in 1986, Steve Fortune published an algorithm that could calculate Voronoi diagrams on a plane in $O(n \log n)$.
This vastly improved the time needed to compute large Voronoi diagrams that were, up to that point in time, prohibitive to compute.

\begin{comment}
We are going to need a new problem
\begin{problem}
Practice profiling and optimizing your solutions to at least three other labs.
Some suggested labs are:
\begin{itemize}
\item Givens (Lab \ref{lab:givens})
\item Householder (Lab \ref{lab:Canonical_Transformations})
\item Image Segmentation (Lab \ref{lab:ImgSeg_eigenvalues})
\item Eigenvalue Solvers (Lab \ref{lab:EigSolve})
\item Vectorization (Lab \ref{lab:Python_Vectorization})
\item SVD (Lab \ref{lab:SVD})
\end{itemize}
Your solution should include a before and an after profiling showing the overall optimization in each of your solutions.  It should also include a list of changes, the reasoning behind the changes, and the effect of the changes on runtime.  If there are no reasonable gains to be made in performance, please note that as well.  It is often more important for code to be readable than to execute quickly.
\end{problem}
\end{comment}


% \clearpage
% \begin{subappendices}
\newappendix{Grid-world visualizations} \label{sec:gridworld-vis}

The grid-world environment we use consists of a 40x40 environment with actions up, down, left, and right, with a single start state in the lower left and a single goal state in the upper right.
In \Cref{fig:gridworld_algorithm_states} we show the state of each algorithm after training on the grid-world for 100 episodes.
While DDQN and BBE have only visited a small fraction of the states, \algshort{} has covered most of the environment and will soon solve it.
As they continue to run BBE will eventually find the goal while DDQN will not.

A video version of \Cref{fig:gridworld_algorithm_states}, which shows its evolution over time, is available in the supplement.
We find that watching the qualitative behavior of each algorithm can be enlightening.
Note that as the figures and videos are rendered by sampling states and actions, some noise in the form of missing states may appear.

\begin{figure}[h]
    \centering
    \begin{subfigure}[b]{\textwidth}
        \centering
        \includegraphics[height=0.15\textwidth]{figures/deep/gridworld_100_ddqn.png}
        \caption{DDQN}
    \end{subfigure}

    \begin{subfigure}[b]{\textwidth}
        \centering
        \includegraphics[height=0.15\textwidth]{figures/deep/gridworld_100_bbe.png}
        \caption{BBE}
    \end{subfigure}

    \begin{subfigure}[b]{\textwidth}
        \centering
        \includegraphics[height=0.15\textwidth]{figures/deep/gridworld_100_deep.png}
        \caption{DEEP}
    \end{subfigure}
    \caption{The state of each algorithm after 100 episodes in the grid-world environment.
        Note that the novelty reward and novelty value shown for DDQN are for visualization purposes only, as the algorithm does not use them.
        The ``Task value'' shown for BBE is the sum of the task and novelty rewards, which BBE treats as its objective.
        Un-visited states are marked as zero in each plot.
        The annotation (max) indicates that the value shown is the maximum value for any action at that state.
        ``Last trajectory'' shows the states visited in the last training episode.
        Figures generated by sampling.}
        \label{fig:gridworld_algorithm_states}
\end{figure}

\newappendix{Pseudo-count implementation} \label{sec:count_implementation}

% Our implementation of the kernel-based pseudo-count described in \Cref{sec:kernel_counts} has two

Define the Gaussian kernel with dimension $d$ as
\begin{align}
    k_\text{Gauss}(x, x_i) = (2 \pi) ^ {-\frac{d}{2}} \det(\bm{\Sigma})^{-\frac{1}{2}} \exp \left\{-\frac{1}{2} (x - x_i)^\intercal \bm{\Sigma}^{-1} (x - x_i) \right\}.
\end{align}
We can normalize this function to have a maximum at $k(x, x) = 1$ simply by removing the normalizer (everything outside the exponential) by noting that $e^0 = 1$.
This gives the kernel we use:
\begin{align} \label{eq:count_kernel}
    k(x, x_i) = \exp \left\{-\frac{1}{2} (x - x_i)^\intercal \bm{\Sigma}^{-1} (x - x_i) \right\},
\end{align}
where the covariance is a diagonal matrix $\bm{\Sigma} = \text{diag}(\sigma_1^2, \ldots, \sigma_d^2)$.

To compute $\hat N_n(s, a)$ using this kernel, we perform the following steps:
\begin{enumerate}
    \item Normalize $s$ and $a$:
    \begin{align}
        \bar{s} = \frac{s - \mathcal{S}_\text{min}}{\mathcal{S}_\text{max} - \mathcal{S}_\text{min}} && \bar{a} = \frac{a - \mathcal{A}_\text{min}}{\mathcal{A}_\text{max} - \mathcal{A}_\text{min}}
    \end{align}
    \item Define $x = [\bar{s}, \bar{a}]$ as the concatenation of the normalized state and action.
    \item Compute the kernel from \cref{eq:count_kernel} and sum across all of the previous normalized observations $x_i$:
    \begin{align}
        \hat N_n(s, a) = \hat N_n(x) = \sum_{i=1}^n k(x, x_i)
    \end{align}
\end{enumerate}

For this final step, we leverage the MIT-licensed Kernel Operations library (KeOps, \citet{keops}), a high-performance GPU-accelerated library for efficient computation of reductions on symbolically-defined matrices.
This substantially outperforms implementations in other frameworks, including fully-JITted JAX \citep{jax2018github} version, especially as the dimension of the data grows.

To avoid having to tune the covariance for each environment, we adapt the scaling rule of \citep{Henderson2012NormalRB}.
This rule of thumb requires assumptions on the data (notably, that it comes i.i.d. from a Normal) which are violated in the exploration problem.
However, we find this scaling to be useful in practice.
The rule of thumb bandwidth for dimension $j$ of the data for a multivariate Gaussian kernel is
\begin{align}
    h_j^{ROT} = \left( \frac{4}{2+d}  \right) ^ \frac{1}{4+d} \hat{\sigma}_j n ^ {- \frac{1}{4+d}},
\end{align}
where $\hat{\sigma}_j$ is the empirical variance of dimension $j$ of the data.
Making the assumption that our states are normalized to be in $[-1, 1]$ and distributed uniformly, we can set $\hat{\sigma}_j \approx 0.3$.
As such, in every experiment with continuous-valued states, we set the kernel variance for dimension $j$ as
\begin{align}
    \sigma_j = 0.3 \left( \frac{4}{2+d} \right) ^ \frac{1}{4+d} n ^ {- \frac{1}{4+d}} .
\end{align}
In experiments we find that changes to this scale of less than an order of magnitude make little difference.

Throughout we use 1 for the kernel variance on the action dimensions.

\subsection{Updating the kernel estimator}
Updates to the kernel count estimator consist of appending new normalized observations to the set $\{x_i\}$.
However, we find that computing this kernel becomes prohibitively slow beyond roughly 100K entries, and our experiments run for up to 1M steps.
We take two steps to avoid this slowdown.
Both rely on nonuniform weighting of entries in the observation table when computing the count, leading us to maintain an additional array of weights in addition to the table of observations.

\paragraph{Avoiding duplicate entries.}
If a new observation $x$ has $k(x, x_i) > 0.95$ with some set $M$ of existing indices, we do not add $x$ to the table, and instead add $\nicefrac{1}{|M|}$ to each entry $i \in M$ of the weights table.
In essence, if there is an exact duplicate for $x$ in the table already, we simply count that existing entry twice.
While this is helpful, the probability of observing exact matches decreases rapidly in the dimension of the observations, so this step plays a limited role.

\paragraph{Evicting previous entries.}
Once the length of the observations table reaches some maximum ($2^{15} = 32768$ in our experiments), we evict an existing entry in the table uniformly at random when we make a new entry, thus maintaining the table at that maximum size.
This introduces risk that the exploration bonus would not go to zero in the limit of many environment steps, which we avoid by re-distributing the weight of the evicted observations among those still in the table.
We do this redistribution uniformly; that is, if we evict the entry at location $i$, with weight $w_i$, we add $w_i / (n-1)$ to the weight of each of the $(n-1)$ other entries.
Our reweighting procedure maintains the same \emph{total} amount of count when evicting observations and ensures that bonuses go to zero in the limit of data.
In experiments we find that the exploration rewards earned when using a very small observation table (and thus, many evictions) were practically indistinguishable from using an observation table of unlimited size.

\subsection{Tabular environments}
For the grid-world environment used in \Cref{fig:gridworld_warmstart,fig:gridworld_visits}, we use a tabular visit count rather than pseudo-counts.

\newappendix{Details on rapid Q updates} \label{sec:fast_updates_appendix}

We aim to rapidly update $\qex$ to maximize reward on the non-stationary exploration MDP $\mdp_{R^+_n}$ and thus explore rapidly.
This has three components: (1) updating using the current reward function $R^+_n$ rather than logged rewards, (2) performing many updates to $\qex$ at every timestep using a large learning rate, and (3) using an optimistic version of $\qex$ which is aware of the high value of taking actions that have not yet been explored.
However, aggressively updating $\qex$ poses its own problems; most significantly, Q-learning with function approximation has a tendency to diverge if updated too aggressively with too few new samples.
We use three modifications to the typical Bellman update with target networks \citep{mnih2015human} to mitigate this issue while incorporating optimism.

\begin{itemize}
    \item \textbf{Soft DoubleDQN update.} The DoubleDQN \citep{Hasselt2016DeepRL} update reduces overestimation in Q-learning by selecting and evaluating actions using different parameters.
    We use a soft version of the DoubleDQN update by replacing the max operator with an exponential-Q policy over uniform random actions using a low temperature.
    \item \textbf{Value clipping.} To further mitigate the problem of Q-learning overestimation and divergence, we clip the Bellman targets to be within the range of possible Q values for $\mdp_{R^+_n}$.
    Given that the rewards $r^+$ are scaled to be in $[0, 1]$, any policy would have a value $\qex(s, a) \in [0, \bar{r}]$, where $\bar{r} = \nicefrac{1}{1 - \gamma}$.
    \item \textbf{Optimistic targets.} We use the optimistic adjustment in \cref{eq:optimism} when computing targets.
\end{itemize}

Define the softmax-Q policy for some Q function $Q$ as
\begin{align}
    \pi(a \mid s; Q) = \frac{\exp \big\{ \nicefrac{Q(s, a)}{\tau} \big\}}{\int_\mathcal{A} \exp \big\{ \nicefrac{Q(s, a')}{\tau}  ~da' \big\}}
\end{align}
which we approximate using self-normalized importance sampling with a uniform proposal distribution.
The target for updating $\qex$ is
\begin{align} \label{eq:update_target}
    y(s, a, s') = \text{clip} \left( R^+_n(s, a) + \gamma \E_{a' \sim \pi(\cdot \mid s'; \qex^+)} \Big[ \qex^+(s', a'; \theta^-) \Big], 0, \bar{r} \right).
\end{align}
where $R^+_n(s, a)$ is the current exploration bonus, which we recompute at update time; $\qex^+(s', a'; \theta^-)$ is the target network for the exploration value function, with optimism applied.
We then minimize the squared error between $y(s, a, s')$ and $\qex(s, a)$.

\newappendix{Environments for exploration} \label{sec:environments_appendix}

To enable benchmarking the performance of exploration methods on continuous control, we constructed a new set of environments.
Our motivation comes from the challenges of performing resets and defining shaped rewards in real-world robotics, where it is not possible to measure and set states exactly.
Unlike in simulation, it may be difficult or impossible to implement uniform resets of the robot and the objects in the scene; states with a walking robot standing upright or a block in midair require significant expertise to reach.
Similarly many shaped rewards in simulation rely on precise knowledge of the locations of objects in a scene to provide rewards corresponding to e.g. an objects distance from a goal.
We make modifications which capture the spirit of these real-world constraints, though these exact environments might still be difficult to construct in the real world:
\begin{itemize}
    \item Small reset distributions.
    Instead of resetting every object in the scene uniformly in the space, we randomize each object's configuration over a smaller set of starting states.
    This reflects some properties of real environments, such as walking robots starting on the ground instead of midair, or the object in a manipulation task not starting in its goal receptacle.
    \item Sparse rewards.
    While dense rewards are difficult to construct without real-time monitoring of object positions, sparse rewards are often simpler.
    A picking task, for example, can provide a sparse reward simply by checking whether the desired object is inside a receptacle.
\end{itemize}

As the base environments for our benchmark, we select four tasks from DeepMind Control Suite \citep{tassa2018deepmind}, an Apache-licensed standard set of benchmarks implemented using the commercial MuJoCo physics simulator \citep{todorov2012mujoco}.
Denoting the state (observation) and action dimensions of an environment as $\text{dim}(\mathcal{S}) \rightarrow \text{dim}(\mathcal{A})$, these environments are:
\begin{itemize}
    \item \textbf{Ball-in-cup catch} (manipulation, $8d \rightarrow 2d$).
    \item \textbf{Reacher hard} (goal-directed, $6d \rightarrow 2d$).
    \item \textbf{Finger turn\_hard} (manipulation, goal-directed, $12d \rightarrow 2d$).
    \item \textbf{Walker walk} (locomotion, $24d \rightarrow 6d$).
\end{itemize}

We modify each environment to remove the accommodations of wide reset distributions and sparse rewards which make them easy to solve without directed exploration.
The new environments and their changes are as follows:
\begin{itemize}
    \item \textbf{Ball-in-cup explore} (manipulation, $8d \rightarrow 2d$). The original task resets the ball uniformly in the reachable space, including already in the cup (the goal state).
    We modify the environment to only reset the ball in a region below the cup, as if the ball was hanging and gently swinging.
    The original task already has sparse rewards.
    \item \textbf{Reacher explore} (goal-directed, $6d \rightarrow 2d$).
    The original task samples arm positions and goal positions uniformly, resulting in the arm being reset very near the goal.
    We modify the reset distribution to only include states with the arm mostly extended to the right and targets opposite it on the left in a cone with angle $\pi/2$.
    Note that this task is somewhat easier than the original since the policy only needs to navigate between smaller regions of the space, but is harder due to the resets not providing exploration.
    The original task already has sparse rewards.
    \item \textbf{Finger explore} (manipulation, goal-directed, $12d \rightarrow 2d$).
    The original task resets the finger uniformly, the angle of the spinner uniformly, and the goal location uniformly on the circle reachable by the tip of the spinner.
    We modify the environment to reset the finger joints each in the quadrant pointing away from the spinner, the spinner pointing in the downward quadrant, and the target in the upward quadrant.
    Similarly to Reacher explore, this task is simpler than the original but harder to explore in.
    The original task already has sparse rewards.
    \item \textbf{Walker explore} (locomotion, $24d \rightarrow 6d$).
    The original environment resets the walker just above the ground with random joint angles, leading to it frequently starting upright and in position to begin walking.
    We modify the environment by allowing time to progress for 200 steps in the underlying physics (20 environment steps), which is enough time for the walker to fall to the floor.
    This simulates the walker starting in a random lying-down configuration.
    The original rewards include a sparse reward for being upright and above a certain height and a linear component for forward velocity.
    We replace the forward velocity reward with a sparse version which provides reward only when the agent is moving at or above the target speed.
\end{itemize}

The code for these environments is included in the supplement.

\newappendix{Experimental implementation details} \label{sec:appendix_benchmark_implementation}

\subsection{Computing infrastructure}
We implemented \algshort{} using JAX \citep{jax2018github} and the neural network library Flax \citep{flax2020github} which is built on it, both of which are Apache-licensed libraries released by Google.
The SAC implementation we use is from \citet{pytorch_sac} (MIT-licensed), built on Pytorch \citep{Paszke2019PyTorchAI} (custom BSD-style license).
The experiments in this paper take about a week to run using 16 late-model NVidia GPUs by running 2-4 seeds of each experiment at once on each GPU.

\subsection{Network architectures and training}
For the Q networks used as $\qex$ in continuous environments and as the task policy in the grid-world experiments, we use fully-connected networks with two hidden layers of 512 units each and ReLU activations.
These networks flatten and concatenate the state and action together to use as input and produce a 1d value prediction.
This enables us to use the same networks and training for discrete and continuous actions rather than using the usual discrete-action trick of simultaneously producing a Q-value estimate for every action given the state.

These Q networks are trained using the Adam optimizer \citep{kingma2014adam} with learning rate $10^{-3}$.
$\qex$ is updated with two Bellman updates with batch size 128 per environment step.
We update the target network after every environment step for $\qex$ to allow very rapid information propagation about changing rewards.
For the Q network defining $\pitask$ in the grid-world we use a learning rate of $10^{-4}$ and update the target network after every 50 Bellman updates.

\subsection{Other hyperparameters}
We draw 64 samples from $\pitask$ when computing the behavior policy and 64 samples from a uniform distribution over actions when updating $\qex$ as described in Appendix \Cref{sec:fast_updates_appendix}.
We set the temperature for Boltzmann sampling from all Q-network policies as $\tau = 0.1$.
$\qex$ uses a discount $\gamma = 0.99$.

% \newpage
\newappendix{Additional benchmark results} \label{sec:benchmark_results_appendix}

We performed an experiment to check whether it was simply the separation of learning two separate Q functions which enabled \algshort{}'s performance.
To do this, we modified SAC + BBE to learn one Q function for the task reward function and one Q function for the exploration reward function.
The policy was then trained to maximize the sum of those two Q functions.
This baseline, which we call SAC 2Q, performed uniformly worse than SAC + BBE, but we include its results here for completeness.

\begin{figure}[h]
    \centering
    \begin{subfigure}[b]{0.24\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/deep/neurips_SAC2Q_ball_in_cup.pdf}
        \caption{Ball-in-cup}
    \end{subfigure}
    \begin{subfigure}[b]{0.24\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/deep/neurips_SAC2Q_ball_in_cup_explore.pdf}
        \caption{Ball-in-cup explore}
    \end{subfigure}
    \hfill
    \begin{subfigure}[b]{0.24\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/deep/neurips_SAC2Q_reacher.pdf}
        \caption{Reacher}
    \end{subfigure}
    \begin{subfigure}[b]{0.24\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/deep/neurips_SAC2Q_reacher_explore.pdf}
        \caption{Reacher explore}
    \end{subfigure}
    \vspace{1em}

    \begin{subfigure}[b]{0.24\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/deep/neurips_SAC2Q_finger.pdf}
        \caption{Finger}
    \end{subfigure}
    \begin{subfigure}[b]{0.24\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/deep/neurips_SAC2Q_finger_explore.pdf}
        \caption{Finger explore}
    \end{subfigure}
    \hfill
    \begin{subfigure}[b]{0.24\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/deep/neurips_SAC2Q_walker.pdf}
        \caption{Walker}
    \end{subfigure}
    \begin{subfigure}[b]{0.24\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/deep/neurips_SAC2Q_walker_explore.pdf}
        \caption{Walker explore}
    \end{subfigure}
    \caption{SAC 2Q performs uniformly worse than SAC + BBE.}
\end{figure}
% \end{subappendices}
\printendnotes


\documentclass[12pt]{article}

\usepackage{titling}
\newcommand{\subtitle}[1]{%
	\posttitle{%
		\par\end{center}
	\begin{center}\large#1\end{center}
	\vskip0.5em}%
}
\usepackage{longtable}
\usepackage{amsfonts, amsmath, amssymb, bm} %Math fonts and symbols
\usepackage{dcolumn, multirow} % decimal-aligned columns, multi-row cells
\usepackage{graphicx} % graphics commands
\usepackage{subfig}
\usepackage{float}
\usepackage[margin=1in]{geometry} % sets page layout
\usepackage{setspace}% allows toggling of double/single-spacing
\usepackage{verbatim}% defines environment for un-evaluated code
\usepackage{rotating}% defines commands for rotating text and floats
\usepackage[sort]{natbib}% defines citation commands and environments.
\usepackage{url}
\usepackage{caption}
\usepackage{endnotes}
\usepackage{psfrag}
\usepackage{epstopdf}
\usepackage{epsfig}
\usepackage[pdftex,
            pdfauthor={Tom Hanna},
            pdftitle={The Title},
            pdfsubject={The Subject},
            pdfkeywords={Some Keywords},
            pdfcreator={TeXstudio}]{hyperref}
\usepackage{fancyhdr, amsfonts, amsmath, amssymb, graphicx, color, fullpage}
\pagestyle{fancy}
\usepackage{fourier}
\usepackage[lf]{venturis}
\usepackage[T1]{fontenc}
\DeclareMathOperator*{\argmax}{arg\,max}

\usepackage{outlines}
\usepackage{enumitem}[shortlabels]
\setlist[enumerate,1]{label=\Roman*)}
\setlist[enumerate,2]{label=\Alph*)}
\setlist[enumerate,3]{label=\roman*)}
\setlist[enumerate,4]{label=\alph*)}

\setlength\parindent{0pt}
\setlength{\headsep}{0.2in}

\usepackage{tikz}
\usetikzlibrary{trees}

\usetikzlibrary{shapes,decorations,arrows,calc,arrows.meta,fit,positioning}
\tikzset{
    -Latex,auto,node distance =1 cm and 1 cm,semithick,
    state/.style ={ellipse, draw, minimum width = 0.7 cm},
    point/.style = {circle, draw, inner sep=0.04cm,fill,node contents={}},
    bidirected/.style={Latex-Latex,dashed},
    el/.style = {inner sep=2pt, align=left, sloped}
}

\usepackage[sectionbib]{chapterbib}

\title{Sample Lab Presentation: Lab 9}

\author{Tom Hanna \\ University of Houston \\  Department of Political Science \\ tlhanna@uh.edu}

\begin{document}
\clearpage\maketitle
\thispagestyle{empty}
\setcounter{page}{1}

	\lhead{Tom Hanna}

	\chead{Lab 9 Sample}

	\rhead{POLS6481}

	{\textcolor{white}.}

\newpage

\doublespacing

	{\textcolor{white}.}

\section*{Raw code}

\begin{verbatim}
> library(here)
here() starts at C:/Users/tomha/Documents/3 - R Studio Projects/Teaching/POLS6481-Spring2021-UH-lab
> library(foreign)
> library(tseries) #For lagging
Registered S3 method overwritten by 'quantmod':
  method            from
  as.zoo.data.frame zoo

    ‘tseries’ version: 0.10-48

    ‘tseries’ is a package for time series analysis and computational finance.

    See ‘library(help="tseries")’ for details.

> library(lmtest) #For test for joint significance
Loading required package: zoo

Attaching package: ‘zoo’

The following objects are masked from ‘package:base’:

    as.Date, as.Date.numeric

> library(plm) #For plm command  --- if needed install.packages("plm")
> library(pglm)
Loading required package: maxLik
Loading required package: miscTools

Please cite the 'maxLik' package as:
Henningsen, Arne and Toomet, Ott (2011). maxLik: A package for maximum
likelihood estimation in R. Computational Statistics 26(3), 443-458.
DOI 10.1007/s00180-010-0217-1.

If you have questions, suggestions, or comments regarding the
 'maxLik' package, please use a forum or 'tracker' at maxLik's R-Forge site:
https://r-forge.r-project.org/projects/maxlik/
> #This is using data on non-democratic nations involvement in Militarized
> Interstate Disputes
> #(MIDS), the data is from the Correlates of War dataset. Additionally,
> #there are variables from the Varieties of Democracy (VDEM) project and
> from Jeff Colgan's
> #Revolutionary Leader's database. This is unbalanced panel data, so
> #I won't be using the PLM package.
> library(readr)
> NDC <- read_csv(here("data","nondemocraciesconflict.csv"))

-- Column specification -----------------------------------------------------------------------------
cols(
  .default = col_double(),
  country_name = col_character(),
  v2lpname = col_logical(),
  v2reginfo = col_character(),
  leader = col_character(),
  stabb = col_character()
)
i Use `spec()` for the full column specifications.

Warning messages:
1: Missing column names filled in: 'X1' [1]
2: Duplicated column names deduplicated: 'X1' => 'X1_1' [149]
> View(NDC)
> names(NDC) #I like to do this because it makes organizing and using variables easier
  [1] "X1"                         "COWcode"                    "year"
  [4] "country_name"               "v2x_polyarchy"              "v2x_libdem"
  [7] "v2x_partipdem"              "v2x_delibdem"               "v2x_egaldem"
 [10] "v2x_regime"                 "v2x_regime_amb"             "v2x_ex_military"
 [13] "v2x_ex_confidence"          "v2x_ex_direlect"            "v2x_ex_hereditary"
 [16] "v2x_ex_party"               "v2x_neopat"                 "v2xnp_client"
 [19] "v2x_frassoc_thick"          "v2x_jucon"                  "v2xlg_legcon"
 [22] "v2x_cspart"                 "v2lpname"                   "v2exhoshog"
 [25] "v2exremhog"                 "v2exdjdshg"                 "v2exdfvthg"
 [28] "v2reginfo"                  "v2regint"                   "v2regendtype"
 [31] "v2regimpgroup"              "v2regsupgroupssize"         "v2lgoppart"
 [34] "v2dlencmps"                 "v2juhcind"                  "v2juncind"
 [37] "v2juhccomp"                 "v2jucomp"                   "v2jureview"
 [40] "v2stfisccap"                "v2svstterr"                 "v2mecenefi"
 [43] "v2mecenefm"                 "v2exl_legitideol"           "v2exl_legitideolcr_0"
 [46] "v2exl_legitideolcr_1"       "v2exl_legitideolcr_2"       "v2exl_legitideolcr_3"
 [49] "v2exl_legitideolcr_4"       "v2xpe_exlsocgr"             "v2x_gencl"
 [52] "v2x_rule"                   "v2xcl_prpty"                "v2xcs_ccsi"
 [55] "v2x_clpol"                  "v2x_clpriv"                 "v2clfmove"
 [58] "v2xcl_dmove"                "v2cldmovem"                 "v2cldmovew"
 [61] "v2cldiscm"                  "v2cldiscw"                  "v2clslavem"
 [64] "v2clslavef"                 "v2clstown"                  "v2clprptym"
 [67] "v2clprptyw"                 "v2clacjstm"                 "v2clacjstw"
 [70] "e_legparty"                 "e_autoc"                    "e_peaveduc"
 [73] "e_migdppc"                  "e_migdppcln"                "e_cow_exports"
 [76] "e_cow_imports"              "e_total_fuel_income_pc"     "e_total_oil_income_pc"
 [79] "e_miurbani"                 "e_mipopula"                 "e_civil_war"
 [82] "e_miinteco"                 "v2exrmhgnp_0"               "v2exrmhgnp_1"
 [85] "v2exrmhgnp_2"               "v2exrmhgnp_3"               "v2exrmhgnp_4"
 [88] "v2exrmhgnp_5"               "v2exrmhgnp_6"               "v2exrmhgnp_7"
 [91] "v2exrmhgnp_8"               "v2exctlhg_0"                "v2exctlhg_1"
 [94] "v2exctlhg_2"                "v2exctlhg_3"                "v2exctlhg_4"
 [97] "v2exctlhg_5"                "v2exctlhg_6"                "v2exctlhg_7"
[100] "v2exctlhg_8"                "v2regsupgroups_0"           "v2regsupgroups_1"
[103] "v2regsupgroups_2"           "v2regsupgroups_3"           "v2regsupgroups_4"
[106] "v2regsupgroups_5"           "v2regsupgroups_6"           "v2regsupgroups_7"
[109] "v2regsupgroups_8"           "v2regsupgroups_9"           "v2regsupgroups_10"
[112] "v2regsupgroups_11"          "v2regsupgroups_12"          "v2regsupgroups_13"
[115] "v2exl_legitlead"            "ccode.x"                    "onsets"
[118] "revonsets"                  "sideaonsets"                "force_onsets"
[121] "force_revonsets"            "force_sideaonsets"          "leader"
[124] "age0"                       "age"                        "usedforce"
[127] "irregulartransition"        "foundingleader"             "foreigninstall"
[130] "radicalideology"            "democratizing"              "revolutionaryleader"
[133] "chg_executivepower"         "chg_politicalideology"      "chg_propertyownership"
[136] "chg_womenandethnicstatus"   "chg_religioningovernment"   "chg_revolutionarycommittee"
[139] "totalcategorieschanged"     "muslim"                     "durable"
[142] "cinc"                       "majorpower"                 "peaceyears"
[145] "dem"                        "conflict.status"            "trade"
[148] "rule.inverse"               "X1_1"                       "dispnum"
[151] "stabb"                      "ccode.y"                    "stday"
[154] "stmon"                      "styear"                     "endday"
[157] "endmon"                     "endyear"                    "sidea"
[160] "revstate"                   "revtype1"                   "revtype2"
[163] "fatality"                   "fatalpre"                   "hiact"
[166] "hostlev"                    "orig"                       "version"
[169] "outcome"                    "settle"                     "recip"
[172] "numa"                       "numb"                       "legitideol"
[175] "nationalist"                "socialist"                  "conservative"
[178] "autonomist"                 "religious"                  "totalcat"
[181] "mideology"                  "mnationalist"               "msocialist"
[184] "mconservative"              "mautonomist"                "mreligious"
[187] "initiator"                  "revisionist"                "policyrevisionist"
[190] "regimerevisionist"          "territorialrevisionist"     "otherrevisionist"
[193] "nonterritorial"             "backdown"                   "hostile"
[196] "hostileorig"                "hostilepolicy"              "hostileregime"
[199] "personofleader"             "coldwar"                    "transformativeideology"
[202] "cinc2"
> ## The following produces an error because there are duplicates - some countries (COWcodes)
> ## appear in multiple disputes in the same year. This is okay, because at this point I am
> ## just demonstrating how to copy the console output to LaTeX.
>
> panelNDC <- pdata.frame(NDC, c("COWcode","year"))
Warning message:
In pdata.frame(NDC, c("COWcode", "year")) :
  duplicate couples (id-time) in resulting pdata.frame
 to find out which, use, e.g., table(index(your_pdataframe), useNA = "ifany")
> pdim(panelNDC)
Error in pdim.default(index[[1L]], index[[2L]]) :
  duplicate couples (id-time)
>
\end{verbatim}

\pagebreak

\section*{Part 2: Plots}

\begin{itemize}
\item Plot of 1984, peace years versus transformative ideology.

\begin{figure}[H]
\centering
\caption[]{Data from 1984}
\label{fig:ti-vs-peaceyears-1984}
\includegraphics[width=0.9\linewidth]{ti-vs-peaceyears-1984}
\end{figure}

\item Plot of 1992, peace years versus transformative ideology.

\begin{figure}[H]
\centering
\caption[Data from 1992]{This is the plot of peace years versus transformative ideology for nondemocracies in 1992.}
\label{fig:ti-vs-oeaceyears-1992}
\includegraphics[width=0.9\linewidth]{ti-vs-oeaceyears-1992}
\end{figure}

\end{itemize}

\section*{Part 3: Results Tables}

\paragraph*{My results} show that XYZ (See Table 1, column Logit Model). As you can see in Table 1, the Logit Model was consistent with the Fixed Effects Logit Model.

% Table created by stargazer v.5.2.2 by Marek Hlavac, Harvard University. E-mail: hlavac at fas.harvard.edu
% Date and time: Tue, Apr 27, 2021 - 9:57:43 PM
\begin{table}[H] \centering
  \caption{Transformative Ideology Effect on Nonterritorial Revisionist Demands}
  \label{}
\begin{tabular}{@{\extracolsep{5pt}}lcc}
\\[-1.8ex]\hline
\hline \\[-1.8ex]
 & \multicolumn{2}{c}{\textit{Dependent variable:}} \\
\cline{2-3}
\\[-1.8ex] & \multicolumn{2}{c}{Non-territorial Revisionist Demands} \\
\\[-1.8ex] & (Logit) & (Fixed Effects Logit Model)\\
\hline \\[-1.8ex]
 Transformative Ideology & 1.056$^{***}$ & 0.778$^{**}$ \\
  & (0.205) & (0.375) \\
  & & \\
 Combined Index of & 0.030$^{***}$ & 0.044 \\
  National Capabilities & (0.011) & (0.036) \\
  & & \\
 Cold War & 0.157 & $-$0.026 \\
 (dummy variable) & (0.127) & (0.183) \\
  & & \\
 Peace Years & $-$0.023 & $-$0.026 \\
  & (0.015) & (0.020) \\
  & & \\
 Constant & $-$1.167$^{***}$ & $-$1.576$^{***}$ \\
  & (0.118) & (0.594) \\
  & & \\
\hline \\[-1.8ex]
Observations & 1,768 & 1,768 \\
Log Likelihood & $-$1,059.119 & $-$861.691 \\
Akaike Inf. Crit. & 2,128.237 & 1,985.381 \\
\hline
\hline \\[-1.8ex]
\textit{Note: CFE ommitted for space}  & \multicolumn{2}{r}{$^{*}$p$<$0.1; $^{**}$p$<$0.05; $^{***}$p$<$0.01} \\
\end{tabular}
\end{table}

\newpage

\singlespacing

{\textcolor{white}.}

\bibliographystyle{apsr}

\bibliography{reference}

\end{document}

Demonstration/Presentation/sample latex file 2.tex