\documentclass[xcolor={dvipsnames}]{beamer}

\usepackage[utf8]{inputenc}
\usepackage[english]{babel}
\usepackage{xcolor,colortbl}
\usepackage{fancyvrb}
\usepackage{minted}
\usepackage{hyperref}
\usepackage{authoraftertitle}
\usepackage[normalem]{ulem}

\usetheme{Boadilla}

\author{Andrew Cooper}
\title{Xen Test Framework}
\subtitle{Testing from a guest's perspective}
\institute{Citrix XenServer}
\date{Thursday $13^{th}$ July 2017}

\begin{document}

\frame{\titlepage}

\begin{frame}{Views on testing}

  \begin{itemize}
  \item<2-> Ever thought?
    \begin{itemize}
    \item ``This bugfix is so simple, it is clearly correct''
    \item<3-> ``I've only got time for a dev test at the moment''
    \item<4-> ``Writing a test for that will be too hard''
    \end{itemize}
  \bigskip
  \item<5-> Who actually likes testing?
  \bigskip
  \item<6-> Testing is frequently a lower priority activity than it should be
    \begin{itemize}
    \item Therefore, it tends not to get done
    \item Especially if there is no pressure to do so
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Testing is perceived as being hard}

  \begin{itemize}
  \item<2-> Because it is...
    \begin{itemize}
    \item \textbf{... without good infrastructure}
    \end{itemize}
    \bigskip

  \item<3-> Technical problems boil down to the fact that:
    \begin{itemize}
    \item Writing tests is hard
    \item Automating written tests is hard
    \end{itemize}
    \bigskip

  \item<4-> The Xen Test Framework, and associated infrastructure, intend to
      remove these obstacles
  \end{itemize}
\end{frame}

\begin{frame}[fragile]
  \frametitle{``Writing tests is hard''}

  \begin{itemize}
  \item Easy 4-step guide:
  \bigskip

  \item<2->[] \verb!root@testbox:~/xtf# ./make-new-test.sh mytest!
  \item<3->[] \verb!root@testbox:~/xtf# $EDITOR tests/mytest/main.c!
  \item<4->[] \verb!root@testbox:~/xtf# make!
  \item<5->[] \verb!root@testbox:~/xtf# ./xtf-runner mytest!
  \medskip

  \item<6->[] \verb!Combined test results:!
  \item<6->[] \verb!test-pv32pae-mytest                      SUCCESS!
  \item<6->[] \verb!test-pv64-mytest                         SUCCESS!
  \item<6->[] \verb!test-hvm32-mytest                        SUCCESS!
  \item<6->[] \verb!test-hvm32pse-mytest                     SUCCESS!
  \item<6->[] \verb!test-hvm32pae-mytest                     SUCCESS!
  \item<6->[] \verb!test-hvm64-mytest                        SUCCESS!

  \end{itemize}
\end{frame}

\begin{frame}[fragile]
  \frametitle{``Writing tests is hard''}
\begin{minted}{c}
#include <xtf.h>

const char test_title[] = "XSA-203 PoC";
bool test_needs_fep = true;

void test_main(void)
{
    asm volatile (_ASM_XEN_FEP
                  "1: vmfunc; 2:"
                  /* Ignore #UD on older Xen versions. */
                  _ASM_EXTABLE(1b, 2b)
                  :: "a" (0));

    /* If Xen is alive, it didn't hit the NULL pointer. */
    xtf_success("Success: Not vulnerable to XSA-203\n");
}
\end{minted}

\end{frame}

\begin{frame}[fragile]
  \frametitle{``Automating written tests is hard''}

  \begin{itemize}
  \item Easy 2-step guide:
  \bigskip

  \item<2->[] Get patch to me via xen-devel
  \bigskip

  \item<3->[] \verb!me@box:~/xtf$ git push upstream master!
  \bigskip

  \item<4-> New test will be picked up automatically by OSSTest
  \item<4-> Will start blocking pushes to master if a regression is detected
  \end{itemize}

\end{frame}

\begin{frame}[fragile]
  \frametitle{History of XTF}

  \begin{itemize}
  \item XSA-106 was initially reported to XenServer (August 2014)
    \begin{itemize}
    \item ``\verb!x86_emulate()! doesn't perform DPL checks for software
      exceptions''
    \item These are: \verb!int3!, \verb!into!, \verb!int $x! and \verb!icebp!
    \item A windows userspace PoC existed, which would cause a BSOD
    \end{itemize}

  \item<2-> On inspection:
    \begin{itemize}
    \item No checks of the IDT Entry at all
    \item Also important to check the descriptor type and present bit
    \item All software events injected as hardware exceptions
    \end{itemize}

  \item<3-> First stab at a fix changed the BSOD, but didn't fix it

  \item<4-> Needed an easier repro!
    \begin{itemize}
    \item Started with \verb!hvmloader!
    \item Removed all BIOS-related bits, added an IDT
    \item Borrowed the Force Emulation Prefix from PV guests
    \end{itemize}

  \item<4-> A mess, but it did work
  \end{itemize}

\end{frame}

\begin{frame}[fragile]
  \frametitle{History of XTF}

  \begin{itemize}
  \item The successful case semantics were wrong
    \begin{itemize}
    \item Should have had Trap semantics, actually had Fault semantics
    \item Fixed the test code up to detect and break infinite loops
    \end{itemize}
  \item<2-> The \verb!icebp! instruction bypasses DPL checks
    \begin{itemize}
    \item Also sets the External bit in error codes
    \end{itemize}
  \item<3-> The first proposed fix didn't actually work on non-Intel hardware
    \begin{itemize}
    \item Without NRIPS, AMD hardware can't correctly inject a software
      exception which faults for IDT-related reasons
    \item With NRIPS, AMD hardware experimentally still can't inject
      \verb!icebp! properly if it would fault
    \end{itemize}
  \item<4-> XSA-106 was released (September 2014)
    \medskip
  \item<5-> XSA-156 was released (November 2015)
    \begin{itemize}
    \item The fix was buggy, and caused infinite loops when the guest used
      \verb!int3!
      \item Should have been caught during development
      \item Would have been caught if the XSA-106 code had been in automation
    \end{itemize}
  \end{itemize}

\end{frame}

\begin{frame}[fragile]
  \frametitle{So what is XTF, exactly?}

  \begin{itemize}
  \item A microkernel core:
    \begin{itemize}
    \item Performs minimal setup at boot (pagetables, exceptions, console, etc)
    \item Non-essential infrastructure available from the library
    \item Uniform reporting mechanism (success, skip, error, failure)
    \end{itemize}
  \item<2-> A multi-guest build system:
    \begin{itemize}
    \item Write one main.c, compile for any/all environments
    \item PV or HVM, 32bit or 64bit, unpaged/PSE/PAE/Long mode
    \end{itemize}
  \item<3-> A utility to run one or more tests from dom0:
    \begin{itemize}
    \item \verb!./xtf-runner $MYTEST!
    \item Scriptable
    \end{itemize}
  \item<4-> A suite of tests:
    \begin{itemize}
    \item Sorted into broad categories (functional, XSA, utilities)
    \end{itemize}
  \item<5-> Quick and easy to use:
    \begin{itemize}
    \item \verb!time ./xtf-runner --all --quiet!
    \item Haswell test box, 7.2s for all current tests (62) to run
      sequentially
    \item My Edit/Compile/Rerun cycle is a matter of seconds
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Examples of tests}

  \begin{itemize}
  \item CPUID Faulting (all environments)
    \begin{itemize}
    \item Xen 4.8 introduced guest CPUID faulting support
    \item 71 LoC (137 inc. docs), test/probe/enable/retest/disable/retest cycle
    \item Many subsequent CPUID changes in Xen, made with full confidence
    \end{itemize}

  \item<2-> NMI Taskswitch Priv (hvm32pae)
    \begin{itemize}
    \item Xen 4.9 regression with task switching (fixed around rc9!)
    \item 93 LoC (186 inc. docs), NMI Task Gate, userspace-initiated self-NMI
    \end{itemize}

  \item<3-> XSA-203 CVE-2016-10025 (hvm32, shown before)
    \begin{itemize}
    \item NULL pointer dereference with \verb!vmfunc! emulation
    \item 12 LoC (42 inc. docs)
    \end{itemize}

  \item<4-> XSA-186 CVE-2016-7093 (hvm32, hvm64)
    \begin{itemize}
    \item Bad truncation of \verb!%eip!, underflowing the instruction cache
    \item 119 LoC (241 inc. docs), 16bit code segment, executing above 64k
    \item Suspected Broadwell TLB erratum
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Pagetable emulation test}

  \begin{itemize}
  \item Started when cleaning up the XSA-176 PoC
    \begin{itemize}
    \item Mysteriously descheduled itself and ceased executing
    \end{itemize}
  \item<2-> Found a large number of pagetable walking bugs
    \begin{itemize}
    \item Comprehensive test of Xen's pagewalk against real hardware
    \item Skylake with PKRU, 4.2s to run, 2252800 unique pagewalks
    \end{itemize}
  \end{itemize}

  \onslide<3->
  \begin{center}
    \scriptsize
    \begin{BVerbatim}
Issues fixed:
 * 2-level PSE36 superpages now return the correct translation.
 * 2-level L2 superpages without CR0.PSE now return the correct translation.
 * SMEP now inhibits a user instruction fetch even if NX isn't active.
 * Supervisor writes without CR0.WP now set the leaf dirty bit.
 * L4e._PAGE_GLOBAL is strictly reserved on AMD.
 * 3-level l3 entries have all reserved bits checked.
 * 3-level entries can no longer alias Xen's idea of paged or shared.
    \end{BVerbatim}
  \end{center}

  \begin{itemize}
  \item<4-> Still under development.  Outstanding issues:
    \begin{itemize}
    \item Xen leaks EFER.NX into guests on Intel hardware
    \item AMD Zen doesn't order A/D updates with loads
    \item Intel and AMD's implementation of SMAP differs
    \end{itemize}
  \end{itemize}

\end{frame}

\begin{frame}[fragile]
  \frametitle{Further testing ideas}

  \begin{itemize}
  \item Currently, tests are ``boot microkernel, wait for it to exit''
    \begin{itemize}
    \item Easy, and very effective for a lot of tasks
    \end{itemize}

  \item<2-> Device Model / Introspection testing
    \begin{itemize}
    \item Dom0 test agent connects to the IOREQ/VM\_EVENT ring
    \item Microkernel makes a set of specific actions
    \item Test agent checks for correct requests in the ring, and responds
    \item Microkernel checks for correct results of the responses
    \end{itemize}

  \item<3-> Configuration testing
    \begin{itemize}
    \item In dom0, iterate over VM configuration options
    \item Boot the microkernel for each configuration, and check
    \end{itemize}

  \item<4-> Performance testing
    \begin{itemize}
    \item Microbenchmarking or multi-domain stress testing
    \end{itemize}

  \item<5-> Combine with coverage
    \begin{itemize}
    \item See which paths are actually covered by a test
    \end{itemize}

  \end{itemize}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Any Questions?}

  \begin{itemize}
  \item Source:
    \begin{itemize}
    \item \verb! git://xenbits.xen.org/xtf.git!
    \item \color{blue}\url{http://xenbits.xen.org/gitweb/?p=xtf.git}
    \end{itemize}
    \medskip
  \item Documentation:
    \begin{itemize}
    \item \color{blue}\url{http://xenbits.xen.org/docs/xtf/}
    \end{itemize}
    \medskip
  \item Examples:
    \begin{itemize}
    \item Already 37 tests upstream and running automatically
    \item \color{blue}\url{http://xenbits.xen.org/docs/xtf/test-index.html}
    \end{itemize}
    \medskip
  \item Discussion:
    \begin{itemize}
    \item \color{blue}\href{mailto:xen-devel@lists.xenproject.org}
      {xen-devel@lists.xenproject.org}
    \item \color{blue}\url{irc://#xendevel@chat.freenode.net}
    \end{itemize}
  \end{itemize}
\end{frame}

\end{document}

\iffalse

\begin{frame}
  \frametitle{...}
\end{frame}

  \begin{itemize}
    \item
  \end{itemize}

\fi
