\documentclass[12pt]{article}
\usepackage{multirow,amssymb,amsmath,amsthm,framed,color,tikz,titlesec,setspace,mathtools,cancel}
\usepackage[framemethod=tikz]{mdframed}
\usetikzlibrary{matrix,arrows}
\usetikzlibrary{backgrounds,calc}
\usepackage{tocstyle}
\usetocstyle{standard}

%\textwidth6.25in \textheight8.5in \oddsidemargin.25in
\topmargin0in

\newtheorem{theorem}{Theorem}[section]
%\newtheorem{theorem}{Theorem}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{remark}[theorem]{Remark}

%\newtheoremstyle{myexercise}{2pt}{2pt}{\footnotesize\normalfont}{0pt}{\footnotesize\scshape}{.}{.5em}{}
\newtheoremstyle{myexercise}{20pt}{20pt}{\footnotesize\normalfont}{0pt}{\bf\footnotesize\scshape}{.}{.5em}{}
\theoremstyle{myexercise}
\newtheorem{exercise}{Exercise}[section]

\newcommand{\rem}[1]{%\medskip\setlength{\leftskip}{15pt}\setlength{\rightskip}{10pt}
\begin{spacing}{0.85}
{\footnotesize #1}
\end{spacing}
\vspace{-1.5mm}
%\setlength{\leftskip}{0pt}\setlength{\rightskip}{0pt}
}


\makeatletter
\renewcommand{\section}{\@startsection {section}{1}{\z@}%
                                   {-3.5ex \@plus -1ex \@minus -.2ex}%
                                   {2.3ex \@plus.2ex}%
                                   {\normalfont\large\bfseries}}
\renewcommand{\subsection}[1]{\@startsection{subsection}{2}{\z@}%
    {-2.5ex\@plus -1ex \@minus -.2ex}%
    {-.0em}
    {\reset@font\bfseries}{\!\!#1}{$ $}}
\makeatother

\def\proof{\bigskip\noindent{\em Proof. }}
\def\qed{\hfill$\square$}
\def\ueq{\stackrel{\ast}{\sim}}
\def\oeq{\stackrel{\rm o}{\sim}}
\def\new{\bigskip\noindent}
\def\note{$\circ\ $}
\def\bull{$\bullet$}
\def\IR{{\mathbb R}}
\def\IC{{\mathbb C}}
\def\IF{{\mathbb F}}
\def\IP{{\mathbb P}}
\def\IZ{{\mathbb Z}}
\def\la{{\langle}}
\def\ra{{\rangle}}
\def\cO{{\mathcal O}}
\def\i{{\mathbf i}}
\def\bu{{\mathbf u}}

\def\beq{\begin{equation}}
\def\eeq{\end{equation}}

\renewcommand{\theequation}{\arabic{section}.\arabic{equation}}


\begin{document}

\pagenumbering{roman}

\thispagestyle{empty}

\centerline{University of Alabama at Birmingham} \centerline{Department
of Mathematics}\vspace{7cm}

%\noindent\rule{14cm}{0.8pt}\vspace{5cm}

\centerline{\Large\bf Numerical Linear Algebra}\vspace{2cm}


\centerline{\large Lecture Notes for MA 660}
\centerline{(1997--2014)}\vspace{1cm}

\centerline{Dr Nikolai Chernov}\vspace{5.7cm}

\centerline{Summer 2014}

\newpage


{\footnotesize \setcounter{tocdepth}{2}

\tableofcontents }

\newpage
\pagenumbering{arabic} \setcounter{page}{1}

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\setcounter{section}{-1} \refstepcounter{section} \label{Ch0}

\centerline{Chapter \ref{Ch0}}\vspace{5mm}

\centerline{\Large\bf Review of Linear Algebra}\vspace{3mm}

\addcontentsline{toc}{section}{\ref{Ch0}. Review of Linear Algebra}

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\setcounter{subsection}{0} \setcounter{exercise}{0}
\setcounter{equation}{0}

\subsection{Matrices and vectors}

The set of $m\times n$ matrices ($m$ rows, $n$ columns) with entries in
a field $\IF$ is denoted by $\IF^{m\times n}$. We will only consider
two fields: complex ($\IF=\IC$) and real ($\IF=\IR$). For any matrix
$A\in\IF^{m\times n}$, we denote its entries by
$$
      A =
   \begin{bmatrix} a_{11} & a_{12} & \cdots & a_{1n} \\
                   a_{21} & a_{22} & \cdots & a_{2n} \\
                   \vdots & \vdots & \ddots & \vdots \\
                   a_{m1} & a_{m2} & \cdots & a_{mn}
   \end{bmatrix}.
$$
The vector space $\IF^n$ consists of column vectors with $n$
components:
$$
     x=\begin{bmatrix} x_1\\ \vdots \\ x_n \end{bmatrix}
     \in \IF^n.
$$
Note: all vectors, by default, are \emph{column} vectors {\footnotesize
(not row vectors)}.

\subsection{Product of a matrix and a vector}

The product $y=Ax$ is a vector in $\IF^m$:
$$
      y= Ax =
   \left[\!\!\!
   \begin{array}{c} \phantom{\vdots} \\ a_1 \\ \phantom{\vdots} \end{array}
   \!\!\!\right | \left . \!\!\!\!
   \begin{array}{c} \phantom{\vdots} \\ a_2 \\ \phantom{\vdots} \end{array}
   \!\!\!\right | \left . \!\!\!\!
   \begin{array}{c} \phantom{\vdots} \\ \cdots \\ \phantom{\vdots} \end{array}
   \!\!\!\right | \left . \!\!\!\!
   \begin{array}{c} \phantom{\vdots} \\ a_n \\ \phantom{\vdots} \end{array}
   \!\!\!\right ]
   \begin{bmatrix} x_1\\ \vdots \\ x_n \end{bmatrix}
   =x_1\!\!\begin{bmatrix}
   \phantom{\vdots} \\ a_1 \\ \phantom{\vdots}
   \!\!\end{bmatrix}
   +x_2\!\!\begin{bmatrix}
   \phantom{\vdots} \\ a_2 \\ \phantom{\vdots}
   \!\!\end{bmatrix}
   +\cdots+x_n\!\!\begin{bmatrix}
   \phantom{\vdots} \\ a_n \\ \phantom{\vdots}
   \!\!\end{bmatrix}
$$
where $a_1,\ldots,a_n$ denote the columns of the matrix $A$.

Note: $Ax$ is a linear combination of the columns of $A$. Thus
$$
  Ax \in \,\text{span}\{a_1,\ldots,a_n\}.
$$

\subsection{Matrix as a linear transformation}

Every matrix $A\in\IF^{m\times n}$ defines a linear transformation
$$
   \IF^n\to\IF^m\qquad{\rm by}\qquad
   x\mapsto Ax.
$$
We will denote that transformation by the same letter, $A$.

\subsection{Range and rank of a matrix}

The {\em range} of $A\in\IF^{m\times n}$ is a vector subspace of
$\IF^m$ defined by
$$
   {\rm Range}\, A = \{Ax\colon x\in\IF^n\} \subset \IF^m.
$$
The range of $A$ is a subspace of $\IF^m$ spanned by the columns of
$A$:
$$
   {\rm Range}\, A = \,\text{span}\{a_1,\ldots,a_n\}.
$$
The {\em rank} of $A$ is defined by
$$
   {\rm rank}\, A = {\rm dim}({\rm Range}\, A).
$$

\subsection{Kernel (nullspace) of a matrix}

The {\em kernel} (also called {\em nullspace}) of $A\in\IF^{m\times n}$
is a vector subspace of $\IF^n$:
$$
   {\rm Ker}\, A = \{x\colon Ax=0\} \subset \IF^n.
$$
We have the following \textbf{matrix rank formula}:
\beq  \label{RKA}
       {\rm rank}\, A =
       {\rm dim}({\rm Range}\, A) =
       n-\,{\rm dim}({\rm Ker}\, A).
\eeq

\rem{An intuitive explanation: $n$ is the total number of dimensions
in the space $\IF^n$ to which the matrix $A$ is applied. The kernel
collapses to zero, thus ${\rm dim}({\rm Ker}\, A)$ dimensions
disappear. The rest survive, are carried over to $\IF^m$, and make
the range of $A$.}

\subsection{Surjective/injective/bijective transformations}
\label{SurjectInject}

The transformation from $\IF^n$ to $\IF^m$ defined by a matrix $A$
is\smallskip

\noindent $-$ \emph{surjective} iff ${\rm Range}\,A=\IF^m$. This is
only possible if $n\geq m$.

\noindent $-$ \emph{injective} iff ${\rm Ker}\,A=\{0\}$. This is only
possible if $n\leq m$.

\noindent $-$ \emph{bijective} iff it is surjective and
injective.\smallskip

In the last case, $m=n$ and we call $A$ an {\em isomorphism}.

\subsection{Square matrices and inverse of a matrix} \label{sqmat}

Every square matrix $A\in\IF^{n\times n}$ defines a linear
transformation $\IF^n\to\IF^n$, called an {\em operator} on $\IF^n$.
The {\em inverse} of a square matrix $A\in\IF^{n\times n}$ is a square
matrix $A^{-1}\in\IF^{n\times n}$ uniquely defined by
$$
    A^{-1}A=AA^{-1}=I \qquad (\text{identity matrix}).
$$
A matrix $A\in\IF^{n\times n}$ is said to be {\em invertible} ({\em
nonsingular}) iff $A^{-1}$ exists.\\ The matrix is {\em noninvertible}
({\em singular}) iff $A^{-1}$ does not exist.\smallskip

The following are equivalent:\\

(a) $A$ is invertible

(b) rank$\,A=n$

(c) Range$\,A=\IF^n$

(d) Ker$\, A=\{0\}$

(e) $0$ is not an eigenvalue of $A$

(f) $\det A\neq 0$\\

\noindent Note that
$$
      (AB)^{-1} = B^{-1}A^{-1}
$$

\subsection{Upper and lower triangular matrices}
\label{UpperLowerTriang}

A square matrix $A\in\IF^{n\times n}$ is {\em upper triangular} if
$a_{ij}=0$ for all $i>j$.

A square matrix $A\in\IF^{n\times n}$ is {\em lower triangular} if
$a_{ij}=0$ for all $i<j$.

A square matrix $D$ is {\em diagonal} if $d_{ij}=0$ for all $i\neq
j$. In that case we write $D=\,{\rm diag} \{d_{11}, \ldots,
d_{nn}\}$.

Note: if $A$ and $B$ are upper (lower) triangular, then so are their
product $AB$ and inverses $A^{-1}$ and $B^{-1}$ (if the inverses
exist).

\begin{center}
\begin{tikzpicture}
  \node[anchor=west] at (-2.7,0.8) {\footnotesize upper};
  \node[anchor=west] at (-3,0.5) {\footnotesize triangular};
  \node[anchor=west] at (-2.95,0.2) {\footnotesize matrices:};
  \fill[gray!30] (0,0.9)--(0,1)--(1,1)--(1,0)--(0.9,0)--cycle;
  \draw[thick] (0,0) -- (0,1) -- (1,1) -- (1,0) -- cycle;
  \node[anchor=south west] at (0,0) {$0$};
  \node[anchor=west] at (0.95,0.5) {$\times$};
  \begin{scope}[xshift=1.5cm]
  \fill[gray!30] (0,0.9)--(0,1)--(1,1)--(1,0)--(0.9,0)--cycle;
  \draw[thick] (0,0) -- (0,1) -- (1,1) -- (1,0) -- cycle;
  \node[anchor=south west] at (0,0) {$0$};
  \node[anchor=west] at (0.95,0.5) {$=$};
  \end{scope}
  \begin{scope}[xshift=3cm]
  \fill[gray!30] (0,0.9)--(0,1)--(1,1)--(1,0)--(0.9,0)--cycle;
  \draw[thick] (0,0) -- (0,1) -- (1,1) -- (1,0) -- cycle;
  \node[anchor=south west] at (0,0) {$0$};
  \end{scope}
  \begin{scope}[xshift=6cm]
  \fill[gray!30] (0,0.9)--(0,1)--(1,1)--(1,0)--(0.9,0)--cycle;
  \draw[thick] (0,0) -- (0,1) -- (1,1) -- (1,0) -- cycle;
  \node[anchor=south west] at (0,0) {$0$};
  \node[anchor=west] at (0.94,1) {\scriptsize -1};
  \node[anchor=west] at (1.07,0.5) {$=$};
  \end{scope}
  \begin{scope}[xshift=7.62cm]
  \fill[gray!30] (0,0.9)--(0,1)--(1,1)--(1,0)--(0.9,0)--cycle;
  \draw[thick] (0,0) -- (0,1) -- (1,1) -- (1,0) -- cycle;
  \node[anchor=south west] at (0,0) {$0$};
  \end{scope}
\end{tikzpicture}
\end{center}


\begin{center}
\begin{tikzpicture}
  \node[anchor=west] at (-2.7,0.8) {\footnotesize lower};
  \node[anchor=west] at (-3,0.5) {\footnotesize triangular};
  \node[anchor=west] at (-2.95,0.2) {\footnotesize matrices:};
  \fill[gray!30] (0,0)--(0,1)--(0.1,1)--(1,0.1)--(1,0)--cycle;
  \draw[thick] (0,0) -- (0,1) -- (1,1) -- (1,0) -- cycle;
  \node[anchor=north east] at (1,1) {$0$};
  \node[anchor=west] at (0.95,0.5) {$\times$};
  \begin{scope}[xshift=1.5cm]
  \fill[gray!30] (0,0)--(0,1)--(0.1,1)--(1,0.1)--(1,0)--cycle;
  \draw[thick] (0,0) -- (0,1) -- (1,1) -- (1,0) -- cycle;
  \node[anchor=north east] at (1,1) {$0$};
  \node[anchor=west] at (0.95,0.5) {$=$};
  \end{scope}
  \begin{scope}[xshift=3cm]
  \fill[gray!30] (0,0)--(0,1)--(0.1,1)--(1,0.1)--(1,0)--cycle;
  \draw[thick] (0,0) -- (0,1) -- (1,1) -- (1,0) -- cycle;
  \node[anchor=north east] at (1,1) {$0$};
  \end{scope}
  \begin{scope}[xshift=6cm]
  \fill[gray!30] (0,0)--(0,1)--(0.1,1)--(1,0.1)--(1,0)--cycle;
  \draw[thick] (0,0) -- (0,1) -- (1,1) -- (1,0) -- cycle;
  \node[anchor=north east] at (1,1) {$0$};
  \node[anchor=west] at (0.94,1) {\scriptsize -1};
  \node[anchor=west] at (1.07,0.5) {$=$};
  \end{scope}
  \begin{scope}[xshift=7.62cm]
  \fill[gray!30] (0,0)--(0,1)--(0.1,1)--(1,0.1)--(1,0)--cycle;
  \draw[thick] (0,0) -- (0,1) -- (1,1) -- (1,0) -- cycle;
  \node[anchor=north east] at (1,1) {$0$};
  \end{scope}
\end{tikzpicture}
\end{center}

\subsection{Eigenvalues and eigenvectors}

We say that $\lambda\in\IF$ is an {\em eigenvalue} for a square matrix
$A\in\IF^{n\times n}$ with an {\em eigenvector} $x\in\IF^n$ if
$$
      Ax=\lambda x\qquad\text{and}\qquad x\neq 0.
$$
Eigenvalues are the roots of the characteristic polynomial
$$
    C_A(\lambda)=\det (\lambda I-A)=0.
$$
By the fundamental theorem of algebra, every polynomial of degree $n$
with complex coefficients has exactly $n$ complex roots
$\lambda_1,\ldots,\lambda_n$ (counting multiplicities). This implies
$$
   C_A(\lambda) = \prod_{i=1}^n (\lambda - \lambda_i).
$$

\subsection{Eigenvalues of real matrices} \label{RealCompMat}

A real-valued square matrix $A\in\IR^{n\times n}$ can be treated in
\emph{two ways}:

First, we can treat it as a complex matrix $A\in\IC^{n\times n}$ (we
can do that because real numbers make a subset of the complex plane
$\IC$). Then $A$ has $n$ eigenvalues (real or non-real complex). An
important note: since the characteristic polynomial $C_A(\lambda)$ has
real coefficients, its non-real complex roots (i.e., non-real complex
eigenvalues of $A$) come in conjugate pairs. Thus the eigenvalues of
$A$ can be listed as follows:
$$
   c_1,\ldots,c_p,\ \ a_1\pm\i b_1,\ldots,a_q\pm\i b_q
$$
where $c_i$ ($i=1,\ldots,p$) are all \emph{real} eigenvalues and
$a_j\pm \i b_j$ ($j=1,\ldots,q$) are all non-real complex eigenvalues
(here $a_j$ and $b_j\neq 0$ are real numbers); and we denote
$\i=\sqrt{-1}$. Note that $p+2q=n$.

Second, we can treat $A$ as a matrix over the real field $\IR$, then we
can only use real numbers. In that case the eigenvalues of $A$ are only
$$
   c_1,\ldots,c_p
$$
(from the previous list). Thus $A$ may have fewer than $n$ eigenvalues,
and it may have none at all. For instance, $A=\begin{bmatrix*}[r]0 & -1\\
1 & 0\end{bmatrix*}$ has no (real) eigenvalues. However, if $A$ does
have real eigenvalues $c_1,\ldots,c_p$, then for each eigenvalue $c_i$
($i=1,\ldots,p$) there is a real eigenvector, i.e., $Ax=c_ix$ for some
$x\in\IR^n$.

\subsection{Diagonalizable matrices} \label{EVDecomp}

A square matrix $A\in\IF^{n\times n}$ is {\em diagonalizable} (over
$\IF$) iff
 \beq  \label{EDec}
         A = X\Lambda X^{-1},
 \eeq
where $\Lambda =\,\rm{diag}\{\lambda_1,\ldots,\lambda_n\}$ is a
diagonal matrix and $X\in\IF^{n\times n}$. In this case
$\lambda_1,\ldots,\lambda_n$ are the eigenvalues of $A$ and the columns
$x_1,\ldots,x_n$ of the matrix $X$ are the corresponding eigenvectors.

Indeed, we can rewrite \eqref{EDec} in the form $AX=X\Lambda$ and note
that
$$
     AX = \left[\!\!\!
   \begin{array}{c} \phantom{\lambda_1} \\ \phantom{\ddots} \\ \phantom{\lambda} \end{array}
   \begin{array}{c} \phantom{\lambda} \\ A \\ \phantom{\lambda} \end{array}
   \begin{array}{c} \phantom{\lambda} \\ \phantom{\ddots} \\ \phantom{\lambda_n} \end{array}
   \!\!\!\right ]
      \left[\!\!\!
   \begin{array}{c} \phantom{\vdots} \\ x_1 \\ \phantom{\vdots} \end{array}
   \!\!\!\right | \left . \!\!\!\!
   \begin{array}{c} \phantom{\vdots} \\ x_2 \\ \phantom{\vdots} \end{array}
   \!\!\!\right | \left . \!\!\!\!
   \begin{array}{c} \phantom{\vdots} \\ \cdots \\ \phantom{\vdots} \end{array}
   \!\!\!\right | \left . \!\!\!\!
   \begin{array}{c} \phantom{\vdots} \\ x_n \\ \phantom{\vdots} \end{array}
   \!\!\!\right ]
   =
      \left[\!\!\!
   \begin{array}{c} \phantom{\vdots} \\ Ax_1 \\ \phantom{\vdots} \end{array}
   \!\!\!\right | \left . \!\!\!\!
   \begin{array}{c} \phantom{\vdots} \\ Ax_2 \\ \phantom{\vdots} \end{array}
   \!\!\!\right | \left . \!\!\!\!
   \begin{array}{c} \phantom{\vdots} \\ \cdots \\ \phantom{\vdots} \end{array}
   \!\!\!\right | \left . \!\!\!\!
   \begin{array}{c} \phantom{\vdots} \\ Ax_n \\ \phantom{\vdots} \end{array}
   \!\!\!\right ]
$$
and
$$
   X\Lambda  =
      \left[\!\!\!
   \begin{array}{c} \phantom{\vdots} \\ x_1 \\ \phantom{\vdots} \end{array}
   \!\!\!\right | \left . \!\!\!\!
   \begin{array}{c} \phantom{\vdots} \\ x_2 \\ \phantom{\vdots} \end{array}
   \!\!\!\right | \left . \!\!\!\!
   \begin{array}{c} \phantom{\vdots} \\ \cdots \\ \phantom{\vdots} \end{array}
   \!\!\!\right | \left . \!\!\!\!
   \begin{array}{c} \phantom{\vdots} \\ x_n \\ \phantom{\vdots} \end{array}
   \!\!\!\right ]
      \left[\!\!\!
   \begin{array}{c} \lambda_1 \\ \phantom{\ddots} \\ 0 \end{array}
   \begin{array}{c} \phantom{\lambda} \\ \ddots \\ \phantom{\lambda} \end{array}
   \begin{array}{c} 0 \\ \phantom{\ddots} \\ \lambda_n \end{array}
   \!\!\!\right ]
   =
      \left[\!\!\!
   \begin{array}{c} \phantom{\vdots} \\ \lambda_1x_1 \\ \phantom{\vdots} \end{array}
   \!\!\!\right | \left . \!\!\!\!
   \begin{array}{c} \phantom{\vdots} \\ \lambda_2x_2 \\ \phantom{\vdots} \end{array}
   \!\!\!\right | \left . \!\!\!\!
   \begin{array}{c} \phantom{\vdots} \\ \cdots \\ \phantom{\vdots} \end{array}
   \!\!\!\right | \left . \!\!\!\!
   \begin{array}{c} \phantom{\vdots} \\ \lambda_nx_n \\ \phantom{\vdots} \end{array}
   \!\!\!\right ],
$$
therefore
$$
   Ax_1=\lambda_1x_1,\quad Ax_2=\lambda_2x_2,\quad\ldots\quad
   Ax_n=\lambda_nx_n.
$$
The formula \eqref{EDec} is often called the \emph{eigenvalue
decomposition} of $A$.

\subsection{Trace}

The {\em trace} of a matrix $A\in\IF^{n\times n}$ is defined by
$$
    {\rm tr}\,A = \sum_{i=1}^n a_{ii}.
$$
Trace has the following properties:

(a) tr$\,AB=\,$tr$\,BA$;

(b) if $A=X^{-1}BX$, then tr$\,A=\,$tr$\,B$;

(c) tr$\,A=\lambda_1+\cdots+\lambda_n$ (the sum of all complex
eigenvalues).

\newpage

\subsection{Transpose of a matrix}

For any matrix $A=(a_{ij})\in\IF^{m\times n}$ we denote by
$A^T=(a_{ji})\in\IF^{n\times m}$ the {\em transpose}\footnote{Another
popular notation for the transpose is $A^t$, but we will always use
$A^T$.} of $A$. Note that
$$
      (AB)^T = B^TA^T, \qquad  \bigl(A^T\bigr)^T=A.
$$
If $A$ is a square matrix, then
$$
   \det A^T= \det A,
   \qquad \bigl(A^T\bigr)^{-1}=\bigl(A^{-1}\bigr)^T.
$$
A matrix $A$ is symmetric if $A^T=A$. Only square matrices can be
symmetric.

\subsection{Conjugate transpose of a matrix}

For any matrix $A=(a_{ij})\in\IC^{m\times n}$ we denote by
$A^{\ast}=(\bar{a}_{ji})\in\IF^{n\times m}$ the {\em conjugate
transpose}\footnote{Another popular notation for the conjugate
transpose is $A^H$, but we will use $A^\ast$.} of $A$ (also called
\emph{adjoint} of $A$):
$$
   A=\begin{bmatrix} a&b\\c&d\\e&f\end{bmatrix}
   \qquad\Longrightarrow\qquad
   A^\ast=\begin{bmatrix} \bar{a}&\bar{c}&\bar{e}\\\bar{b}&\bar{d}&\bar{f}\end{bmatrix}
$$
Note that
$$
      (AB)^{\ast} = B^{\ast}A^{\ast},
      \qquad \bigl(A^{\ast}\bigr)^{\ast}=A.
$$
If $A$ is a square matrix, then
\beq \label{detAast}
   \det A^{\ast}=\,\overline{\det A},
   \qquad \bigl(A^{\ast}\bigr)^{-1}=\bigl(A^{-1}\bigr)^{\ast}.
\eeq
{\footnotesize For any real matrix $A\in\IR^{m\times n}$, we have
$A^{\ast}=A^T$. However, if $A\in\IC^{m\times n}$ is a non-real
complex matrix, then its conjugate transpose $A^\ast$ is different
from its transpose $A^T$.}

\subsection{Convention on the use of conjugate transpose}

For non-real complex matrices $A$ (and vectors $x$), just transpose
$A^T$ (respectively, $x^T$) usually does not give us anything good. For
complex matrices and vectors, we should always use \emph{conjugate}
transpose.

In our formulas we will use $A^\ast$ for both complex and real
matrices, and we will use $x^\ast$ for both complex and real vectors.
We should just keep in mind that for real matrices $A^\ast$ can be
replaced with $A^T$ and for real vectors $x^\ast$ can be replaced with
$x^T$.

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\refstepcounter{section} \label{Ch1}

\centerline{Chapter \ref{Ch1} }\vspace{5mm}

\centerline{\Large\bf Norms and Inner Products}\vspace{3mm}

\addcontentsline{toc}{section}{\ref{Ch1}. Norms and Inner Products}

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\setcounter{subsection}{0} \setcounter{exercise}{0}
\setcounter{equation}{0}

\subsection{Norms}
\label{8.1}

A {\em norm} on a vector space $V$ over $\IC$ is a real valued function
$\|\cdot \|$ on $V$ satisfying three axioms:
\begin{enumerate}
\item $\|v\|\geq 0$ for all $v\in V$ and $\|v\|=0$ if and only if
    $v=0$.\item $\|cv\|=|c|\, \|v\|$ for all $c\in\IC$ and $v\in
    V$.\item $\|u+v\|\leq \|u\|+\|v\|$ for all $u,v\in
    V$\phantom{mmm} (\textbf{triangle inequality}).\end{enumerate}

If $V$ is a vector space over $\IR$, then a norm is a function
$V\to\IR$ satisfying the same axioms, except $c\in\IR$.

{\footnotesize Norm is a general version of \emph{length} of a vector.}

\subsection{1-norm, 2-norm, and $\infty$-norm} \label{norms12inf}

Some common norms on $\IC^n$ and $\IR^n$ are:
\begin{align}
   \|x\|_1 &=\sum\nolimits_{i=1}^n |x_i|  \tag{1-norm}\\
   \|x\|_2 &=\Bigl(\sum\nolimits_{i=1}^n |x_i|^2\Bigr)^{1/2}\tag{2-norm}\\
   \|x\|_p &=\Bigl(\sum\nolimits_{i=1}^n |x_i|^p\Bigr)^{1/p} \tag{$p$-norm, $p\geq 1$}\\
   \|x\|_{\infty} &=\max_{1\leq i\leq n} |x_i| \tag{$\infty$-norm}
\end{align}
The 2-norm in $\IR^n$ corresponds to the Euclidean distance.\medskip

{\scriptsize Some norms on $C[a,b]$, the space of continuous functions
on $[a,b]$:
\begin{align}
       \|f\|_1 &=\int_a^b|f(x)|\, dx   \tag{1-norm}\\
       \|f\|_2 &=\Bigl(\int_a^b|f(x)|^2\, dx\Bigr)^{1/2} \tag{2-norm}\\
       \|f\|_p &=\Bigl(\int_a^b|f(x)|^p\, dx\Bigr)^{1/p} \tag{$p$-norm, $p\geq 1$}\\
       \|f\|_{\infty} &=\max_{a\leq x\leq b} |f(x)| \tag{$\infty$-norm}
\end{align}}


\subsection{Unit vectors, normalization} \label{UnitNormalize}

We say that $u\in V$ is a \emph{unit vector} if $\|u\|=1$.\smallskip

For any $v\neq 0$, the vector $u=v/\|v\|$ is a unit vector.\smallskip

{\footnotesize We say that we \emph{normalize} a vector $v\neq 0$ when
we transform it to $u=v/\|v\|$.}

\subsection{Unit sphere, unit ball}

The \emph{unit sphere} in $V$ is the set of all unit vectors:
 \beq \label{UnitSphere}
   {\mathbb S}_1=\{v\in V\colon\, \|v\|=1\}
 \eeq

The \emph{unit ball} in $V$ is the set of all vectors whose norm is
$\leq 1$:
 \beq \label{UnitBall}
   {\mathbb B}_1=\{v\in V\colon\, \|v\|\leq 1\}
 \eeq

The unit ball is the unit sphere with its interior.

\subsection{Images of unit spheres}

The figure shows unit spheres in $\IR^2$ for three common norms:
1-norm, 2-norm (the Euclidean norm), and $\infty$-norm.

\begin{center}
\begin{tikzpicture}
         \draw[black!60,->,-latex] (-1.3,0) -- (1.3,0) node[above]{$x$};
         \draw[black!60,->,-latex] (0,-1.3) -- (0,1.3) node[right]{$y$};
         \draw[thick] (-1,0) -- (0,1) -- (1,0) -- (0,-1) -- cycle;
         \node[anchor=north] at (0,-1.3) {1-norm};
         \draw[black!60,->,-latex,xshift=4cm] (-1.3,0) -- (1.3,0) node[above]{$x$};
         \draw[black!60,->,-latex,xshift=4cm] (0,-1.3) -- (0,1.3) node[right]{$y$};
         \draw[thick,xshift=4cm] (0,0) circle(1);
         \node[anchor=north,xshift=4cm] at (0,-1.3) {2-norm};
         \draw[black!60,->,-latex,xshift=8cm] (-1.3,0) -- (1.3,0) node[above]{$x$};
         \draw[black!60,->,-latex,xshift=8cm] (0,-1.3) -- (0,1.3) node[right]{$y$};
         \draw[thick,xshift=8cm] (-1,-1) -- (-1,1) -- (1,1) -- (1,-1) -- cycle;
         \node[anchor=north,xshift=8cm] at (0,-1.3) {$\infty$-norm};
\end{tikzpicture}
\end{center}

\subsection{Frobenius norm}  \label{FrobNorm}

The space $\IC^{m\times n}$ of matrices can be naturally identified
with $\IC^{mn}$. Hence we can define norms on it in a similar way. In
particular, an analogue of the 2-norm is
$$
   \|A\|_{\rm F}=\biggl(\sum_i\sum_j |a_{ij}|^2\biggr)^{1/2}.
$$
It is called {\em Frobenius norm} of a matrix. It is easy to verify
that
$$
   \|A\|_F^2=\,{\rm tr}\,(A^{\ast}A)=\,{\rm tr}\,(AA^{\ast}).
$$


\subsection{Induced matrix norms} \label{8.8}

Recall that every matrix $A\in \IF^{m\times n}$ defines a linear
transformation $\IF^n\to\IF^m$. Suppose the spaces $\IF^n$ and $\IF^m$
are equipped with certain norms, $\|\cdot\|$. Then we define the {\em
induced norm} (also called {\em operator norm}) of $A$ as follows:
 \beq \label{IndMatNorm}
    \|A\|
    =\sup_{\|x\|=1} \|Ax\|
    =\sup_{x\neq 0}\frac{\|Ax\|}{\|x\|}
 \eeq
{\footnotesize This norm is the maximum expansion factor, i.e., the
largest factor by which the matrix $A$ stretches vectors $x\in \IF^n$.}

Respectively, we obtain $\|A\|_1$, $\|A\|_2$, and $\|A\|_{\infty}$, if
the spaces $\IF^n$ and $\IF^m$ are equipped with $\|\cdot \|_1$,
$\|\cdot \|_2$, and $\|\cdot \|_{\infty}$. \vspace{5mm}

{\footnotesize For students familiar with topology: The supremum in
\eqref{IndMatNorm} is always attained and can be replaced by maximum.
This follows from the compactness of ${\mathbb S}_1$ and the continuity
of $\|\cdot\|$. For the 2-norm, this can be also proved by an algebraic
argument, see Corollary~\ref{12.15aa}.\medskip

There are norms on $\IC^{n\times n}$ that are not induced by any
norm on $\IC^n$, for example $\|A\|=\max_{i,j}|a_{ij}|$ (see
Exercise~\ref{ExAnormNotVnorm}).}


\subsection{Computation of $\|A\|_1$, $\|A\|_2$, and $\|A\|_{\infty}$}

We have simple rules for computing $\|A\|_1$ and $\|A\|_{\infty}$:
\begin{align}
  \|A\|_1&=\max_{1\leq j\leq n}\sum_{i=1}^m|a_{ij}|
     \tag{``maximum column sum''} \\
  \|A\|_{\infty}&=\max_{1\leq i\leq m}\sum_{j=1}^n|a_{ij}|
   \tag{``maximum row sum''}
\end{align}
Unfortunately, there is no explicit formulas for $\|A\|_2$ in terms of
the $a_{ij}$'s. A substantial part of this course will be devoted to
the computation of $\|A\|_2$.

\subsection{Inequalities for induced matrix norms}
\label{InducedInequalities}

Any induced matrix norm satisfies
$$
   \|Ax\|\leq \|A\|\, \|x\| \qquad {\rm and}\qquad
   \|AB\|\leq \|A\|\, \|B\|.
$$

\newpage

\subsection{Inner products}\label{InnProd}

Let $V$ be a vector space over $\IC$. An {\em inner product} on $V$ is
a complex valued function of two vector arguments, i.e., $V\times
V\to\IC$, denoted by $\la\cdot,\cdot\ra$, satisfying four axioms:
\begin{enumerate}\item $\la u,v\ra=\overline{\la v,u\ra}$ for
all $u,v\in V$. \item $\la cu,v\ra=c\la u,v\ra$ for all $c\in\IC$
and $u,v\in V$. \item $\la u+v,w\ra=\la u,w\ra+\la v,w\ra$ for all
$u,v,w\in V$. \item $\la u,u\ra\geq 0$ for all $u\in V$, and $\la
u,u\ra=0$ iff $u=0$.
\end{enumerate}
These axioms imply the following rules:
 \beq \label{IP2}
  \la u,v+w\ra=\la u,v\ra+\la u,w\ra
  \qquad\text{and}\qquad
  \la u,cv\ra = \bar{c}\la u,v\ra.
 \eeq
i.e., the inner product is linear in the first argument (due to axioms
2 and 3) but ``conjugate linear'' in the second.

If $V$ is a vector space over $\IR$, then an inner product is a real
valued function of two arguments, i.e., $V\times V\to\IR$ satisfying
the same axioms (except $c\in\IR$, and there is no need to take
conjugate).

\subsection{Standard inner product} \label{StIP}

A standard (canonical) inner product in $\IC^n$ is
$$
   \la x,y\ra=\sum_{i=1}^n x_i\bar{y}_i
   =y^{\ast}x.
$$
A standard (canonical) inner product in $\IR^n$ is
$$
   \la x,y\ra=\sum_{i=1}^n x_i y_i
   =y^Tx.
$$
A standard inner product in $C([a,b])$, the space of complex-valued
continuous functions, is:
$$
   \la f,g\ra = \int_a^b f(x)\,\bar{g}(x)\,dx
$$
For real-valued functions, we can replace $\bar{g}(x)$ with $g(x)$.

\subsection{Cauchy-Schwarz inequality}
\label{8.16}

Let $V$ be an inner product space. Then for any two vectors $u,v\in V$
 \beq \label{CSI}
   |\la u,v\ra |\leq \la u,u\ra^{1/2}\la v,v\ra^{1/2}
 \eeq
The equality holds if and only if $u$ and $v$ are linearly dependent,
i.e.,
 \beq \label{CSI=}
   |\la u,v\ra | = \la u,u\ra^{1/2}\la v,v\ra^{1/2}
   \qquad\Longleftrightarrow\qquad
   au+bv=0
 \eeq
for some scalars $a,b$ of which at least one is different from zero.


\proof The easy case is where $v=0$. Then $\la u,v\ra=0$ and $\la
v,v\ra=0$, so both sides of \eqref{CSI} are zero, hence the claim is
trivial.

We now turn to the difficult case: $v\neq 0$. Consider the function
\begin{align*}
   f(z)&= \la u-zv,u-zv\ra \\
   &= \la u,u\ra
   - z\la v,u\ra
   - \bar{z}\la u,v\ra
   + |z|^2\la v,v\ra
\end{align*}
of a complex variable $z$. Let $z=re^{\i\theta}$ and $\la
u,v\ra=se^{\i\varphi}$ be the polar forms of the complex numbers $z$
and $\la u,v\ra$, respectively. We fix $\theta=\varphi$ and note that
$$
   z\la v,u\ra = re^{\i\varphi}se^{-\i\varphi}=rs
$$
and
$$
  \bar{z}\la u,v\ra = re^{-\i\varphi}se^{\i\varphi} = rs.
$$
We assume that $r$ varies from $-\infty$ to $\infty$, then
 \beq \label{Poly2}
   0\leq f(z)= \la u,u\ra
   - 2sr + r^2\la v,v\ra
 \eeq
Since this holds for all $r\in\IR$, the discriminant of the above
quadratic polynomial has to be $\leq 0$, i.e.
 \beq \label{CSI1}
   s^2- \la u,u\ra\la
   v,v\ra\leq 0.
 \eeq
This proves the inequality \eqref{CSI}.

The equality in \eqref{CSI} means the equality in \eqref{CSI1}, because
$s = |\la u,v\ra|$. The equality in \eqref{CSI1} means that the
discriminant of the quadratic polynomial in \eqref{Poly2} is zero,
hence that polynomial assumes a zero value, i.e.,
$$
  f(z) = \la u-zv,u-zv\ra = 0
$$
for some $z\in\IC$. This, in turn, means $u-zv=0$, thus $u=zv$ for some
$z\in\IC$, i.e., $u$ and $v$ are linearly dependent, as claimed. \hfill
\qed

\subsection{Induced norms} \label{8.17}

If $V$ is an inner product vector space, then
 \beq
   \|v\|=\la v,v\ra^{1/2}
 \eeq
defines a norm on $V$ (all the axioms can be verified by direct
inspection, except the triangle inequality is proved by Cauchy-Schwarz
inequality~\ref{8.16}).

\subsection{Polarization identity} \label{PolId}

Some norms are induced by inner products, others are not. The 2-norm in
$\IR^n$ and in $\IC^n$ (defined in Sect.~\ref{norms12inf}) is induced
by the standard inner product in those spaces (as defined in
Sect.~\ref{StIP}).

In vector spaces over $\IR$, if a norm $\|\cdot\|$ is induced by an
inner product $\la\cdot,\cdot\ra$, then the latter can be computed by
\emph{polarization identity}:
 \beq \label{PolarId}
  \la u,v\ra = \tfrac 14
   \left( \|u+v\|^2-\|u-v\|^2\right)
 \eeq

{\footnotesize A similar but more complicated polarization identity
holds in vector spaces over $\IC$.}

\subsection{Orthogonal vectors} \label{9.1}

Two vectors $u,v\in V$ are said to be {\em orthogonal} if $\la u,v\ra
=0$.

\subsection{Pythagorean theorem} \label{Pythagoras}

If two vectors $u$ and $v$ are orthogonal, then
 \beq \label{PythTm}
      \|u+v\|^2=\|u\|^2+\|v\|^2
 \eeq
{\footnotesize Vectors $u$ and $v$ make two legs of a right triangle,
and $u+v$ is its hypotenuse: \begin{tikzpicture}[baseline=0.2ex]
\draw[->,-latex] (0,0)--(1,0); \node[anchor=north] at (0.5,0) {\tiny
$u$}; \draw[->,-latex] (0,0)--(1,0.5); \node[anchor=south,rotate=28] at
(0.5,0.2) {\tiny $u+v$}; \draw[->,-latex] (1,0)--(1,0.5);
\node[anchor=west] at (1,0.25) {\tiny $v$}; \end{tikzpicture}

Suppose $u_1,\ldots,u_k$ are mutually orthogonal, i.e., $\la
u_i,u_j\ra=0$ for each $i\neq j$. Then one can show, by induction, that
$\|u_1+\cdots +u_k\|^2=\|u_1\|^2+\cdots +\|u_k\|^2$.}

\subsection{Orthogonality and linear independence} \label{OrtLI}

If nonzero vectors $u_1,\ldots,u_k$ are mutually orthogonal, i.e.,
$\la u_i,u_j\ra=0$ for each $i\neq j$, then they are linearly
independent.\medskip

{\footnotesize \emph{Proof}. Suppose a linear combination of these
vectors is zero, i.e.,
$$
   c_1u_1+\cdots+c_ku_k=0.
$$
Then for every $i=1,\ldots,k$ we have
$$
  0=\la 0,u_i\ra = \la c_1u_1+\cdots+c_ku_k,u_i\ra = c_i\la u_i,u_i\ra
$$
hence $c_i=0$ (because $\la u_i,u_i\ra \neq 0$ for the non-zero
vector $u_i$). Thus $c_1=\cdots=c_k=0$. \qed}

\subsection{Orthonormal sets of vectors} \label{Orthonormalsets}

A set $\{u_1,\ldots,u_k\}$ of vectors is said to be \emph{orthonormal}
if all the vectors $u_i$ are mutually orthogonal and have unit length
(i.e., $\|u_i\|=1$ for all $i$).

The orthonormality can be expressed by a single formula:
$$
   \la u_i,u_j\ra = \delta_{ij},
$$
where $\delta_{ij}$ denotes the \emph{Kronecker delta symbol} defined
as follows: $\delta_{ij}=1$ if $i=j$ and $\delta_{ij}=0$ if $i\neq j$.

\subsection{Orthonormal basis (ONB)}

An orthonormal set of vectors $\{u_1,\ldots,u_k\}$ that is a basis in
$V$ is called {\em orthonormal basis} (ONB).

An orthonormal set $\{u_1,\ldots,u_k\}$ is an ONB in $V$ iff
$k=n=\,$dim$\,V$.

\subsection{Fourier expansion}

If $\{u_1,\ldots,u_n\}$ is an ONB, then for any vector $v$ we have
 \beq  \label{Fourier}
   v=\sum_{i=1}^n\la v,u_i\ra\, u_i.
 \eeq
In other words, the numbers $\la v,u_i\ra$ (called {\em Fourier
coefficients}) are the coordinates of the vector $v$ in the basis
$\{u_1,\ldots,u_n\}$.

\subsection{Orthogonal projection} \label{9.7}

Let $u,v\in V$, and $u\neq 0$. The {\em orthogonal projection} of $v$
onto $u$ is
$$
    {\rm Pr}_uv=\frac{\la v,u\ra}{\|u\|^2}\,u.
$$
One can easily check that the vector $w = v-{\rm Pr}_uv$ is
orthogonal to $u$. Thus $v$ is the sum of two vectors: Pr$_uv$
(parallel to $u$) and $w$ (orthogonal to $u$).

\begin{center}
\begin{tikzpicture}
         \draw[->,-latex] (0,0) -- (5,0);
         \node[anchor=south east] at (5,0) {$u$};
         \draw[very thick,->,-latex] (0,0) -- (3,2);
         \node[anchor=south,rotate=30] at (1.5,1) {$v$};
         \draw[very thick,->,-latex] (3,0) -- (3,2);
         \draw[ultra thick,dashed,gray,->,-latex] (1.8,1.1) -- (1.8,0.1);
         \draw[ultra thick,dashed,gray,->,-latex] (1.05,0.6) -- (1.05,0.1);
         \draw[ultra thick,dashed,gray,->,-latex] (2.55,1.6) -- (2.55,0.1);
         \draw[very thick,->,-latex] (0,0) -- (3,0);
         \node[anchor=west] at (3,1) {$w$};
         \node[anchor=north east] at (2,0) {Pr$_uv$};
\end{tikzpicture}
\end{center}

\subsection{Angle between vectors} \label{Angletheta}

If $V$ is a vector space over $\IR$, then for any nonzero vectors
$u,v\in V$ there is a unique angle $\theta\in[0,\pi]$ such that
 \beq  \label{angle}
   \cos\theta=\frac{\la v,u\ra}{\|u\|\, \|v\|}.
 \eeq
{\footnotesize Indeed, by Cauchy-Schwarz inequality~\ref{8.16}, the
above fraction is a real number in the interval $[-1,1]$. Then
recall that the function $\cos^{-1}$ takes $[-1,1]$ onto $[0,\pi]$
bijectively.}

We call $\theta$ the \emph{angle} between $u$ and $v$.

{\footnotesize Note that $\cos\theta=0$ (i.e., $\theta = \pi/2$) if
and only if $u$ and $v$ are orthogonal. Also, $\cos\theta=\pm 1$ if
and only if $u,v$ are proportional, i.e.\ $v=cu$. In that case the
sign of $c$ coincides with the sign of $\cos\theta$.}

\subsection{Orthogonal projection onto a subspace} \label{OrtProjL}

Let $\{u_1,\ldots,u_k\}$ be an orthonormal set of vectors. According to
Sect.~\ref{OrtLI}, they are linearly independent, hence they span a
$k$-dimensional subspace $L=\,$span$\{u_1,\ldots,u_k\}$. For any $v\in
V$,
$$
   {\rm Pr}_Lv = \sum_{i=1}^k\la v,u_i\ra u_i
$$
is the orthogonal projection of $v$ onto $L$. One can easily check
that the vector
$$
    w=v-\,{\rm Pr}_Lv
$$
is orthogonal to all the vectors $u_1,\ldots,u_k$, hence it is
orthogonal to all vectors in $L$. In particular, the vectors
$u_1,\ldots,u_k,w$ are mutually orthogonal, and one can easily check
that
$$
   {\rm span}\{u_1,\ldots,u_k,v\}=\,
   {\rm span}\{u_1,\ldots,u_k,w\}.
$$

\subsection{Degenerate case} \label{DegCase}

In the above construction, we have
$$
   w=0\qquad\Longleftrightarrow\qquad
   v\in\,{\rm span}\{u_1, \ldots,u_k\}.
$$
\smallskip

\rem{Note: In Section~\ref{OrtProjL}, the subspace $L$ is spanned by
an \emph{orthonormal} set of vectors. What if we want to project a
vector $v$ onto a subspace $L$ that is spanned by a non-orthonormal
set of vectors $\{v_1,\ldots,v_k\}$?}

\subsection{Gram-Schmidt orthogonalization} \label{GramSchmidt}

Let $\{v_1,\ldots,v_k\}$ be a linearly independent set of vectors.
They span a $k$-dimensional subspace $L=\,$span$\{v_1,\ldots,v_k\}$.
Our goal is to find an orthonormal set of vectors
$\{u_1,\ldots,u_k\}$ that spans $L$. We will modify and adjust the
vectors $v_1,\ldots,v_k$, one by one.

At our first step, we define
\beq \label{GSstep1}
            w_1=v_1 \qquad \text{and} \qquad
            u_1=w_1/\|w_1\|.
\eeq
At our second step, we define
\beq \label{GSstep2}
            w_2=v_2-\la v_2,u_1\ra\,u_1 \qquad \text{and} \qquad
            u_2=w_2/\|w_2\|.
\eeq
Then recursively, for each $p\geq 2$, we define
\beq  \label{GS}
   w_{p}=v_{p}-\sum_{i=1}^{p-1}\la v_{p},u_i\ra\, u_i,
   \quad \text{and} \quad
   u_p=w_p/\|w_p\|
\eeq
By induction on $p$, one can easily check that
$$
   {\rm span}\{v_1,\ldots,v_p\}=\,{\rm span}\{u_1,\ldots,u_p\}
$$
and $w_p\neq 0$ (in accordance with Sect.~\ref{DegCase}).


The above procedure is called \emph{Gram-Schmidt orthogonalization}.
It gives us an orthonormal set $\{u_1,\ldots,u_k\}$ which spans the
same subspace:
$$
   L=\,{\rm span}\{v_1,\ldots,v_k\}=\,{\rm span}\{u_1,\ldots,u_k\}.
$$

\subsection{Construction of ONB} \label{ONBexist}

Suppose $\{v_1,\ldots,v_n\}$ is a basis in $V$. Then the Gram-Schmidt
orthogonalization gives us an orthonormal basis (ONB)
$\{u_1,\ldots,u_n\}$ in $V$.

As a result, every finite dimensional vector space with an inner
product has an ONB. Furthermore, every set of orthonormal vectors
$\{u_1,\ldots,u_k\}$ can be extended to an ONB.

\subsection{Legendre polynomials (optional)} \label{9.12}

{\footnotesize Let $V=\IP_n(\IR)$, the space of real polynomials of
degree $\leq n$, with the inner product given by $\la f,g\ra=\int_0^1
f(x)g(x)\, dx$. Applying Gram-Schmidt orthogonalization to the basis
$\{1,x,\ldots,x^n\}$ gives the first $n+1$ of the so called {\em
Legendre polynomials}.}

\subsection{Orthogonal complement} \label{9.14}

Let $S\subset V$ be a subset (not necessarily a subspace). Then
$$
 S^{\perp} =
 \{v\in V\colon\, \la v,w\ra =0\ \
 {\rm for}\ {\rm all}\ \ w\in S\}
$$
is called the {\em orthogonal complement} to $S$.

One can easily check that $S^{\perp}$ is a vector subspace of $V$.

We also note that if $W\subset V$ is a subspace of $V$, then
$(W^\perp)^\perp\subset W$. Moreover, if $V$ is finite dimensional,
then $(W^\perp)^\perp = W$ {\footnotesize (see
Exercise~\ref{ExOrthCompl})}.

\subsection{Orthogonal direct sum} \label{ODS}

If $W\subset V$ is a finite dimensional subspace of $V$, then
$V=W\oplus W^{\perp}$.\smallskip

\noindent{\footnotesize \emph{Proof}. Let $\{u_1,\ldots,u_n\}$ be an
ONB of $W$ (one exists according to Sect.~\ref{ONBexist}). For any
$v\in V$ define
$$
  w = \sum_{i=1}^n\la v,u_i\ra\, u_i
$$
One can easily verify that $w\in W$ and $v-w \in W^\perp$. Therefore
$v=w+w'$ where $w \in W$ and $w' \in W^\perp$. Lastly, $W\cap W^\perp =
\{0\}$ because any vector $w\in W\cap W^\perp$ is orthogonal to itself,
i.e., $\la w,w\ra = 0$, hence $w=0$.}

\subsection{Some useful formulas}

Let $\{u_1,\ldots,u_n\}$ be an orthonormal set (not necessarily an ONB)
in $V$. Then for any vector $v\in V$ we have \emph{Bessel's
inequality}:
 \beq \label{Bessel>}
    \|v\|^2\geq\sum_{i=1}^n|\la v,u_i\ra|^2.
 \eeq
If $\{u_1,\ldots,u_n\}$ is an ONB in $V$, then Bessel's inequality
turns into an equality:
 \beq \label{Bessel=}
   \|v\|^2 = \sum_{i=1}^n |\la v,u_i\ra|^2.
 \eeq
More generally, we have \emph{Parceval's identity}:
 \beq \label{Parceval}
   \la v,w\ra = \sum_{i=1}^n
   \la v,u_i\ra \overline{\la w,u_i\ra},
 \eeq
which easily follows from the Fourier expansion \eqref{Fourier}.

Parceval's identity \eqref{Parceval} can be also written as follows.
Suppose
$$
   v=\sum_{i=1}^n a_i u_i\qquad\text{and}\qquad
   w=\sum_{i=1}^n b_i u_i,
$$
so that $(a_1,\ldots,a_n)$ and $(b_1,\ldots,b_n)$ are the coordinates
of the vectors $v$ and $w$, respectively, in the ONB
$\{u_1,\ldots,u_n\}$. Then
 \beq \label{ParcevalIP}
   \la v,w\ra = \sum_{i=1}^n
   a_i\bar{b}_i.
 \eeq
In particular,
 \beq \label{ParcevalNorm}
   \|v\|^2 = \la v,v\ra = \sum_{i=1}^n
   a_i\bar{a}_i = \sum_{i=1}^n|a_i|^2.
 \eeq

\begin{exercise} \label{ExAnormNotVnorm}
Show that the norm $\|A\|=\max_{i,j}|a_{ij}|$ on the space of $n\times
n$ real matrices is not induced by any vector norm in ${\mathbb R}^n$.
Hint: use inequalities from Section~\ref{8.8}.
\end{exercise}

\begin{exercise} \label{ExNeumann}
Prove the \emph{Neumann lemma}: if $\|A\|<1$, then $I-A$ is invertible.
Here $\|\cdot \|$ is a norm on the space of $n\times n$ matrices
induced by a vector norm.
\end{exercise}

\begin{exercise}
Let $V$ be an inner product space, and $\|\cdot \|$ denote the norm
induced by the inner product. Prove the \emph{parallelogram law}
$$\|u+v\|^2+\|u-v\|^2=2\|u\|^2+2\|v\|^2.$$ Based on this, show that
the norms $\|\cdot\|_1$ and $\|\cdot\|_{\infty}$ in $\IC^2$ are not
induced by any inner products.
\end{exercise}

\begin{exercise} \label{ExOrthCompl}
Let $W\subset V$ be a subspace of an inner product space $V$.
\begin{itemize} \item[(i)]Prove that $W\subset (W^{\perp})^{\perp}$.
\item[(ii)] If, in addition, $V$ is finite dimensional, prove that
    $W = (W^{\perp})^{\perp}$.\end{itemize}
\end{exercise}

\begin{exercise}
Let $\{u_1,\ldots,u_n\}$ be an ONB in $\IC^n$. Assuming that $n$ is
even, compute $$\|u_1-u_2+u_3-\cdots +u_{n-1}-u_n\|$$
\end{exercise}

\newpage

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\refstepcounter{section} \label{Ch2}

\centerline{Chapter \ref{Ch2} }\vspace{5mm}

\centerline{\Large\bf Unitary Matrices}\vspace{3mm}

\addcontentsline{toc}{section}{\ref{Ch2}. Unitary Matrices}

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\setcounter{subsection}{0} \setcounter{exercise}{0}
\setcounter{equation}{0}

\subsection{Isometries} \label{9.20}

Let $V$ and $W$ be two inner product spaces (both real or both
complex). An isomorphism $T\colon V\to W$ is called an {\em
isometry} if it preserves the inner product, i.e. $$\la Tv,Tw\ra=\la
v,w\ra$$ for all $v,w\in V$. In this case the spaces $V$ and $W$ are
said to be {\em isometric}.

\subsection{Characterization of isometries - I} \label{CharIsom1}

An isomorphism $T$ is an isometry iff it preserves the induced norm,
i.e., $\|Tv\|=\|v\|$ for all vectors $v\in V$.\smallskip

{\footnotesize \emph{Proof}. This follows from Polarization Identity
\eqref{PolarId}}\medskip

\subsection{Characterization of isometries - II} \label{CharIsom2}

An isomorphism $T$ is an isometry iff $\|Tu\|=\|u\|$ for all
\emph{unit} vectors $u\in V$.\smallskip

{\footnotesize \emph{Proof}. This easily follows from the previous
section and the fact that every non-zero vector $v$ can be normalized
by $u=v/\|v\|$; cf.\ Sect.~\ref{UnitNormalize}.}

\subsection{Characterization of isometries - III} \label{CharIsom3}% \label{9.21}

Let dim$\, V<\infty$. A linear transformation $T\colon V\to W$ is an
isometry iff there exists an ONB $\{u_1,\ldots,u_n\}$ in $V$ such
that $\{Tu_1,\ldots,Tu_n\}$ is an ONB in $W$.\medskip


{\footnotesize \emph{Proof}. If $T$ is an isometry, then for any ONB
$\{u_1,\ldots,u_n\}$ in $V$ the set $\{Tu_1,\ldots,Tu_n\}$ is an ONB in
$W$. Now suppose $T$ takes an ONB $\{u_1,\ldots,u_n\}$ in $V$ to an ONB
$\{Tu_1,\ldots,Tu_n\}$ in $W$. Since $T$ takes a basis in $V$ into a
basis in $W$, it is an isomorphism. Now for each vector
$v=c_1u_1+\cdots+c_nu_n \in V$ we have, by linearity, $Tv =
c_1Tu_1+\cdots+c_nTu_n$, hence $v$ and $Tv$ have the same coordinates
in the respective ONBs. Now by Parceval's Identity \eqref{ParcevalNorm}
we have
$$
   \|v\|^2 = |c_1|^2+\cdots+|c_n|^2 = \|Tv\|^2
$$
Thus $T$ is an isometry according to Sect.~\ref{CharIsom1}.}

\subsection{Identification of finite-dimensional inner product spaces}
\label{IPSidentify}

Finite dimensional inner product spaces $V$ and $W$ (over the same
field) are \emph{isometric} iff $\dim V=\dim W$.

{\footnotesize (This follows from Section~\ref{CharIsom3}.)}\medskip

\noindent As a result, we can make the following useful
identifications:

\begin{itemize}
\item[\textcopyright ] All complex $n$-dimensional inner product
    spaces can be identified with $\IC^n$ equipped with the
    standard inner product $\la x,y \ra = y^\ast x$.
    \item[\textregistered] All real $n$-dimensional inner product
        spaces can be identified with $\IR^n$ equipped with the
        standard inner product $\la x,y \ra = y^T x$.
\end{itemize}

\noindent {\footnotesize These identifications allow us to focus on
the study of the standard spaces $\IC^n$ and $\IR^n$ equipped with
the standard inner product.}\smallskip

Isometries $\IC^n\to\IC^n$ and $\IR^n\to\IR^n$ are operators that, in a
standard basis $\{e_1,\ldots,e_n\}$, are given by matrices of a special
type, as defined below.

\subsection{Unitary and orthogonal matrices} \label{10.2}

A matrix $Q\in\IC^{n\times n}$ is said to be {\em unitary} if
$Q^{\ast}Q=I$, i.e., $Q^{\ast}=Q^{-1}$.\\
$\phantom{mk}$A matrix $Q\in\IR^{n\times n}$ is said to be {\em
orthogonal} if $Q^TQ=I$, i.e., $Q^T=Q^{-1}$.\medskip

\noindent One can easily verify that
$$
   Q\text{ is unitary}\Leftrightarrow
   Q^{\ast}\text{ is unitary}\Leftrightarrow
   Q^T\text{ is unitary}\Leftrightarrow
   \bar{Q}\text{ is unitary}.
$$
In the real case:
$$
   Q\text{ is orthogonal}\Leftrightarrow
   Q^T\text{ is orthogonal}.
$$

\subsection{Lemma} \label{LmxAy}

Let $A,B\in \IF^{m\times n}$ be two matrices (here $\IF=\IC$ or
$\IF=\IR$) such that $\la Ax,y\ra = \la Bx,y \ra$ for all $x\in \IF^n$
and $y\in \IF^m$. Then $A=B$.\medskip

\emph{Proof}. For any pair of canonical basis vectors $e_j\in\IF^n$ and
$e_i\in\IF^m$ we have $\la Ae_j,e_i\ra = a_{ij}$ and $\la Be_j,e_i\ra =
b_{ij}$, therefore $a_{ij} = b_{ij}$. \hfill \qed

\newpage

\subsection{Matrices of isometries} \label{10.3}

\begin{itemize}
\item[\textcopyright ] The linear transformation of $\IC^n$ defined
    by a matrix $Q\in\IC^{n\times n}$ is an isometry (preserves the standard inner
    product) iff $Q$ is unitary.\item[\textregistered] The linear
    transformation of $\IR^n$ defined by a matrix $Q\in\IR^{n\times
    n}$ is an isometry (preserves the standard inner product) iff $Q$ is orthogonal.
\end{itemize}


\noindent{\footnotesize \emph{Proof} If $Q$ is an isometry, then for
any pair of vectors $x,y$
$$
  \la x,y\ra=\la Qx,Qy\ra =(Qy)^{\ast}Qx=y^\ast Q^\ast Qx = \la Q^\ast Qx,y\ra
$$
hence $Q^\ast Q = I$ due to Lemma~\ref{LmxAy}. Conversely: if $Q^\ast Q
= I$, then
$$
  \la Qx,Qy\ra =(Qy)^{\ast}Qx=y^\ast Q^\ast Qx=y^\ast x=\la x,y\ra,
$$
hence $Q$ is an isometry. \hfill \qed}

\subsection{Group property} \label{Group}

If $Q_1,Q_2\in\IC^{n\times n}$ are unitary matrices, then so is
$Q_1Q_2$.

If $Q\in\IC^{n\times n}$ is a unitary matrix, then so is $Q^{-1}$.

\noindent{\footnotesize In terms of abstract algebra,
unitary $n\times n$ matrices make a \emph{group}, denoted by $U(n)$.\\
Similarly, orthogonal $n\times n$ matrices make a group, denoted by
$O(n)$.}


\subsection{Orthogonal matrices in 2D} \label{9.23}

$Q=\begin{bmatrix}0 & 1 \\ 1 & 0\end{bmatrix}$ defines a reflection
across the diagonal line $y=x$;\\

$Q=\left[\begin{array}{rr}
      \cos\theta & -\sin\theta \\
      \sin\theta & \cos\theta
   \end{array} \right]$
is a counterclockwise rotation by angle $\theta$.\medskip

\noindent{\footnotesize For a complete description of 2D orthogonal matrices, see Exercises~\ref{ExQ+1} and \ref{ExQ-1}.}

\begin{center}
\begin{tikzpicture}
         \begin{scope}[xshift=-0.2cm,yshift=1.2cm,red]
         \draw[fill=red] (0.4,0.4) circle(0.1);
         \draw[thick,fill=white] (0,0) circle(0.5);
         \draw[thick] (-0.25,-0.3)--(0.25,-0.3);
         \draw [thick, fill=red] (0,-0.1) circle (0.05);
         \draw[rotate=90] (0.15,0.15) ellipse (0.1 and 0.05);
         \draw[rotate=90] (0.15,-0.15) ellipse (0.1 and 0.05);
         \end{scope}
         \begin{scope}[rotate=-90,yshift=1.2cm,xshift=0.2cm,red]
         \draw[fill=red] (-0.4,0.4) circle(0.1);
         \draw[thick,fill=white] (0,0) circle(0.5);
         \draw[thick] (-0.25,-0.3)--(0.25,-0.3);
         \draw [thick, fill=red] (0,-0.1) circle (0.05);
         \draw[rotate=90] (0.15,0.15) ellipse (0.1 and 0.05);
         \draw[rotate=90] (0.15,-0.15) ellipse (0.1 and 0.05);
         \end{scope}
         \draw[->,-latex] (-1,0) -- (2,0) node[above]{$x$};
         \draw[->,-latex] (0,-1) -- (0,2) node[right]{$y$};
         \draw[dashed] (-1,-1) -- (2,2);
         \node[rotate=45,anchor=south] at (-0.5,-0.6) {\scriptsize $y=x$};
         \draw[gray,very thick,->,-latex] (1.5,1.5)--(2,1);
         \draw[gray,very thick,->,-latex] (1.5,1.5)--(1,2);
         \draw[gray,thick,->,-latex] (0.5,0.5)--(0.8,0.2);
         \draw[gray,thick,->,-latex] (0.5,0.5)--(0.2,0.8);
         \begin{scope}[xshift=8.3cm,yshift=0.4cm,red]
         \draw[fill=red] (0.4,0.4) circle(0.1);
         \draw[thick,fill=white] (0,0) circle(0.5);
         \draw[thick] (-0.25,-0.3)--(0.25,-0.3);
         \draw [thick, fill=red] (0,-0.1) circle (0.05);
         \draw[rotate=90] (0.15,0.15) ellipse (0.1 and 0.05);
         \draw[rotate=90] (0.15,-0.15) ellipse (0.1 and 0.05);
         \end{scope}
         \begin{scope}[xshift=7.304cm,yshift=1.326cm,rotate=60,red]
         \draw[fill=red] (0.4,0.4) circle(0.1);
         \draw[thick,fill=white] (0,0) circle(0.5);
         \draw[thick] (-0.25,-0.3)--(0.25,-0.3);
         \draw [thick, fill=red] (0,-0.1) circle (0.05);
         \draw[rotate=90] (0.15,0.15) ellipse (0.1 and 0.05);
         \draw[rotate=90] (0.15,-0.15) ellipse (0.1 and 0.05);
         \end{scope}
         \begin{scope}[xshift=7cm]
         \draw[->,-latex] (-1,0) -- (2,0) node[above]{$x$};
         \draw[->,-latex] (0,-1) -- (0,2) node[left]{$y$};
         \draw[dotted,gray,very thick,->,-latex] (1.5,1.5) arc (45:85:1.9);
         \draw[dotted,gray,very thick] (1.5,1.5) arc (45:25:1.9);
         \draw[dotted,gray,thick,->,-latex] (0.5,0.5) arc (45:85:0.7);
         \draw[dotted,gray,thick] (0.5,0.5) arc (45:25:0.7);
         \end{scope}
         \node[anchor=north] at (0,-1) {reflection};
         \node[anchor=north] at (7,-1) {rotation};
\end{tikzpicture}
\end{center}


\subsection{Characterizations of unitary and orthogonal matrices} \label{UnitaryColumnsRows}

A matrix $Q\in \IC^{n\times n}$ is unitary iff its columns make an ONB in $\IC^n$.

A matrix $Q\in \IC^{n\times n}$ is unitary iff its rows make an ONB in $\IC^n$.\medskip

A matrix $Q\in \IR^{n\times n}$ is orthogonal iff its columns make an ONB in $\IR^n$.

A matrix $Q\in \IR^{n\times n}$ is orthogonal iff its rows make an
ONB in $\IR^n$.\bigskip

\noindent{\footnotesize \emph{Proof}. The idea is illustrated by the
following diagram:
$$
   Q^\ast \qquad\ \times\qquad\ Q\qquad\ \ =\qquad\ \ I \qquad\ \ =\qquad\ \ Q \qquad\ \times\qquad\ Q^\ast
$$
}{\scriptsize
$$
     \left[
   \begin{array}{c} \phantom{nnn} q_1^\ast \phantom{nnn} \\ \hline q_2^\ast \\
   \hline \vdots \\ \hline q_n^\ast \end{array}
   \right ]
       \left[\!\!\!
   \begin{array}{c}  \\ \\ q_1 \\ \phantom{\vdots} \end{array}
   \!\!\!\right | \left . \!\!\!\!
   \begin{array}{c} \\ \\ q_2 \\ \phantom{\vdots} \end{array}
   \!\!\!\right | \left . \!\!\!\!
   \begin{array}{c} \\ \\ \cdots \\ \phantom{\vdots} \end{array}
   \!\!\!\right | \left . \!\!\!\!
   \begin{array}{c} \\ \\ q_n \\ \phantom{\vdots} \end{array}
   \!\!\!\right ]
   =
      \begin{bmatrix} 1 & 0 & \cdots & 0 \\
                   0 & 1 & \cdots & 0\\
                   \vdots & \vdots & \ddots & \vdots \\
                   0 & 0 & \cdots & 1
   \end{bmatrix}
       =
     \left[
   \begin{array}{c} \phantom{nnn} \tilde q_1 \phantom{nnn} \\ \hline \tilde q_2 \\
   \hline \vdots \\ \hline \tilde q_n \end{array}
   \right ]
      \left[\!\!\!
   \begin{array}{c}  \\ \\ \tilde q_1^\ast \\ \phantom{\vdots} \end{array}
   \!\!\!\right | \left . \!\!\!\!
   \begin{array}{c} \\ \\ \tilde q_2^\ast \\ \phantom{\vdots} \end{array}
   \!\!\!\right | \left . \!\!\!\!
   \begin{array}{c} \\ \\ \cdots \\ \phantom{\vdots} \end{array}
   \!\!\!\right | \left . \!\!\!\!
   \begin{array}{c} \\ \\ \tilde q_n^\ast \\ \phantom{\vdots} \end{array}
   \!\!\!\right ]
$$}
{\footnotesize Here $q_i$ denotes the $i$th column of $Q$ and
$\tilde q_i$ denotes the $i$th row of $Q$.

The proof is now a direct inspection. \qed}

\subsection{Determinant of unitary matrices} \label{10.7}

If $Q$ is unitary/orthogonal, then $|{\rm det}\, Q|=1$.


\proof We have
$$
  1=\,{\rm det}\, I=\det Q^{\ast}Q=\det
  Q^\ast\cdot \det Q=\,\overline{\det Q}
  \cdot\det Q= |\det Q|^2.
$$


\noindent {\footnotesize If $Q$ is a real orthogonal matrix, then $
\det A=\pm 1$. Orthogonal $n\times n$ matrices with determinant 1 make
a subgroup of $O(n)$, denoted by $SO(n)$.}

\subsection{Eigenvalues of unitary matrices} \label{10.9}

If $\lambda$ is an eigenvalue of a unitary/orthogonal matrix, then
$|\lambda|=1$.

\proof If $Qx=\lambda x$ for some $x\neq 0$, then
$$
  \la x,x\ra=\la
  Qx,Qx\ra = \la \lambda x, \lambda x\ra
  =\lambda\bar{\lambda}\la x,x\ra
  =|\lambda|^2\la x,x\ra;
$$
therefore $|\lambda|^2=1$. \hfill \qed\medskip

\noindent{\footnotesize Note: orthogonal matrices  $Q\in \IR^{n\times
n}$ may not have any real eigenvalues; see the rotation matrix in
Example~\ref{9.23}. But if an orthogonal matrix has real eigenvalues,
those equal $\pm 1$.}

\newpage

\subsection{Invariance principle for isometries} \label{10.10}

Let $T\colon V\to V$ be an isometry and dim$\,V<\infty$. If a subspace
$W\subset V$ is invariant under $T$, i.e., $TW\subset W$, then so is
its orthogonal complement $W^{\perp}$, i.e., $TW^{\perp}\subset
W^{\perp}$.

\proof The restriction of $T$ to the invariant subspace $W$ is an
operator on $W$. It is an isometry of $W$, hence it is a bijection
$W\to W$, i.e.,
$$
   \forall w\in W\quad \exists w'\in W\colon\quad Tw'=w.
$$
Now, if $v\in W^{\perp}$, then
$$
  \forall w\in W\colon\quad \la w,Tv\ra=\la Tw',Tv\ra=\la w',v\ra=0,
$$
hence $Tv\in W^{\perp}$. This implies $TW^{\perp}\subset W^{\perp}$.
\hfill \qed

\subsection{Orthogonal decomposition for complex isometries} \label{UnitaryD}

For any isometry $T$ of a finite dimensional \textbf{complex} space $V$
there is an ONB of $V$ consisting of eigenvectors of $T$.

\proof Recall that eigenvalues always exist for operators on complex
vector spaces (Sect.~\ref{RealCompMat}). Let $\lambda_1$ be an
eigenvalue of $T$ with an eigenvector $x_1\neq 0$. Then $W_1=\,{\rm
span}\{x_1\}$ is a one-dimensional subspace invariant under $T$. By
Section~\ref{10.10}, the $(n-1)$-dimensional subspace $W_1^\perp$ is
also invariant under $T$. The restriction of $T$ to $W_1^\perp$ is
an operator on $W^\perp$, which is an isometry.

Let $\lambda_2$ be an eigenvalue of the operator $T\colon
W_1^\perp\to W_1^\perp$, with an eigenvector $x_2\in W_1^\perp$.
Then $W_2=\,{\rm span}\{x_2\}$ is a one-dimensional subspace of
$W_1^\perp$ invariant under $T$.

Since both $W_1$ and $W_2$ are invariant under $T$, we have that
$W_1\oplus W_2$ is a two-dimensional subspace of $V$ invariant under
$T$. By Section~\ref{10.10}, the $(n-2)$-dimensional subspace
$(W_1\oplus W_2)^\perp$ is also invariant under $T$. The restriction
of $T$ to $(W_1\oplus W_2)^\perp$ is an isometry, too.

Then we continue, inductively: at each step we ``split off'' a
one-dimensional subspace invariant under $T$ and reduce the
dimensionality of the remaining subspace. Since $V$ is finite
dimensional, we eventually exhaust all the dimensions of $V$ and
complete the proof. \hfill \qed
\medskip

\noindent{\footnotesize Note: the above theorem is \textbf{not true}
for \textbf{real} vector spaces. An isometry of a real vector space may
not have any eigenvectors; see again the rotation matrix in
Example~\ref{9.23}.}

\newpage

\subsection{Lemma} \label{10.11a}

Every operator $T\colon V\to V$ on a finite dimensional \textbf{real}
space $V$ has either a one-dimensional or a two-dimensional invariant
subspace $W\subset V$.

\proof Let $T$ be represented by a matrix $A\in\IR^{n\times n}$ in some
basis. If $A$ has a real eigenvalue, then $Ax=\lambda x$ with some
$x\neq 0$, and we get a one-dimensional invariant subspace span$\{x\}$.
If $A$ has no real eigenvalues, then the matrix $A$, treated as a
complex matrix (cf.\ Sect.~\ref{RealCompMat}), has a complex eigenvalue
$\lambda=a+\i b$, with $a,b\in\IR$ and $\i = \sqrt{-1}$, and a complex
eigenvector $x+\i y$, with $x,y\in\IR^n$. The equation
$$
    A(x+\i y)=(a+\i b)(x+\i y)=(ax-by)+(bx+ay)\i
$$
can be written as
\begin{align*}
         Ax &= a x-b y \\
         Ay &= b x+a y
\end{align*}
This implies that the subspace span$\{x,y\}$ is invariant. \qed
\medskip

{\footnotesize Note: The above subspace span$\{x,y\}$ is
two-dimensional, unless $x$ and $y$ are linearly dependent. With some
extra effort, one can verify that $x$ and $y$ can be linearly dependent
only if $b=0$, i.e.\ $\lambda\in\IR$.}


\subsection{Orthogonal decomposition for real isometries} \label{10.12}

Let $T\colon V\to V$ be an isometry of a finite dimensional
\textbf{real} space $V$. Then $V=V_1\oplus\cdots\oplus V_p$ for some
$p\leq n$, where $V_i$ are mutually orthogonal subspaces, each $V_i$ is
$T$-invariant, and either dim$\,V_i=1$ or dim$\,V_i=2$.

\proof Use induction on dim$\,V$ and apply Sections \ref{10.10} and
\ref{10.11a}. \qed

\new Note: the restriction of the operator $T$ to each
two-dimensional invariant subspace $V_i$ is simply a rotation by some
angle (as in Example~\ref{9.23}); this follows from
Exercises~\ref{ExQ+1} and \ref{ExQ-1}.\medskip

Recall that two $n\times n$ matrices $A$ and $B$ are \emph{similar}
(usually denoted by $A\sim B$) if there exists an invertible matrix $X$
such that $B=X^{-1}AX$. Two matrices are similar if they represent the
same linear operator on an $n$-dimensional space, but under two
different bases. In that case $X$ is the change of basis matrix.

\subsection{Unitary and orthogonal equivalence}
\label{11.17}

\begin{itemize}\item[\textcopyright]
Two complex matrices $A,B\in \IC^{n\times n}$ are said to be {\em
unitary equivalent}
%($A\ueq B$)
if $B=P^{-1}AP$ for some unitary matrix $P$. This can be also written
as $B=P^{\ast}AP$ (because $P^{-1}=P^\ast$).
\item[\textregistered] Two real matrices $A,B\in \IR^{n\times n}$
    are said to be {\em orthogonally equivalent}
%($A\oeq B$)
if $B=P^{-1}AP$ for some orthogonal matrix $P$. This can be also
written as $B=P^TAP$ (because $P^{-1}=P^T$). \end{itemize}

Two complex/real matrices are unitary/orthogonally equivalent if they
represent the same linear operator on a complex/real $n$-dimensional
inner product space under two different orthonormal bases (ONBs). Then
$P$ is the change of basis matrix, which must be unitary/orthogonal,
because it changes an ONB to another ONB (cf.\ Sect.~\ref{CharIsom3}).

In this course we mostly deal with ONBs, thus unitary/orthogonal
equivalence will play the same major role as similarity plays in Linear
Algebra. In particular, for any type of matrices we will try to find
simplest matrices which are unitary/orthogonal equivalent to matrices
of the given type.

\subsection{Unitary matrices in their simples form}

Any unitary matrix $Q\in \IC^{n\times n}$ is unitary equivalent to a
diagonal matrix $D=\,{\rm diag}\{d_1,\ldots,d_n\}$, whose diagonal
entries belong to the unit circle, i.e.\ $|d_i|=1$ for $1\leq i\leq
n$.\medskip

\emph{Proof}. This readily follows from Section~\ref{UnitaryD}.

\subsection{Orthogonal matrices in their simples form}

Any orthogonal matrix $Q\in \IR^{n\times n}$ is orthogonally
equivalent to a block-diagonal matrix
$$
     B=\left[\begin{array}{cccc}
     R_{11} & 0 & \cdots & 0 \\
     0 & R_{22} & \cdots & 0 \\
     \vdots & \vdots & \ddots & \vdots \\
     0 & 0 & \cdots & R_{mm}
     \end{array}\right]
$$
where $R_{ii}$ are $1\times 1$ and $2\times 2$ diagonal blocks.
Furthermore, each $1\times 1$ block is either $R_{ii}=+1$ or
$R_{ii}=-1$, and each $2\times 2$ block is a rotation matrix
$$
  R_{ii}=\left[\begin{array}{rr}\cos\theta_i & -\sin\theta_i \\
  \sin\theta_i & \cos\theta_i\end{array} \right]
$$

\emph{Proof}. This readily follows from Section~\ref{10.12}.

\begin{exercise} \label{Ex222}
Let $A\in\IC^{m\times n}$. Show that
$$
   \|UA\|_2 = \|AV\|_2 = \|A\|_2
$$
for any unitary matrices $U\in\IC^{m\times m}$ and $V\in\IC^{n\times
n}$.
\end{exercise}

\begin{exercise} \label{ExFFF}
Let $A\in\IC^{m\times n}$. Show that
$$
   \|UA\|_F = \|AV\|_F = \|A\|_F
$$
for any unitary matrices $U\in\IC^{m\times m}$ and $V\in\IC^{n\times
n}$. Here $\|\cdot\|_F$ stands for the Frobenius norm.
\end{exercise}

\begin{exercise} \label{ExQ+1}
Let $Q$ be a real orthogonal $2\times 2$ matrix and $\det
Q=1$.\\ Show that $$Q=\begin{bmatrix*}[r] \cos\theta & -\sin\theta\\
\sin\theta & \cos\theta
\end{bmatrix*}$$ for some $\theta\in [0,2\pi)$.\medskip

 \footnotesize{
In geometric terms, $Q$ represents a rotation of ${\mathbb R}^2$ by
angle $\theta$.}
\end{exercise}

\begin{exercise} \label{ExQ-1}
Let $Q$ be a real orthogonal $2\times 2$ matrix and $\det
Q=-1$.\\ Show that $$Q=\begin{bmatrix*}[r] \cos\theta & \sin\theta\\
\sin\theta & -\cos\theta
\end{bmatrix*}$$ for some $\theta\in [0,2\pi)$.\\ Also
prove that $\lambda_1=1$ and $\lambda_2=-1$ are the eigenvalues of
$Q$.\medskip

\footnotesize{In geometric terms, $Q$ represents a reflection of
${\mathbb R}^2$ across the line spanned by the eigenvector
corresponding to $\lambda_1=1$.}
\end{exercise}

\newpage

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\refstepcounter{section} \label{Ch3}

\centerline{Chapter \ref{Ch3} }\vspace{5mm}

\centerline{\Large\bf Hermitian Matrices}\vspace{3mm}

\addcontentsline{toc}{section}{\ref{Ch3}. Hermitian Matrices}

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\setcounter{subsection}{0} \setcounter{exercise}{0}
\setcounter{equation}{0}

\rem{Beginning with this chapter, we will always deal with finite
dimensional inner product spaces (unless stated otherwise). Recall that
all such spaces can be identified with $\IC^n$ or $\IR^n$ equipped with
the standard inner product (Sect.~\ref{IPSidentify}). Thus we will
mostly deal with these standard spaces.}

\subsection{Adjoint matrices} \label{11.6a}

Recall that every complex matrix $A\in \IC^{m\times n}$ defines a
linear transformation $\IC^n \to \IC^m$. Its conjugate transpose
$A^\ast\in \IC^{n\times m}$ defines a linear transformation $\IC^m \to
\IC^n$, i.e., $\qquad$ \begin{tikzpicture}[baseline=0ex] \node at (0,0)
{$\IC^n$}; \node at (2,0) {$\IC^m$};
\def\d{0.35}; \def\e{-0.35};
\draw[thick] (\e,\e)--(\e,\d)--(\d,\d)--(\d,\e)--cycle;
\draw[thick,xshift=2cm] (\e,\e)--(\e,\d)--(\d,\d)--(\d,\e)--cycle;
\draw[->,-latex] (0.5,0.05)--(1.5,0.05); \node[anchor=south] at (1,0)
{\tiny $A$}; \draw[->,-latex] (1.5,-0.05)--(0.5,-0.05);
\node[anchor=north] at (1,0) {\tiny $A^\ast$};
\end{tikzpicture}

\noindent Furthermore, for any $x\in\IC^n$ and $y\in \IC^m$ we have
$$
   \la Ax,y \ra = y^\ast Ax = (A^\ast y)^\ast x = \la x, A^\ast y \ra
$$
Likewise, $\la y,Ax \ra = \la A^\ast y,x \ra$. In plain words, $A$ can
be moved from one argument of the inner product to the other, but it
must be changed to $A^\ast$.\medskip

Likewise, every real matrix $A\in \IR^{m\times n}$ defines a linear
transformation $\IR^n \to \IR^m$. Its transpose $A^T\in \IR^{n\times
m}$ defines a linear transformation $\IR^m \to \IR^n$. Furthermore, for
any $x\in\IR^n$ and $y\in \IR^m$ we have
$$
   \la Ax,y \ra = y^T Ax = (A^T y)^T x = \la x, A^T y \ra
$$
Thus in the real case, too, $A$ can be moved from one argument of the
inner product to the other, but it must be changed to $A^\ast=A^T$.

\subsection{Adjoint transformations}

More generally, for any linear transformation $T\colon V\to W$ of
two inner product vector spaces $V$ and $W$ (both real or both
complex) there exists a unique linear transformation $T^\ast\colon
W\to V$ such that $\forall v\in V,\ \forall w\in W$
$$
   \la Tv,w\ra = \la v,T^{\ast}w\ra.
$$
$T^\ast$ is often called the {\em adjoint} of $T$.

{\footnotesize $T^\ast$ generalizes the conjugate transpose of a
matrix.}\medskip

{\footnotesize The existence and uniqueness of $T^\ast$ can be proved
by a general argument, avoiding identification of the spaces $V$ and
$W$ with $\IC^n$ or $\IR^n$. The argument is outlined below, in
Sections~\ref{11.1} to \ref{11.4}. (This part of the course can be
skipped.)

\subsection{Riesz representation theorem} \label{11.1}

Let $f\in V^{\ast}$, i.e., let $f$ be a linear functional on $V$. Then
there is a unique vector $u\in V$ such that
$$
   f(v)=\la v,u\ra\ \ \ \ \ \ \forall v\in V
$$

\proof Let $B=\{u_1,\ldots,u_n\}$ be an ONB in $V$. Then for any
$v=\sum c_iu_i$ we have $f(v)=\sum c_if(u_i)$ by linearity. Also, for
any $u=\sum d_iu_i$ we have $\la v,u\ra = \sum c_i\bar{d}_i$. Hence,
the vector $u=\sum \overline{f(u_i)}\,u_i$ will suffice. To prove the
uniqueness of $u$, assume $\la v,u\ra = \la v,u'\ra$ for all $v\in V$.
Setting $v=u-u'$ gives $\la u-u',u-u'\ra = 0$, hence $u=u'$.

\subsection{Quasi-linearity} \label{11.2}

The identity $f\leftrightarrow u$ established in the previous theorem
is ``quasi-linear'' in the following sense: $f_1+f_2\leftrightarrow
u_1+u_2$ and $cf\leftrightarrow \bar{c}u$. In the real case, it is
perfectly linear, though, and hence it is an isomorphism between
$V^{\ast}$ and $V$.

\subsection{Remark}\label{11.2a}

If dim$\, V=\infty$, then Theorem~\ref{11.1} fails. Consider, for
example, $V=C[0,1]$ (real functions) with the inner product $\la
F,G\ra=\int_0^1F(x)G(x)\, dx$. Pick a point $t\in [0,1]$. Let $f\in
V^{\ast}$ be a linear functional defined by $f(F)=F(t)$. It does not
correspond to any $G\in V$ so that $f(F)=\la F,G\ra$. In fact, the lack
of such functions $G$ has led mathematicians to the concept of {\em
delta-functions}: a delta-function $\delta_t(x)$ is ``defined'' by
three requirements: $\delta_t(x)\equiv 0$ for all $x\neq t$,
$\delta_t(t)=\infty$ and $\int_0^1F(x)\delta_t(x)\, dx=F(t)$ for every
$F\in C[0,1]$.

\subsection{Existence and uniqueness of adjoint transformation}
\label{11.4}

Let $T\colon V\to W$ be a linear transformation. Then there is a unique
linear transformation $T^{\ast}\colon W\to V$ such that $\forall v\in
V$ and $\forall w\in W$
$$
   \la Tv,w\ra = \la v,T^{\ast}w\ra.
$$

\proof Let $w\in W$. Then $f(v)\colon =\la Tv,w\ra$ defines a linear
functional $f\in V^{\ast}$. By the Riesz representation theorem, there
is a unique $v'\in V$ such that $f(v)=\la v,v'\ra$. Then we define
$T^{\ast}$ by setting $T^{\ast}w=v'$. The linearity of $T^{\ast}$ is a
routine check. Note that in the complex case the conjugating bar
appears twice and thus cancels out. The uniqueness of $T^{\ast}$ is
obvious. \qed }\bigskip

We now return to the main line of our course.

\subsection{Relation between Ker$\,T^\ast$ and Range$\, T$} \label{11.7}

Let $T\colon V\to W$ be a linear transformation. Then
$$
   {\rm Ker}\,T^{\ast}=\, ({\rm Range}\, T)^{\perp}
$$

{\footnotesize \noindent\emph{Proof}. Suppose $y\in\,{\rm
Ker}\,T^{\ast}$, i.e., $T^\ast y=0$. Then for every $x\in V$
$$
  0=\la x,0\ra = \la x,T^\ast y\ra = \la Tx,y\ra.
$$
Thus $y$ is orthogonal to all vectors $Tx$, $x\in V$, hence $y \in\,
({\rm Range}\, T)^{\perp}$.

Conversely, if $y \in\, ({\rm Range}\, T)^{\perp}$, then $y$ is
orthogonal to all vectors $Tx$, $x\in V$, hence
$$
  0 = \la Tx,y\ra=\la x,T^\ast y\ra
$$
for all $x\in V$. In particular, we can put $x=T^\ast y$ and see
that $\la T^\ast y,T^\ast y\ra=0$, hence $T^\ast y=0$, i.e.,
$y\in\,{\rm Ker}\,T^{\ast}$. \hfill \qed}
\medskip

Next we will assume that $V=W$, in which case $T$ is an operator.

\subsection{Selfadjoint operators and matrices} \label{11.8}

A linear operator $T\colon V\to V$ is said to be {\em selfadjoint} if
$T^{\ast}=T$.

A square matrix $A$ is said to be {\em selfadjoint} if
$A^{\ast}=A$.\medskip

\noindent In the real case, this is equivalent to $A^T=A$, i.e., $A$ is
a symmetric matrix. In the complex case, selfadjoint matrices are
called {\em Hermitian matrices}.

\subsection{Examples}

The matrix $\left[\begin{smallmatrix}1 & 3\\ 3 &
2\end{smallmatrix}\right]$ is symmetric. The matrix
$\left[\begin{smallmatrix}1 & 3+i\\ 3-i & 2\end{smallmatrix}\right]$ is
Hermitian.

The matrices $\left[\begin{smallmatrix}1 & 3+i\\ 3+i &
2\end{smallmatrix}\right]$ and $\left[\begin{smallmatrix}1+i & 3+i\\
3-i & 2\end{smallmatrix}\right]$ are \textbf{not} Hermitian
(why?).\medskip

\noindent Note: the diagonal components of a Hermitian matrix must
be real numbers!

\subsection{Hermitian property under unitary equivalence}
\label{SAequiv}

\begin{itemize}\item[\textcopyright]
If $A$ is a complex Hermitian matrix unitary equivalent to $B$, then
$B$ is also a complex Hermitian matrix.
\item[\textregistered] If $A$ is a real symmetric matrix orthogonally
    equivalent to $B$, then $B$ is also a real symmetric matrix.
\end{itemize}

{\footnotesize \emph{Proof}. If $A^\ast=A$ and  $B=P^\ast AP$, then
$B^\ast = P^\ast A^\ast P=P^\ast AP=B$. \hfill \qed}

\subsection{Invariance principle for selfadjoint operators} \label{11.11}

Let $T$ be a selfadjoint operator and a subspace $W$ be $T$-invariant,
i.e., $TW\subset W$. Then $W^{\perp}$ is also $T$-invariant, i.e.,
$TW^{\perp}\subset W^{\perp}$.

{\footnotesize \proof If $v\in W^{\perp}$, then for any $w\in W$ we
have $\la Tv,w\ra =\la v,Tw\ra=0$, so $Tv\in W^{\perp}$.}

\subsection{Spectral Theorem} \label{11.20}

Let $T\colon V\to V$ be a selfadjoint operator. Then there is an ONB
consisting of eigenvectors of $T$, and all the eigenvalues of $T$
are real numbers.

\proof First let $V$ be a \emph{complex} space. Then we apply
Principle~\ref{11.11} to construct an ONB of eigenvectors, exactly as
we did in Section~\ref{UnitaryD}.

Now $T$ is represented in the canonical basis $\{e_1,\ldots,e_n\}$
by a Hermitian matrix $A$. In the (just constructed) ONB of
eigenvectors, $T$ is represented by a diagonal matrix
$D=\,$diag$\{d_1,\ldots,d_n\}$, and $d_1,\ldots,d_n$ are the
eigenvalues of $T$. Since $A$ and $D$ are unitary equivalent, $D$
must be Hermitian, too (by \ref{SAequiv}). Hence the
diagonal components $d_1,\ldots,d_n$ of $D$ are real numbers. This
completes the proof of Spectral Theorem for complex spaces.

Before we proceed to real spaces, we need to record a useful fact
about complex Hermitian matrices. Every Hermitian matrix $A \in
\IC^{n\times n}$ defines a selfadjoint operator on $\IC^n$. As we
just proved, there exists an ONB of eigenvectors. Hence $A$ is
unitary equivalent to a diagonal matrix
$D=\,$diag$\{d_1,\ldots,d_n\}$. Since $D$ is Hermitian, its diagonal
entries $d_1,\ldots,d_n$ are real numbers. Therefore
 \beq  \label{HlambdaR}
%\fbox{Every Hermitian matrix has only real eigenvalues}
  \boxed{\phantom\int\text{Every Hermitian matrix has only real eigenvalues}\phantom\int}
%\end{framed}
 \eeq

Now let $V$ be a \emph{real} space. In the canonical basis
$\{e_1,\ldots,e_n\}$, the self-adjoint operator $T\colon V\to V$ is
represented by a real symmetric matrix $A$. The latter,
\emph{considered as a complex matrix}, is Hermitian, thus it has
only real eigenvalues (see above). Hence the construction of an ONB
of eigenvectors, as done in Section~\ref{UnitaryD}, works again. The
proof is now complete. \qed\medskip

{\footnotesize Note: For every real symmetric matrix $A$, the
characteristic polynomial is $C_A(x)=\prod_i (x-\lambda_i)$, where all
$\lambda_i$'s are real numbers.}

\newpage

\subsection{Characterization of Hermitian matrices} \label{11.21}

\begin{itemize}\item[\textcopyright]
A complex matrix is Hermitian iff it is unitary equivalent to a
diagonal matrix with real diagonal entries. \item[\textregistered] A
real matrix is symmetric iff it is orthogonally equivalent to a
diagonal matrix (whose entries are automatically real).
\end{itemize}
More generally: if an operator $T\colon V\to V$ has an ONB of
eigenvectors, and all its eigenvalues are real numbers, then $T$ is
self-adjoint.

\subsection{Eigendecomposition for Hermitian matrices} \label{11.22a}

Let $A=QDQ^{\ast}$, where $Q$ is a unitary matrix and $D$ is a diagonal
matrix. Denote by $q_i$ the $i$th column of $Q$ and by $d_i$ the $i$th
diagonal entry of $D$. Then
$$
         Aq_i = d_iq_i,\quad\quad 1\leq i\leq n
$$
i.e.\ the columns of $Q$ are eigenvectors of $A$ whose eigenvalues
are the corresponding diagonal components of $D$ (cf.\
Section~\ref{EVDecomp}).

\subsection{Inverse of a selfadjoint operator} \label{11.24}

(a) If an operator $T$ is selfadjoint and invertible, then so is
$T^{-1}$.

(b) If a matrix $A$ is selfadjoint and nonsingular, then so is
$A^{-1}$.

\proof (a) By Spectral Theorem~\ref{11.20}, there is an ONB
$\{u_1,\ldots,u_n\}$ consisting of eigenvectors of $T$, and the
eigenvalues $\lambda_1,\ldots,\lambda_n$ of $T$ are real numbers.
Since $T$ is invertible, $\lambda_i\neq 0$ for all $i=1,\ldots,n$.
Now
$$
   Tu_i=\lambda_iu_i
   \qquad\Longrightarrow\qquad
   T^{-1}u_i = \lambda_i^{-1}u_i
$$
hence $T^{-1}$ has the same eigenvectors $\{u_1,\ldots,u_n\}$, and
its eigenvalues are the reciprocals of those of $T$, so they are
real numbers, too. Therefore, $T^{-1}$ is selfadjoint.

(b) If $A$ is selfadjoint, then $A=P^\ast DP$ with a unitary matrix $P$
and a diagonal matrix $D=\,{\rm diag}\{d_1,\ldots,d_n\}$ with real
diagonal entries. Now
$$
   A^{-1} = P^{-1}D^{-1}(P^\ast)^{-1} = P^\ast D^{-1} P
$$
where $D^{-1}=\,{\rm diag}\{d_1^{-1},\ldots,d_n^{-1}\}$ is also a
diagonal matrix with real diagonal entries. Therefore $A^{-1}$ is
selfadjoint. \hfill \qed

\newpage

\subsection{Projections} \label{11.12}

Let $V=W_1 \oplus W_2$, i.e., suppose our vector space $V$ is a direct
sum of two subspaces. Recall that for each $v\in V$ there is a unique
decomposition $v=w_1+w_2$ with $w_1\in W_1$ and $w_2\in W_2$.

The operator $P\colon V\to V$ defined by\\
$$
   Pv = P(w_1+w_2) = w_2
$$
is called {\em projection} (or {\em projector}) of $V$ \emph{on} $W_2$
\emph{along} $W_1$.

Note that
$$
  {\rm Ker}\,P=W_1
  \qquad\text{and}\qquad
  {\rm Range}\,P=W_2.
$$

Also note that $P^2=P$.

\begin{center}
\begin{tikzpicture}
         \draw[ultra thick,gray] (-1,0) -- (2,0);
         \draw[ultra thick,gray] (-0.5,-1) -- (0.75,1.5);
         \draw[->,-latex] (0,0) -- (2,1);
         \draw[->,-latex] (0,0) -- (0.5,1);
         \draw[->,-latex] (0,0) -- (1.5,0);
         \draw[gray,dashed] (2,1) -- (0.5,1);
         \draw[gray,dashed] (2,1) -- (1.5,0);
         \node[anchor=west] at (1.9,0.9) {\footnotesize $v$};
         \node[anchor=east] at (0.5,0.9) {\footnotesize $w_1$};
         \node[anchor=north] at (1.5,0) {\footnotesize $w_2$};
         \node[anchor=west] at (0.65,1.4) {\footnotesize $W_1$};
         \node[anchor=west] at (1.9,0) {\footnotesize $W_2$};
         \begin{scope}[xshift=8cm]
         \draw[ultra thick,gray] (-3,0) -- (3,0);
         \draw[ultra thick,gray] (-0.5,-1) -- (0.75,1.5);
         \foreach \x in {{-2},...,{2}} \draw[dashed,->,-latex] (0.4+\x,0.8) --
         (0.1+\x,0.2);
         \foreach \x in {{-2},...,{2}} \draw[dashed,->,-latex] (-0.4+\x,-0.8) --
         (-0.1+\x,-0.2);
         \node[anchor=west] at (0.75,1.4) {\footnotesize Ker$\,P$};
         \node[anchor=west] at (3,0) {\footnotesize Range$\,P$};
         \end{scope}
\end{tikzpicture}
\end{center}

\subsection{Projections (alternative definition)}

An operator $P\colon V\to V$ is a projection iff $P^2=P$.\medskip

{ \footnotesize \noindent\emph{Proof}. $P^2=P$ implies that for every
$v\in V$ we have $P(v-Pv)=0$, so
$$
  w_1 \colon = v-Pv \in \,{\rm Ker}\, P
$$
Denoting $w_2=Pv$ we get $v=w_1+w_2$ with $w_1\in\,$Ker$\,P$ and
$w_2\in\,{\rm Range}\, P$. Furthermore,
$$
  Pv = Pw_1+Pw_2 = 0+P(Pv) = P^2v=Pv=w_2
$$
We also note that Ker$\,P\cap\,{\rm Range}\,P=\{0\}$. Indeed, for any
$v\in\,{\rm Range}\,P$ we have $Pv=v$ (as above) and for any
$v\in\,{\rm Ker}\, P$ we have $Pv=0$. Thus $v\in \,{\rm
Ker}\,P\cap\,{\rm Range}\,P$ implies $v=0$. Hence $V=\,{\rm Ker}\,
P\oplus\,{\rm Range}\, P$. \qed }\medskip

\subsection{``Complimentary'' projections}

Let $V=W_1\oplus W_2$. There is a unique projection $P_1$ on $W_1$
along $W_2$ and a unique projection $P_2$ on $W_2$ along $W_1$. They
``complement'' each other adding up to the identity:
$$
   P_1+P_2=I.
$$

\newpage

\subsection{Orthogonal projections} \label{11.14}

Let $V$ be an inner product vector space (not necessarily finite
dimensional) and $W\subset V$ a finite dimensional subspace. Then the
projection on $W$ along $W^{\perp}$ is called {\em orthogonal
projection} on $W$.

{\footnotesize The assumption dim$\,W<\infty$ is made to ensure that
$V=W\oplus W^{\perp}$ (cf.\ Sect.~\ref{ODS}).}

\subsection{Characterization of orthogonal projections} \label{11.16}

Let $P$ be a projection. Then $P$ is an orthogonal projection if and
only if $P$ is selfadjoint.\medskip

{ \footnotesize \noindent\emph{Proof}. Let $P$ be a projection on $W_2$
along $W_1$, and $V=W_1\oplus W_2$. For any vectors $v,w\in V$ we have
$v=v_1+v_2$ and $w=w_1+w_2$ with some $v_i,w_i\in W_i$, $i=1,2$.

Now, if $P$ is an orthogonal projection, then
$$
  \la Pv,w\ra=\la v_2,w\ra =\la
  v_2,w_2\ra=\la v,w_2\ra=\la v,Pw\ra.
$$
If $P$ is not an orthogonal projection, then there are $v_1\in W_1$ and
$w_2\in W_2$ so that $\la v_1,w_2\ra\neq 0$. Then $\la v_1,Pw_2\ra\neq
0=\la Pv_1,w_2\ra$. \hfill \qed }


\begin{exercise}
Let $V$ be an inner product space and $W\subset V$ a finite dimensional
subspace with ONB $\{u_1,\ldots,u_n\}$. For every
$x\in V$ define $$P(x)=\sum_{i=1}^n\langle x,u_i\rangle u_i$$\\
(i) Prove that $x-P(x)\in W^{\perp}$, hence $P$ is the orthogonal
projection onto $W$.\\ (ii) Prove that $\|x-P(x)\|\leq \|x-z\|$ for
every $z\in W$, and that if $\|x-P(x)\|=\|x-z\|$ for some $z\in W$,
then $z=P(x)$.
\end{exercise}

\begin{exercise}
(JPE, May 1999) Let $P\in \IC^{n\times n}$ be a projector. Show that
$\|P\|_2 \geq 1$ with equality if and only if $P$ is an orthogonal
projector.
\end{exercise}

\newpage

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\refstepcounter{section} \label{Ch4}

\centerline{Chapter \ref{Ch4} }\vspace{5mm}

\centerline{\Large\bf Positive Definite Matrices}\vspace{3mm}

\addcontentsline{toc}{section}{\ref{Ch4}. Positive Definite Matrices}

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\setcounter{subsection}{0} \setcounter{exercise}{0}
\setcounter{equation}{0}

Recall that an inner product $\la\cdot, \cdot\ra$ is a
complex-valued function of two vector arguments, cf.\
Section~\ref{InnProd}. It is linear in the first argument and
``conjugate linear'' in the second.

In the space $\IC^n$, any function $f\colon \IC^n \times \IC^n \to \IC$
with the last two properties can be defined by
$$
    f(x,y) =
    \la Ax,y\ra = y^\ast Ax,
$$
where $A\in\IC^{n\times n}$ is a matrix. Then $f(x,y)$ automatically
satisfies axioms 2 and 3 of the inner product (Sect.~\ref{InnProd}), as
well as the rules \eqref{IP2}.

But axiom 1 of the inner product (Sect.~\ref{InnProd}) may not hold for
any such $f(x,y)$. It holds if and only if
$$
    \la Ax,y\ra = f(x,y) =
    \overline{f(y,x)} = \overline{\la Ay,x\ra}
    = \la x,Ay\ra=\la A^\ast x,y\ra,
$$
i.e., iff $A=A^\ast$, which means that $A$ must be a Hermitian matrix.

Furthermore, if we want $f(x,y)$ satisfy \emph{all} the axioms of
the inner product, then we need to ensure the last axiom 4:
$$
   f(x,x)=\la Ax,x\ra >0\qquad \forall x\neq 0.
$$

\subsection{Positive definite matrices}
\label{12.9}

A matrix $A$ is said to be {\em positive definite} if it is
Hermitian and
$$
   \la Ax,x\ra =x^\ast Ax>0\qquad \forall x\neq 0.
$$
In the real case, a matrix $A$ is {\em positive definite} if it is
symmetric and
$$
   \la Ax,x\ra =x^T Ax>0\qquad \forall x\neq 0.
$$
If we replace ``$>0$'' with ``$\geq 0$'' in the above formulas, we
get the definition of {\em positive semi-definite}\footnote{Perhaps,
a more natural term would be ``non-negative definite'', but we will
use the official name {\em positive semi-definite}.} matrices.

\newpage

{\footnotesize Symmetric positive definite matrices play important role
in optimization theory (the minimum of a function of several variables
corresponds to a critical point where the matrix of second order
partial derivatives is positive definite) and probability theory (the
covariance matrix of a random vector is symmetric positive
definite).}\medskip

{\small Interestingly, the condition $\la Ax,x\ra >0$ for all
$x\in\IC^n$ implies that the matrix $A$ is Hermitian, and therefore
positive definite. We will prove this in Sections~\ref{12.7} and
\ref{12.8}. (It is just a curious fact, with little importance, so the
next two sections can be skipped.)

\subsection{Lemma} \label{12.7}

Let $A,B$ be complex matrices. If $\la Ax,x\ra=\la Bx,x\ra$ for all
$x\in \IC^n$, then $A=B$. (Proof: see exercises.)

\medskip\noindent{\footnotesize Note: this lemma holds only in complex spaces, it fails in real spaces (see exercises).}

\subsection{Sufficient condition for positive definiteness} \label{12.8}

If $A$ is a complex matrix such that $\la Ax,x\ra\in\IR$ for all
$x\in\IC^n$, then $A$ is Hermitian. In particular, if $\la Ax,x\ra
>0$ for all non-zero vectors $x\in\IC^n$, then $A$ is positive
definite.

\proof We have
$$
  \la Ax,x\ra = \overline{\la Ax,x\ra}= \la x,Ax\ra=\la A^{\ast}x,x\ra,
$$
hence $A=A^{\ast}$ by Lemma~\ref{12.7}. Now the condition $\la
Ax,x\ra>0$ for all $x\neq 0$ implies that $A$ is positive definite.
\qed}\medskip

{\scriptsize Next we define positive definite and positive
semi-definite operators in general spaces in a more abstract way; see
Sections~\ref{12.1}--\ref{PDfT} below. (These sections can be skipped
in class and assigned to more advanced students as a reading project.)

\subsection{Bilinear forms} \label{12.1}

A {\em bilinear form} on a complex vector space $V$ is a
complex-valued function of two vector arguments, i.e., $f\colon
V\times V\to\IC$ such that
\begin{align*}
    f(u_1+u_2,v) &=f(u_1,v)+f(u_2,v) \\
    f(cu,v) &=cf(u,v) \\
    f(u,v_1+v_2) &=f(u,v_1)+f(u,v_2) \\
    f(u,cv) &=\bar{c}f(u,v)
\end{align*}
for all vectors $u,v,u_i,v_i\in V$ and scalars $c\in\IC$. In other
words, $f$ must be linear in the first argument and ``conjugate
linear'' in the second.

A bilinear form on a real vector space $V$ is a mapping $f\colon
V\times V\to\IR$ that satisfies the same properties, except $c$ is a
real scalar, i.e., $\bar{c}=c$.

\subsection{Representation of bilinear forms} \label{12.3}

Let $V$ be a finite dimensional inner product space. Then for every
bilinear form $f$ on $V$ there is a unique linear operator $T\colon
V\to V$ such that
$$
   f(u,v)=\la Tu,v\ra\ \ \ \ \ \ \forall u,v\in V
$$

\noindent{\scriptsize \emph{Proof}. For every $v\in V$ the function
$g(u)=f(u,v)$ is linear in $u$, so by Riesz Representation
Theorem~\ref{11.1} there is a vector $w\in V$ such that $f(u,v)=\la
u,w\ra$. Define a map $S\colon V\to V$ by $Sv=w$. It is then a routine
check that $S$ is linear. Setting $T=S^{\ast}$ proves the existence.
The uniqueness is obvious. \hfill \qed}

\subsection{Corollary} \label{12.4}

\begin{itemize}\item[\textcopyright] Every bilinear form on
$\IC^n$ can be represented by $f(x,y)=\la Ax,y\ra$ with some
$A\in\IC^{n\times n}$. \item[\textregistered] Every bilinear form on
$\IR^n$ can be represented by $f(x,y)=\la Ax,y\ra$ with some
$A\in\IR^{n\times n}$.\end{itemize}

Bilinear forms generalize the notion of inner product in abstract
spaces. In order for a bilinear form to become an inner product,
though, it needs two additional properties: conjugate symmetry
$f(x,y)=\overline{f(y,x)}$ and the positivity $f(x,x)>0$ for all $x\neq
0$.

\subsection{Hermitian/symmetric forms} \label{12.5}

A bilinear form $f$ on a complex space $V$ is \emph{Hermitian} if
$f(u,v)=\overline{f(v,u)}\ \ \forall u,v\in V$.

A bilinear form $f$ on a real space $V$ is \emph{symmetric} if
$f(u,v)=f(v,u)\ \ \forall u,v\in V$.

\subsection{Quadratic forms}

For a Hermitian bilinear form $f$, the function $q\colon V\to \IR$
defined by $q(u)=f(u,u)$ is called the {\em quadratic form}
associated with $f$. Note that $q(u)\in\IR$ even in the complex
case, because $f(u,u)=\overline{f(u,u)}$.

\subsection{Theorem} \label{12.6}

A linear operator $T\colon V\to V$ is selfadjoint if and only if the
bilinear form $f(u,v)=\la Tu,v\ra$ is Hermitian (in the real case,
symmetric).

\proof If $T$ is selfadjoint, then $f(u,v)= \la Tu,v\ra =\la
u,Tv\ra=\overline{\la Tv,u\ra}=\overline{f(v,u)}.$ If $f$ is
Hermitian, then $\la u,Tv\ra =\overline{\la Tv,u\ra}
=\overline{f(v,u)}=f(u,v)= \la Tu,v\ra =\la u,T^{\ast}v\ra,$
therefore $T=T^{\ast}$. \qed\medskip

Thus, Hermitian bilinear forms on $\IC^n$ are defined by Hermitian
matrices.

\subsection{Positive definite forms and operators}

A Hermitian (symmetric) bilinear form $f$ on a vector space $V$ is
said to be {\em positive definite} if $f(u,u)>0$ for all $u\neq 0$.
A selfadjoint operator $T\colon V\to V$ is said to be {\em positive
definite} if $\la Tu,u\ra >0$ for all $u\neq 0$.\medskip

If we replace ``$>0$'' with ``$\geq 0$'' in the above formulas, we get
the definition of {\em positive semi-definite} bilinear forms and
operators.

\subsection{Theorem} \label{PDfT}

The following are equivalent: \begin{itemize}\item[(a)] a bilinear form
$f(u,v)$ is an inner product. \item[(b)] $f(u,v)=\la Tu,v\ra$, where
$T$ is a positive definite operator.
\end{itemize}
}\smallskip

We now return to the main line of our course.

\newpage

\begin{spacing}{0.85}
{\footnotesize \noindent \textbf{Special note}: The rest of
Chapter~\ref{Ch4} and the entire Chapter~\ref{Ch5} are extremely
important. The instructor should go slow, cover every minor detail,
and not hesitate spending extra time explaining things. The students
are advised to attend classes, closely follow the lectures, and do
their best to fully grasp the material. Covering this part of the
course well will allow the class to move faster through subsequent
chapters. On the other hand, failure to fully understand this part
of the course will cause major troubles later, and it might be
virtually impossible to learn some of the subsequent topics.}
\end{spacing}\medskip

Let $A \in \IC^{n\times n}$ be a
Hermitian matrix. By \eqref{HlambdaR}, all its eigenvalues
$\lambda_1, \ldots, \lambda_n$ are real numbers, hence they can
be ordered. In particular, we can define
 \beq \label{lambdaminmax}
  \lambda_{\min} = \min_{1\leq i\leq n} \lambda_i
     \qquad\text{and}\qquad
  \lambda_{\max} = \max_{1\leq i\leq n} \lambda_i.
\eeq

\subsection{Properties of Hermitian matrices} \label{12.10bb}

Let $A \in \IC^{n\times n}$ be a Hermitian matrix. Then
\begin{itemize}
\item[(a)] For every vector $x\in \IC^n$ we have
\beq \label{Axxreal}
       \la Ax,x\ra \in \IR.
\eeq
\item[(b)] For every vector $x\in \IC^n$ we have
\beq \label{LmAxx}
     \lambda_{\min} \|x\|_2^2 \leq
     \la Ax,x\ra \leq  \lambda_{\max} \|x\|_2^2.
\eeq
\item[(c)] We have
\beq \label{AxxLeft}
   \la Ax,x\ra = \lambda_{\min} \|x\|_2^2
   \qquad\Longleftrightarrow\qquad
   Ax=\lambda_{\min}x
\eeq
and
\beq \label{AxxRight}
   \la Ax,x\ra = \lambda_{\max} \|x\|_2^2
   \qquad\Longleftrightarrow\qquad
   Ax=\lambda_{\max}x
\eeq
\end{itemize}
Part (c) implies that the left and right inequalities in
\eqref{LmAxx} cannot be improved -- they turn into equalities for certain nonzero vectors $x$.

%\begin{spacing}{0.85}
%{\footnotesize
\medskip\noindent{\em Proof}. By Spectral
Theorem~\ref{11.20}, there is an ONB $\{u_1,\ldots,u_n\}$ of
eigenvectors of $A$, corresponding to the eigenvalues $\lambda_1,
\ldots, \lambda_n$. Then for any vector $x=\sum c_iu_i$ we have $Ax
= \sum \lambda_ic_iu_i$. Due to \eqref{ParcevalIP}, we have
\begin{align*}
     \la Ax,x\ra&=\lambda_1c_1\bar{c}_1+\cdots +\lambda_nc_n\bar{c}_n\\
       &=\lambda_1|c_1|^2+\cdots +\lambda_n|c_n|^2.
\end{align*}
Since $\lambda_i\in\IR$, we conclude that $\la Ax,x\ra\in \IR$ thus proving part (a). Next,
$$
     \la Ax,x\ra \geq \lambda_{\min}\bigl(|c_1|^2+\cdots +|c_n|^2\bigr)
     =\lambda_{\min}\|x\|_2^2,
$$
where we used \eqref{ParcevalNorm}. We also see that
$$
  \la Ax,x\ra - \lambda_{\min}\|x\|_2^2
  = \sum_{i=1}^n |c_i|^2(\lambda_i-\lambda_{\min}),
$$
where all the terms are nonnegative because $\lambda_i \geq
\lambda_{\min}$. Thus, the left inequality in \eqref{LmAxx} turns into
an equality if and only if $c_i=0$ for all $i$'s such that
$\lambda_i>\lambda_{\min}$. This means exactly that $x$ is an
eigenvector corresponding to the smallest eigenvalue $\lambda_{\min}$. The left inequality in \eqref{LmAxx} and the claim \eqref{AxxLeft} are now proved.

Similarly,
$$
     \la Ax,x\ra \leq \lambda_{\max}\bigl(|c_1|^2+\cdots +|c_n|^2\bigr)
     =\lambda_{\max}\|x\|_2^2
$$
and
$$
   \lambda_{\max}\|x\|_2^2 - \la Ax,x\ra
  = \sum_{i=1}^n |c_i|^2(\lambda_{\max}-\lambda_i).
$$
Thus, the right inequality in \eqref{LmAxx} turns into an equality if
and only if $c_i=0$ for all $i$'s such that $\lambda_{\max}
> \lambda_i$, which means exactly that $x$ is an eigenvector
corresponding to the largest eigenvalue $\lambda_{\max}$. The right inequality in \eqref{LmAxx} and the claim \eqref{AxxRight} are now proved, too.
\qed
%}\end{spacing}
\medskip


\subsection{Eigenvalues of positive definite matrices} \label{12.10}

Let $A \in \IC^{n\times n}$ be a Hermitian matrix. It is positive definite iff all its eigenvalues are positive.

Similarly, a Hermitian matrix $A\in \IC^{n\times n}$ is positive
semi-definite iff all its eigenvalues are nonnegative.

\medskip\emph{Proof}. This readily follows from Section~\ref{12.10bb}.
\qed


\subsection{Inverse of a positive definite matrix} \label{12.11}

If a matrix $A$ is positive definite, then so is $A^{-1}$.

\subsection{Characterization of positive definite matrices} \label{12.13}

A matrix $A\in\IC^{n\times n}$ is positive definite iff there is a
nonsingular matrix $B$ such that $A=B^{\ast}B$.

\proof ``$\Leftarrow$'' If $A=B^{\ast}B$, then
$A^{\ast}=B^{\ast}(B^{\ast})^{\ast}=A$, so $A$ is Hermitian. Now
$$
  \la Ax,x\ra=\la Bx,Bx\ra>0
  \qquad \forall x\neq 0,
$$
so $A$ is positive definite (note that $x\neq 0$ implies $Bx\neq 0$,
because $B$ is nonsingular).

``$\Rightarrow$'' According to Sections~\ref{11.21} and \ref{12.10},
$A=P^{-1}DP$, where $D=\,{\rm diag}\,\{d_1,\ldots,d_n\}$ is a diagonal
matrix with positive diagonal entries $d_i>0$, and $P$ is a unitary
matrix. We construct a new diagonal matrix
$$
  D_{1/2}=\,{\rm
  diag}\,\{\sqrt{d_1},\ldots,\sqrt{d_n}\}.
$$
Now $A=B^2$ where $B=P^{-1}D_{1/2}P$. Lastly we note that $B$ is
Hermitian, due to Section~\ref{11.21}, hence $A=B^\ast B$. \qed

\subsection{Characterization of positive semi-definite matrices} \label{12.13a}

A matrix $A\in\IC^{n\times n}$ is positive semi-definite if and
only if there is a matrix $B$ (not necessarily nonsingular) such that
$A=B^{\ast}B$.

\medskip{\footnotesize The proof is similar to the above, so we omit it.}

\subsection{Full rank and rank deficient matrices} \label{17.14}

A matrix $A\in\IC^{m\times n}$ is said to have {\em full rank} if
$$
   {\rm rank}\, A=\min\{m,n\}
$$
Otherwise, $A$ is said to be {\em rank deficient}.

\subsection{Products $A^{\ast}A$ and $AA^{\ast}$} \label{16.8}

Let $A$ be a rectangular $m\times n$ matrix. Then the matrices
$A^{\ast}A$ and $AA^{\ast}$ are Hermitian and positive
semi-definite. If $m\neq n$ and $A$ has full rank, then the
smaller of the two matrices $A^{\ast}A$ and $AA^{\ast}$ is
positive definite.

\proof First we verify the Hermitian property:
$$
   (A^{\ast}A)^\ast = A^\ast(A^{\ast})^\ast = A^\ast A
$$
and similarly $(AA^{\ast})^\ast = AA^\ast$. Next we verify positive
semidefiniteness:
$$
   \la A^{\ast}Ax,x\ra = \la Ax,Ax\ra \geq 0
$$
for any $x\in\IC^n$ and similarly
$$
   \la AA^{\ast}y,y\ra = \la A^\ast y,A^\ast y\ra \geq 0
$$
for any $y\in\IC^m$.

Now let $A$ have full rank. If $m\geq n$, then Ker$(A)=\{0\}$, hence
$Ax\neq 0$ for any $0\neq x\in \IC^n$, so that $\la A^{\ast}Ax,x\ra
= \la Ax,Ax\ra >0$, which implies that $A^\ast A$ is positive
definite. If $m<n$, then Range$(A)=\IC^m$, hence for any $0\neq y\in
\IC^m$ there is $0\neq x\in\IC^n$ such that $y=Ax$. Therefore
$$
  \la A^\ast y,x\ra = \la y,Ax\ra = \la y,y\ra >0
$$
which implies $A^\ast y\neq 0$. Therefore
$$
     \la AA^{\ast}y,y\ra = \la A^\ast y,A^\ast y\ra > 0
$$
which implies that $AA^\ast$ is positive definite. \qed\medskip

\subsection{Spectral radius} \label{12.10aaa}

Let $A\in \IC^{n\times n}$ be a square matrix and $\lambda_1,
\ldots, \lambda_n$ its eigenvalues. Then
$$
  \rho_A= \max_{1\leq i\leq n} |\lambda_i|
$$
is called the \emph{spectral radius} of $A$. It is the radius of the
smallest disk $\{|z|\leq r\}$ in the complex plane that contains all
the eigenvalues of $A$.\medskip

Note: if $A$ is a Hermitian matrix, then
$$
  \rho_A=\max\{|\lambda_{\max}|,|\lambda_{\min}|\}
$$
in the notation of \eqref{lambdaminmax}.

\newpage

\subsection{Spectral radius for Hermitian matrices} \label{12.10aa}

If $A \in \IC^{n\times n}$ is a Hermitian matrix, then
$$
   \|A\|_2 = \rho_A.
$$

\medskip\noindent{\em Proof}. We use the notation of the
proof in Section~\ref{12.10bb}:
\begin{align*}
   \|Ax\|_2^2 = \la Ax,Ax\ra &=\lambda_1^2|c_1|^2+\cdots +\lambda_n^2|c_n|^2\\
      &\leq \bigl[\max_{1\leq i\leq n} \lambda_i^2\bigr]\,
     \sum_{j=1}^n |c_j|^2 =\rho_A^2\, \|x\|_2^2.
\end{align*}
Taking the square root and assuming $x\neq 0$ we get
$$
   \|Ax\|_2/\|x\|_2 \leq \rho_A.
$$
Now there exists $k\in [1,n]$ such that $|\lambda_k| = \rho_A$. Let
$x=u_k$, i.e., $x$ is an eigenvector corresponding to $\lambda_k$.
Then $c_k=1$ and $c_i=0$ for all $i\neq k$. Therefore
$$
   \|Ax\|_2^2 = \lambda_k^2=\rho_A^2.
$$
Combining the above estimates gives us
$$
   \|A\|_2 = \max_{x\neq 0} \|Ax\|_2/\|x\|_2 = \rho_A
$$
as desired. \qed\medskip


The next theorem is particularly important.

\subsection{Theorem on the 2-norm of matrices} \label{12.15}

For any matrix $A\in\IC^{m\times n}$ we have
$$
    \|A\|_2^2 = \|A^{\ast}\|_2^2
    = \|A^{\ast}A\|_2 = \|AA^{\ast}\|_2
    =\lambda_{\max}
$$
where $\lambda_{\max}$ denotes the largest eigenvalue of both
$A^{\ast}A$ and $AA^{\ast}$.

\medskip{\footnotesize Note: This theorem gives a practical method for computing $\|A\|_2$
for small matrices, where $m=2$ or $n=2$. In that case one of the
two matrices $A^{\ast}A$ and $AA^{\ast}$ is of size $2\times 2$,
thus its eigenvalues are the roots of a quadratic polynomial.}

\medskip\noindent \emph{Proof of Theorem \ref{12.15}} is long and will be
done in four steps. In the proof, $\|\cdot\|$ will always denote the
2-norm.

\medskip\noindent\textbf{Lemma}. For every vector $z\in{\mathbb C}^n$ we
have $\|z\|=\max_{\|y\|=1}|\langle y,z\rangle |$.

\medskip\noindent{\em Proof}. Indeed, by the Cauchy-Schwarz inequality
$$
  |\la y,z\ra |\leq \la y,y\ra^{1/2}\la z,z\ra^{1/2} = \|z\|
$$
and the equality is attained whenever $y$ is parallel to $z$. So we can
set $y=\pm\frac{z}{\|z\|}$ and achieve the maximum. $\Box$ \medskip
\medskip

\noindent\textbf{Step 1}. To prove that $\|A\| = \|A^{\ast}\|$ we write
$$
  \|A\| = \sup_{\|x\|=1} \|Ax\| =
  \sup_{\|x\|=1}\sup_{\|y\|=1}|\langle y,Ax\rangle |=
  \sup_{\|x\|=1}\sup_{\|y\|=1}|\langle A^\ast y,x\rangle |
$$
$$
  =\sup_{\|y\|=1}\sup_{\|x\|=1}|\langle x, A^\ast y\rangle |
   = \sup_{\|y\|=1} \|A^\ast y\| =\|A^\ast\|.
$$

\medskip\noindent\textbf{Step 2}. To prove that $\|A\|^2 =
\|A^{\ast}A\|$ we write
$$
  \|A^\ast A\| = \sup_{\|x\|=1} \|A^\ast Ax\| =
  \sup_{\|x\|=1}\sup_{\|y\|=1}|\langle y,A^\ast Ax\rangle |=
  \sup_{\|x\|=1}\sup_{\|y\|=1}|\langle Ay,Ax\rangle |.
$$
Then we again use the Cauchy-Schwarz inequality:
$$
   |\langle Ay,Ax\rangle |\leq \|Ax\|\,\|Ay\| \leq \|A\|\,\|A\|=\|A\|^2
$$
hence $\|A^\ast A\| \leq \|A\|^2$. On the other hand,
$$
  \|A^\ast A\| = \sup_{\|x\|=1}\sup_{\|y\|=1}|\langle Ay,Ax\rangle |
  \geq \sup_{\|x\|=1}|\langle Ax,Ax\rangle |=\|A\|^2.
$$
Combining the upper and lower bounds gives us  $\|A^\ast A\| =
\|A\|^2$.

\medskip\noindent\textbf{Step 3}. Using an obvious symmetry we conclude that
$\|A^\ast \|^2 = \|A A^\ast \|$.


\medskip\noindent\textbf{Step 4}. According to Section~\ref{12.10aa}, we have
$$
  \|A^\ast A\|_2 = \max |\lambda_i(A^\ast A)|.
$$
Recall that $A^\ast A$ is a Hermitian positive semi-definite matrix, so
its eigenvalues $\lambda_i(A^\ast A)$ are real and $\geq 0$, hence
$\max |\lambda_i(A^\ast A)| = \lambda_{\max}(A^\ast A)$, the largest
eigenvalue of $A^\ast A$. The same argument applies to $AA^\ast$. In
particular, we see that
$$
 \lambda_{\max}(A^\ast A)=\lambda_{\max}(AA^\ast ).
$$
This completes the proof of Theorem~\ref{12.15}. \qed

\subsection{Example}

Let $A=\begin{bmatrix} 3\\ 4\end{bmatrix}\in \IC^{2\times 1}$. For
any unit vector $x\in \IC^1$ we can write $x=[e^{{\mathbf
i}\theta}]$, therefore $Ax=\begin{bmatrix} 3e^{{\mathbf i}\theta}\\
4e^{{\mathbf i}\theta}\end{bmatrix}$ and
$$
   \|Ax\|=\sqrt{|3e^{{\mathbf i}\theta}|^2+|4e^{{\mathbf i}\theta}|^2} = \sqrt{3^2+4^2}=5
$$
which implies $\|A\|=5$. Now let us find the norm of $A^\ast =
\begin{bmatrix} 3 & 4\end{bmatrix}\in \IC^{1\times 2}$, i.e.,
$\|A^\ast\| = \sup_{\|y\|=1}\|A^\ast y\|$. For simplicity, we will
only use \emph{real} unit vectors $y\in \IC^2$, which can be
described by $y=\begin{bmatrix} \cos\varphi \\
\sin\varphi \end{bmatrix}$ for $\varphi\in [0,2\pi]$. We have
$A^\ast y = 3\cos\varphi+4\sin\varphi$, thus $\|A^\ast
y\|=|3\cos\varphi+4\sin\varphi|$. Finding the maximum of this
function (over the interval $0\leq \varphi \leq 2\pi$) is a
Calculus-I problem: the maximum is achieved at $\cos\varphi=\pm 3/5$
and $\sin\varphi = \pm 4/5$, and we get
$$
   \|A^\ast\| = \max_{\|y\|=1}\|A^\ast y\| = \frac 95 + \frac{16}{5} = \frac{25}{5}=5.
$$
We see that, indeed, $\|A\| = \|A^\ast\|$. Note that $A^\ast A =
[25]\in \IC^{1\times 1}$, so obviously $\|A^\ast A\|=25$, in full
agreement with Theorem~\ref{12.15}.

\subsection{Corollary for the 2-norm of matrices} \label{12.15aa}

Let $\lambda_{\max}$ again denote the largest eigenvalue of $A^\ast A$.
Then we have
$$
      \|Ax\|_2=\|A\|_2\|x\|_2
      \qquad \Longleftrightarrow \qquad
      A^{\ast}Ax=\lambda_{\max}x.
$$
Hence, the supremum in Section~\ref{8.8} is attained (on the
eigenvectors of $A^\ast A$ corresponding to $\lambda_{\max}$) and can
be replaced by maximum. Moreover, this implies that the 2-norm of a
real matrix is the same, whether it is computed in the complex space or
in the real space.

\medskip\noindent \emph{Proof}. On the one hand
$$
      \|Ax\|_2^2 = \langle Ax,Ax\rangle = \langle A^\ast Ax,x\rangle
$$
and on the other hand
$$
         \|A\|^2 = \lambda_{\max},
$$
so for any vector $x$ with $\|x\|=1$ we have
$$
      \|Ax\|_2^2=\|A\|_2^2
      \qquad \Longleftrightarrow \qquad
      \langle A^{\ast}Ax,x\rangle=\lambda_{\max},
$$
and lastly we use Section~\ref{12.10bb}. \hfill $\Box$

\begin{exercise}
Let $A\in\IC^{n\times n}$ satisfy $A^\ast=-A$. Show that the matrix
$I-A$ is invertible. Then show that the matrix $(I-A)^{-1}(I+A)$ is
unitary.
\end{exercise}

\begin{exercise}
Let $A=(a_{ij})$ be a complex $n\times n$ matrix. Assume
that $\langle Ax,x\rangle =0$ for all $x\in{\mathbb C}^n$. Prove that\\
(a) $a_{ii}=0$ for $1\leq i\leq n$ by substituting $x=e_i$\\ (b)
$a_{ij}=0$ for $i\neq j$ by substituting $x=pe_i+qe_j$ then using (a)
and putting $p,q=\pm 1,\pm {\bf i}$ (here ${\bf i}=\sqrt{-1}$) in
various combinations.\\ Conclude that $A=0$.
\end{exercise}

\begin{exercise}
Let $A,B$ be complex $n\times n$ matrices such that $\la Ax,x\ra=\la
Bx,x\ra$ for all $x\in \IC^n$. Use the previous exercise to prove
that $A=B$.
\end{exercise}

\begin{exercise}
Find a real $2\times 2$ matrix $A\neq 0$ such that $\langle
Ax,x\rangle=0$ for all $x\in{\mathbb R}^2$. Thus find two real $2\times
2$ matrices $A$ and $B$ such that $\la Ax,x\ra=\la Bx,x\ra$ for all
$x\in \IR^2$, but $A\neq B$.
\end{exercise}

\begin{exercise}
Find a real $2\times 2$ matrix $A$ such that $\langle Ax,x\rangle>0$
for all $x\in{\mathbb R}^2$, but $A$ is not positive definite.
\end{exercise}

\newpage

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\refstepcounter{section} \label{Ch5}

\centerline{Chapter \ref{Ch5} }\vspace{5mm}

\centerline{\Large\bf Singular Value Decomposition}\vspace{3mm}

\addcontentsline{toc}{section}{\ref{Ch5}. Singular Value Decomposition}

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\setcounter{subsection}{0} \setcounter{exercise}{0}
\setcounter{equation}{0}

Recall that every matrix $A\in\IC^{m\times n}$ defines a linear
transformation $\IC^n\to\IC^m$. Let $B$ be an ONB in $\IC^n$ and
$B'$ be ONB in $\IC^m$. Then $T$ is represented in the bases $B$ and
$B'$ by the matrix $U^\ast AV$, where $U\in\IC^{m\times m}$ and
$V\in\IC^{n\times n}$ are unitary matrices. The following theorem
shows that one can always find two bases $B$ and $B'$ so that the
matrix $U^\ast AV$ will be diagonal. This can be written as $U^\ast
AV=D$ or equivalently, $A=UDV^{\ast}$.

\begin{center}
\begin{tikzpicture}
  \node[anchor=west] at (-3,0.5) {\footnotesize SVD ($m>n$)};
  \draw[thick,fill=gray!30] (0,-0.5) -- (0,1) -- (1,1) -- (1,-0.5) -- cycle;
  \node[anchor=west] at (0.95,0.5) {$=$};
  \node[anchor=north] at (0.5,-0.5) {$A$};
  \begin{scope}[xshift=1.5cm]
  \draw[thick,fill=gray!30] (0,-0.5) -- (0,1) -- (1.5,1) -- (1.5,-0.5) -- cycle;
  \node[anchor=west] at (1.45,0.5) {$\times$};
  \node[anchor=north] at (0.75,-0.5) {$U$};
  \end{scope}
  \begin{scope}[xshift=3.5cm]
  \fill[gray!30] (0,0.9)--(0,1)--(0.1,1)--(1,0.1)--(1,0)--(0.9,0)--cycle;
  \draw[thick] (0,-0.5) -- (0,1) -- (1,1) -- (1,-0.5) -- cycle;
  \node[anchor=west] at (0.95,0.5) {$\times$};
  \node[anchor=south west] at (0,-0.3) {$0$};
  \node[anchor=north east] at (1,1) {$0$};
  \node[anchor=north] at (0.5,-0.5) {$D$};
  \end{scope}
  \begin{scope}[xshift=5cm]
  \draw[thick,fill=gray!30] (0,0) -- (0,1) -- (1,1) -- (1,0) -- cycle;
  \node[anchor=north] at (0.5,-0) {$V^\ast$};
  \end{scope}
\end{tikzpicture}
$\phantom{mmmmmm}$
\end{center}

\begin{center}
\begin{tikzpicture}
  \node[anchor=west] at (-3.5,0.5) {\footnotesize SVD ($m<n$)};
  \draw[thick,fill=gray!30] (-0.5,0) -- (-0.5,1) -- (1,1) -- (1,0) -- cycle;
  \node[anchor=west] at (0.95,0.5) {$=$};
  \node[anchor=north] at (0.25,0) {$A$};
  \begin{scope}[xshift=1.5cm]
  \draw[thick,fill=gray!30] (0,0) -- (0,1) -- (1,1) -- (1,0) -- cycle;
  \node[anchor=west] at (0.95,0.5) {$\times$};
  \node[anchor=north] at (0.5,0) {$U$};
  \end{scope}
  \begin{scope}[xshift=3cm]
  \fill[gray!30] (0,0.9)--(0,1)--(0.1,1)--(1,0.1)--(1,0)--(0.9,0)--cycle;
  \draw[thick] (0,0) -- (0,1) -- (1.5,1) -- (1.5,0) -- cycle;
  \node[anchor=west] at (1.45,0.5) {$\times$};
  \node[anchor=south west] at (0,0) {$0$};
  \node[anchor=north east] at (1.3,1) {$0$};
  \node[anchor=north] at (0.75,0) {$D$};
  \end{scope}
  \begin{scope}[xshift=5cm]
  \draw[thick,fill=gray!30] (0,-0.5) -- (0,1) -- (1.5,1) -- (1.5,-0.5) -- cycle;
  \node[anchor=north] at (0.75,-0.5) {$V^\ast$};
  \end{scope}
\end{tikzpicture}
$\phantom{mmmmmm}$
\end{center}
\vspace{-5mm}\noindent Note: $D\in \IC^{m\times n}$ is said to be
diagonal if $D_{ij}=0$ for $i\neq j$. It has exactly $p=\min\{m,n\}$
diagonal entries and can be denoted by
$D=\,$diag$\{d_1,\ldots,d_p\}$.

\subsection{Singular value decomposition (SVD)}
\label{17.8}

Let $A\in\IC^{m\times n}$ and denote $p=\min\{m,n\}$. Denote the
rank of $A$ by $r$ ($0\leq r\leq p$). Then there are unitary
matrices $U\in\IC^{m\times m}$ and $V\in\IC^{n\times n}$ and a real
diagonal matrix $D=\, {\rm diag}
\{\sigma_1,\ldots,\sigma_p\}\in\IR^{m\times n}$ such that
\beq  \label{SVD}
         A=UDV^{\ast}
\eeq
and
$$
   \sigma_1\geq\cdots\geq \sigma_r>\sigma_{r+1}=\cdots
   =\sigma_p=0,
$$
The matrix $D$ is uniquely determined by $A$.\medskip

\noindent{\footnotesize $\ \ $The matrices $U$ and $V$ are
\emph{not} unique, for every $A$ there are many choices of $U$ and
$V$.}

\proof Observe that \eqref{SVD} is equivalent to
$A^{\ast}=VD^TU^{\ast}$, which is an SVD for $A^\ast$. Therefore,
$A$ has an SVD if and only if $A^{\ast}$ does. This allows us to
assume that $m\geq n$, without loss of generality.

We now fix $l=m-n\geq 0$ and use induction on $n$ (for every $n\geq 1$
we automatically take $m=n+l$).

Denote $\sigma_1=\|A\|_2$. If $\sigma_1=0$, then $A$ is a zero matrix
($a_{ij}=0$ for all $i,j$), and an SVD is constructed trivially: $D$ is
a zero matrix and $U$ and $V$ are arbitrary unitary matrices. So let us
assume that $\sigma_1>0$.

Due to Corollary~\ref{12.15aa}, there is a unit vector $v_1\in\IC^n$
such that $\|Av_1\|_2 = \|A\|_2=\sigma_1$. Then the vector
$u_1=Av_1/\sigma_1$ is a unit vector in $\IC^m$.

According to Section~\ref{ONBexist}, we can extend $v_1$ to an ONB
$\{v_1,v_2',\ldots,v_n'\}$ in $\IC^n$ and $u_1$ to an ONB
$\{u_1,u_2',\ldots,u_m'\}$ in $\IC^m$. Let $V_1$ denote the unitary
matrix whose columns are $v_1,v_2',\ldots,v_n'$ and $U_1$ denote the
unitary matrix whose columns are $u_1,u_2',\ldots,u_m'$. Then the
product $S=U_1^{\ast}AV_1$ has the following structure (as one can
verify directly):
 \beq
   U_1^{\ast}AV_1=S=\left[\begin{array}{cc}
   \sigma_1 & w^{\ast} \\ 0 & B \end{array}\right]
     \tag{$\ast$}
 \eeq
If $n=1$, then $S=\left[\begin{smallmatrix}\sigma_1\\ 0
\end{smallmatrix}\right]$ is a diagonal matrix yielding an SVD
$$
  A = U_1SV_1^\ast
$$
for $A$. This is the basis for our induction.

Suppose now $n>1$, observe that
$$
   \|S\|_2 = \|S^\ast\|_2 \geq    \left\|\left[\begin{array}{cc}
   \sigma_1 & 0 \\ w & B^\ast \end{array}\right]
   \left[\begin{array}{c}
   1 \\ 0 \end{array}\right]\right\|_2
   = \left\|\left[\begin{array}{c}
   \sigma_1 \\ w \end{array}\right]\right\|_2
  =\sqrt{\sigma_1^2+w^\ast w}.
$$
On the other hand, the matrices $U_1^\ast$ and $V_1$ are unitary, hence
we have
$$
    \|S\|_2=\|U_1^{\ast}AV_1\|_2=\,\text{(Exercise~\ref{Ex222})}\,=
    \|A\|_2=\sigma_1.
$$
Comparing the above two formulas shows that $w=0$. The matrix $B$ has
dimensions $(m-1)\times (n-1)$, thus by our inductive assumption it has
an SVD, let us denote it by $B=\hat{U}\hat{D}\hat{V}^\ast$. Now it can
be easily verified that
$$
   A=\underbrace{U_1\begin{bmatrix} 1 & 0 \\ 0 & \hat{U}
   \end{bmatrix}}_{U}
   \underbrace{\begin{bmatrix} \sigma_1 & 0 \\ 0 & \hat{D}
   \end{bmatrix}}_{D}
   \underbrace{\begin{bmatrix} 1 & 0 \\ 0 & \hat{V}^\ast
   \end{bmatrix}V_1^\ast}_{V^\ast}
$$
which gives us an SVD for $A$.

To prove the uniqueness of $D$, observe that
$$
  A^\ast A= VD^TDV^\ast
  \qquad\text{and}\qquad
  AA^\ast=UDD^TU^\ast
$$
i.e., $\sigma_1^2,\ldots,\sigma_p^2$ are the eigenvalues of both
$A^\ast A$ and $AA^\ast$, hence they are uniquely determined by $A$.
\qed

\medskip Incidentally, the last part of the proof shows that the matrices
$A^\ast A$ and $AA^\ast$ have common non-zero eigenvalues.\medskip

Note also that the columns of $U$ are eigenvectors of $AA^\ast$ and the
columns of $V$ are eigenvectors of $A^\ast A$, according to
Section~\ref{11.22a}.

\subsection{Singular values and singular vectors}
\label{17.9}

\begin{itemize}
\item The positive numbers $\sigma_1,\ldots,\sigma_r$ are called {\em
    singular values} of $A$.
\item The columns $v_1,\ldots,v_n$ of the matrix $V$ (not those of
    $V^\ast$) are called {\em right singular vectors} for $A$.
\item The columns $u_1,\ldots,u_m$ of the matrix $U$ are called {\em
    left singular vectors} for $A$.
\end{itemize}

\subsection{Real SVD}

If $A\in\IR^{m\times n}$ is a \emph{real} matrix, then it has a
\emph{real} SVD
 \beq
            A=UDV^T
 \eeq
where $U\in\IR^{m\times m}$ and $V\in\IR^{n\times n}$ are \emph{real}
orthogonal matrices.\medskip

\noindent{\footnotesize \emph{Proof} is just a repetition of the proof
in Section~\ref{17.8}, except $v_1,v_2',\ldots,v_n'$ and
$u_1,u_2',\ldots,u_m'$ must be real vectors. \qed }

\medskip The following diagram illustrates the SVD; see also
Section~\ref{17.10}:\vspace{-1mm}

\begin{center}
\begin{tikzpicture}[scale=0.85]
         \draw[thin] (-1.3,0)--(1.3,0);
         \draw[thin] (0,-1.2)--(0,1.3);
         \draw[thin] (0,0) circle(1);
         \draw[thick,->,-latex] (0,0)--(0.97,0.17);
         \node[anchor=south west] at (0.9,0.1) {\tiny $v_1$};
         \draw[thick,->,-latex] (0,0)--(-0.17,0.97);
         \node[anchor=south east] at (-0.1,0.9) {\tiny $v_2$};
         \draw[line width=0.8mm,gray,->,-latex] (1.5,0.9)--(2.5,0.9);
         \node[anchor=south] at (1.9,0.85) {\tiny $V^\ast$};
         \begin{scope}[xshift=4cm]
         \draw[thin] (-1.3,0)--(1.3,0);
         \draw[thin] (0,-1.2)--(0,1.3);
         \draw[thin] (0,0) circle(1);
         \draw[thick,->,-latex] (0,0)--(1,0);
         \node[anchor=south west] at (0.9,-0.1) {\tiny $e_1$};
         \draw[thick,->,-latex] (0,0)--(0,1);
         \node[anchor=south east] at (0,0.9) {\tiny $e_2$};
         \draw[line width=0.8mm,gray,->,-latex] (1.5,0.9)--(2.5,0.9);
         \node[anchor=south] at (1.9,0.85) {\tiny $D$};
         \end{scope}
         \begin{scope}[xshift=8cm]
         \draw[thin] (-1.5,0)--(1.5,0);
         \draw[thin] (0,-1.2)--(0,1.3);
         \draw[thin] (0,0) ellipse(1.4 and 0.7);
         \draw[thick,->,-latex] (0,0)--(1.4,0);
         \node[anchor=south west] at (1.25,-0.15) {\tiny $\sigma_1e_1$};
         \draw[thick,->,-latex] (0,0)--(0,0.7);
         \node[anchor=south east] at (0.09,0.6) {\tiny $\sigma_2e_2$};
         \draw[line width=0.8mm,gray,->,-latex] (1.5,0.9)--(2.5,0.9);
         \node[anchor=south] at (1.9,0.85) {\tiny $U$};
         \end{scope}
         \begin{scope}[xshift=12cm]
         \draw[thin] (-1.5,0)--(1.5,0);
         \draw[thin] (0,-1.2)--(0,1.3);
         \draw[thin,rotate=-20] (0,0) ellipse(1.4 and 0.7);
         \draw[thick,->,-latex,rotate=-20] (0,0)--(1.4,0);
         \node[anchor=west] at (1.2,-0.5) {\tiny $\sigma_1u_1$};
         \draw[thick,->,-latex,rotate=-20] (0,0)--(0,0.7);
         \node[anchor=south] at (0.4,0.6) {\tiny $\sigma_2u_2$};
         \end{scope}
         \draw[dashed,gray,very thick,->,-latex] (1,1.4) arc (100:80:30);
         \node[anchor=south] at (6,1.85) {\tiny $A$};
\end{tikzpicture}
\end{center}

\newpage

\subsection{SVD analysis} \label{17.10}

For $1\leq i\leq r$ we have
$$
       Av_i=\sigma_i u_i,\qquad\qquad
       A^{\ast}u_i=\sigma_i v_i.
$$
We also have
\begin{align*}
   {\rm Ker}\, A&=\,{\rm span}\{v_{r+1},\ldots,v_n\},&
      \qquad
   {\rm Ker}\, A^{\ast}&=\,{\rm span}\{u_{r+1},\ldots,u_m\}\\
   {\rm Range}\, A&=\,{\rm span}\{u_1,\ldots,u_r\},&
      \qquad
   {\rm Range}\, A^{\ast}&=\,{\rm span}\{v_1,\ldots,v_r\}
\end{align*}
and
$$
   {\rm rank}\,A =\,{\rm rank}\, A^\ast = r.
$$
Here is a diagram illustrating the previous relations:
$$
  \begin{array}{lclcl}
    & A & & A^{\ast} & \\ & & & & \\
   v_1 & \stackrel{\times\sigma_1}{\xrightarrow{\hspace*{8mm}}} & u_1 &
         \stackrel{\times\sigma_1}{\xrightarrow{\hspace*{8mm}}} & v_1 \\
   v_2 & \stackrel{\times\sigma_2}{\xrightarrow{\hspace*{8mm}}} & u_2 &
         \stackrel{\times\sigma_2}{\xrightarrow{\hspace*{8mm}}} & v_2 \\
    \vdots & \vdots & \vdots & \vdots & \vdots \\
   v_r & \stackrel{\times\sigma_r}{\xrightarrow{\hspace*{8mm}}} & u_r &
         \stackrel{\times\sigma_r}{\xrightarrow{\hspace*{8mm}}} & v_r \\
   v_{r+1} & \rightarrow 0\phantom{n} & u_{r+1} & \rightarrow 0\phantom{n} & \\
    \vdots & \vdots\phantom{n} & \vdots & \vdots\phantom{n} & \\
   v_n & \rightarrow 0\phantom{n} & u_m & \rightarrow 0\phantom{n} &
   \end{array}
$$

\subsection{Useful relations - I}\label{UsefulRel1}

For any matrix $A\in\IC^{m\times n}$, an SVD for $A^\ast$ is
$$
    A^{\ast}=VD^TU^{\ast},
$$
hence $A$ and $A^\ast$ have the same singular values. In particular,
$$
   \|A\|_2 = \|A^\ast\|_2 = \|D\|_2 = \|D^T\|_2 = \sigma_1.
$$
If $A\in\IC^{n\times n}$ is a \emph{square} invertible matrix, then
$$
   A^{-1} = VD^{-1}U^{\ast}
$$
where $D^{-1}=\,{\rm diag}\{\sigma_1^{-1},\ldots,\sigma_n^{-1}\}$, and
$$
    \|A^{-1}\|_2 = \|D^{-1}\|_2 = \sigma_n^{-1}.
$$
If $A\in\IC^{n\times n}$ is a Hermitian matrix with eigenvalues
$\lambda_1,\ldots,\lambda_n$, then its singular values are
$|\lambda_1|,\ldots,|\lambda_n|$ (this follows from
Section~\ref{11.21}).

\subsection{Computation of SVD for small matrices}  \label{ComSVD}

One can manually compute an SVD of an $m\times 2$ matrix as follows.
First we form the $2\times 2$ matrix $A^\ast A$. Then we find its
eigenvalues $\lambda_1\geq\lambda_2\geq 0$ by solving the corresponding
quadratic equation. (Since $A^\ast A$ is a Hermitiam positive
semi-definite matrix, its eigenvalues are real non-negative numbers.)
Then we find the corresponding unit eigenvectors $v_1,v_2$ of the
matrix $A^\ast A$. Then we compute $\sigma_1=\sqrt{\lambda_1}$,
$\sigma_2=\sqrt{\lambda_2}$ and $u_1=\sigma_1^{-1} Av_1$,
$u_2=\sigma_2^{-1} Av_2$. (Recall that $u_1$ and $u_2$ must be unit
vectors orthogonal to each other.) Lastly, if $m>2$, we extend the
orthonormal set $\{u_1,u_2\}$ to an ONB in $\IC^m$ arbitrarily.

\subsection{Reduced SVD} \label{SVDred}

Let $A\in\IC^{m\times n}$ with $m>n$. Then there is a matrix
$\hat{U}\in\IC^{m\times n}$ with orthonormal columns, a unitary matrix
$V\in\IC^{n\times n}$ and a square diagonal matrix $\hat{D}=\, {\rm
diag} \{\sigma_1,\ldots,\sigma_n\}$ such that
 \beq  \label{reducedSVD}
     A=\hat{U}\hat{D}V^{\ast}.
 \eeq
For $m<n$, the reduced SVD is similar: there exist a unitary matrix
$U\in\IC^{m\times m}$, a matrix $\hat{V}\in\IC^{n\times m}$ with
orthonormal columns, and a square diagonal matrix $\hat{D}=\, {\rm
diag} \{\sigma_1,\ldots,\sigma_m\}$ such that
 \beq  \label{reducedSVD2}
     A=U\hat{D}\hat{V}^{\ast},
 \eeq

\proof Suppose $m>n$. Then we take the (full) SVD $A=UDV^{\ast}$ given
by \eqref{SVD} and erase the last $m-n$ columns of $U$ and the bottom
$m-n$ rows of $D$ (consisting of zeros). For $m<n$, the construction is
similar.
 \qed\medskip

\begin{center}
\begin{tikzpicture}
  \node[anchor=west] at (-3,0.5) {\footnotesize Reduced SVD};
  \node[anchor=west] at (-2.6,0.1) {\footnotesize ($m>n$)};
  \draw[thick,fill=gray!30] (0,-0.5) -- (0,1) -- (1,1) -- (1,-0.5) -- cycle;
  \node[anchor=west] at (0.95,0.5) {$=$};
  \node[anchor=north] at (0.5,-0.61) {$A$};
  \begin{scope}[xshift=1.5cm]
  \draw[dashed,fill=gray!10] (0,-0.5) -- (0,1) -- (1.5,1) -- (1.5,-0.5) -- cycle;
  \draw[thick,fill=gray!30] (0,-0.5) -- (0,1) -- (1,1) -- (1,-0.5) -- cycle;
  \node[anchor=west] at (1.45,0.5) {$\times$};
  \node[anchor=north] at (0.5,-0.5) {$\hat{U}$};
  \end{scope}
  \begin{scope}[xshift=3.5cm]
  \fill[gray!30] (0,0.9)--(0,1)--(0.1,1)--(1,0.1)--(1,0)--(0.9,0)--cycle;
  \draw[dashed] (0,-0.5) -- (0,1) -- (1,1) -- (1,-0.5) -- cycle;
  \draw[thick] (0,0) -- (0,1) -- (1,1) -- (1,0) -- cycle;
  \node[anchor=west] at (0.95,0.5) {$\times$};
  \node[anchor=south west] at (0,0) {$0$};
  \node[anchor=north east] at (1,1) {$0$};
  \node[anchor=north] at (0.5,-0.5) {$\hat{D}$};
  \end{scope}
  \begin{scope}[xshift=5cm]
  \draw[thick,fill=gray!30] (0,0) -- (0,1) -- (1,1) -- (1,0) -- cycle;
  \node[anchor=north] at (0.5,-0) {$V^\ast$};
  \end{scope}
\end{tikzpicture}
$\phantom{mmmmmm}$
\end{center}

\begin{center}
\begin{tikzpicture}
  \node[anchor=west] at (-3.5,0.5) {\footnotesize Reduced SVD};
  \node[anchor=west] at (-3.1,0.1) {\footnotesize ($m<n$)};
  \draw[thick,fill=gray!30] (-0.5,0) -- (-0.5,1) -- (1,1) -- (1,0) -- cycle;
  \node[anchor=west] at (0.95,0.5) {$=$};
  \node[anchor=north] at (0.25,-0.11) {$A$};
  \begin{scope}[xshift=1.5cm]
  \draw[thick,fill=gray!30] (0,0) -- (0,1) -- (1,1) -- (1,0) -- cycle;
  \node[anchor=west] at (0.95,0.5) {$\times$};
  \node[anchor=north] at (0.5,-0.11) {$U$};
  \end{scope}
  \begin{scope}[xshift=3cm]
  \fill[gray!30] (0,0.9)--(0,1)--(0.1,1)--(1,0.1)--(1,0)--(0.9,0)--cycle;
  \draw[dashed] (0,0) -- (0,1) -- (1.5,1) -- (1.5,0) -- cycle;
  \draw[thick] (0,0) -- (0,1) -- (1,1) -- (1,0) -- cycle;
  \node[anchor=west] at (1.45,0.5) {$\times$};
  \node[anchor=south west] at (0,0) {$0$};
  \node[anchor=north east] at (1,1) {$0$};
  \node[anchor=north] at (0.5,0) {$\hat D$};
  \end{scope}
  \begin{scope}[xshift=5cm]
  \draw[dashed,fill=gray!10] (0,-0.5) -- (0,1) -- (1.5,1) -- (1.5,-0.5) -- cycle;
  \draw[thick,fill=gray!30] (0,0) -- (0,1) -- (1.5,1) -- (1.5,0) -- cycle;
  \node[anchor=north] at (0.75,-0.5) {$\hat{V}^\ast$};
  \end{scope}
\end{tikzpicture}
$\phantom{mmmmmm}$
\end{center}

\subsection{Rank-one expansion} \label{17.11}

We have the following expansion of $A$:
 \beq \label{SVDexpansion}
       A=\sum_{i=1}^r \sigma_iu_iv_i^{\ast}
 \eeq
Each term of this sum is an $m\times n$ matrix of rank one
(Exercise~\ref{Exrankone}).

\proof It is enough to observe that for every $v_j$, $1\leq j\leq n$,
$$
       \biggl(\sum_{i=1}^r \sigma_iu_iv_i^{\ast}\biggr)v_j=\sigma_ju_j=Av_j
$$
because $v_i^{\ast}v_j=\delta_{ij}$ (the Kronecker delta symbol defined
in Sect.~\ref{Orthonormalsets}). \qed

\subsection{Useful relations - II} \label{Energy}

Recall the Frobenius norm of a matrix (Section~\ref{FrobNorm}). We have
$$
   \|A\|_F^2 =\,\text{(Exercise~\ref{ExFFF})}\,=
   \|D\|_F^2=\sigma_1^2+\cdots+\sigma_r^2.
$$
Comparing this with Section~\ref{UsefulRel1} gives
\beq \label{A2<AF}
   \|A\|_2 \leq \|A\|_F,
\eeq
{\footnotesize The value of $\|A\|_F^2$ can be interpreted as the
\emph{mass} (or {\em energy}) of the matrix $A$. The mass is
conserved under multiplication by unitary matrices
(Exercise~\ref{ExFFF}), and the SVD pulls all the mass of a matrix
onto its diagonal.}

For any $1\leq k\leq r$, let $A_k$ denote the partial $k$-sum of
\eqref{SVDexpansion}:
 \beq \label{Ak}
       A_k=\sum_{i=1}^k \sigma_iu_iv_i^{\ast}.
 \eeq
It is easy to see that $A_kv_i = \sigma_iu_i$ for $i=1,\ldots,k$ and
$A_kv_i=0$ for $i>k$, therefore rank$\,A_k=k$.

Comparing \eqref{Ak} with \eqref{SVDexpansion} we see that $A_k$ has
singular values $\sigma_1,\ldots,\sigma_k$ and singular vectors
$u_1,\ldots,u_k$ and $v_1,\ldots,v_k$. Thus it agrees with $A$ on
the $k$ largest singular values and the corresponding singular
vectors.

Also, for any unit vectors $u$ and $v$ we have $\|uv^{\ast}\|_F=1$
(Exercise~\ref{Exrankone}), hence
$\|\sigma_iu_iv_i^{\ast}\|_F^2=\sigma_i^2$ and
$\|A_k\|_F^2=\sigma_1^2+ \cdots +\sigma_k^2$.

\subsection{Low-rank approximation}

For each $1\leq k\leq r$ we have
 \beq \label{Akapprox}
   \sigma_{k+1} = \|A-A_k\|_2 =
   \inf_{\substack{B\in\IC^{m\times n}\\
  {\rm rank}\,B\leq k}} \|A-B\|_2
 \eeq
(for $k=r$, we assume here that $\sigma_{r+1}=0$).

Thus, $A_k$ is the best approximation to $A$ by matrices of rank
$\leq k$.

\proof Note that
 \beq \label{Ar-k}
       A-A_k=\sum_{i=k+1}^r \sigma_iu_iv_i^{\ast}
 \eeq
hence the singular values of $A-A_k$ are
$\sigma_{k+1},\ldots,\sigma_r$, of which $\sigma_{k+1}$ is the
largest. Thus $\|A-A_k\|_2 = \sigma_{k+1}$, according to
Sect.~\ref{UsefulRel1}.

Now suppose that there is some matrix $B$ with rank$\, B\leq k$ such
that $\|A-B\|_2<\sigma_{k+1}$. For each nonzero vector $v\in\,{\rm
Ker}\, B$ we have
 \beq \label{ABv2}
   \|Av\|_2=\|(A-B)v\|_2\leq
   \|A-B\|_2\|v\|_2<\sigma_{k+1}\|v\|_2.
 \eeq
On the other hand, let us consider the subspace $L=\,{\rm
span}\{v_1,\ldots,v_{k+1}\}$. For any nonzero vector $v\in L$ we
have
$$
  v=c_1v_1+\cdots+c_{k+1}v_{k+1}.
$$
Due to Section~\ref{17.10}
$$
  Av_2 =c_1\sigma_1u_1+\cdots+c_{k+1}\sigma_{k+1}u_{k+1},
$$
therefore by \eqref{ParcevalNorm}
 \beq \label{ABv3}
  \|Av\|_2^2 = \bigl(\sigma_1^2|c_1|^2+\cdots+\sigma_{k+1}^2|c_{k+1}|^2\bigr)
  \geq \sigma_{k+1}^2\|v\|_2^2.
 \eeq
Note that ${\rm dim}\, L=(k+1)$. On the other hand, due to
\eqref{RKA} we have
$$
  {\rm dim} ({\rm Ker}\, B)=n-\,{\rm rank}\, B\geq n-k.
$$
Since $(k+1) + (n-k)>n$, the subspaces $L$ and Ker$\,B$ must have a
common nonzero vector $v$. That vector must satisfy both \eqref{ABv2}
and \eqref{ABv3}, which is impossible $\Rightarrow$ a contradiction.
\qed

\subsection{Distance to the nearest singular matrix}\label{17.20}

Let $A\in\IC^{n\times n}$ be a nonsingular square matrix. Then
$$
   \min\left\{\|A-A_s\|_2\colon\
   A_s\ {\rm is}\ {\rm singular}\right \}=
    \sigma_n,
$$
i.e., the smallest singular value $\sigma_n$ of $A$ is the
``distance'' from $A$ to the ``nearest'' singular matrix.

\proof Recall: $A_s$ is singular iff rank$A_s\leq n-1$, then use
\eqref{Akapprox}. \qed

\subsection{Small perturbations of matrices}

We say that a matrix $E \in \IC^{m\times n}$ is \emph{small} if all its
components are small. Equivalently, its norm $\|E\|_2$ is small. To see
the equivalence, note that if $|e_{ij}| \leq \varepsilon$ for all
$i,j$, then $\|E\|_2 \leq \|E\|_F \leq \varepsilon\sqrt{mn}$. On the
other hand, if $\|E\|_2\leq\varepsilon$, then $|e_{ij}|\leq \|E{\mathbf
e}_j\|_2\leq \|E\|_2\leq\varepsilon$ for any $i,j$ (here $\mathbf
e_{j}$ denotes the $j$th canonical basis vector.

For a matrix $A$ and a small matrix $E$, we call $A+E$ a \emph{small
perturbation} of $A$.

\subsection{Rank with tolerance $\varepsilon$} \label{17.17}

The {\em rank} of $A\in\IC^{m\times n}$ with {\em tolerance}
$\varepsilon>0$ (also called {\em numerical rank}) is defined by
$$
      {\rm rank}(A,\varepsilon)=
      \min_{\|E\|_2\leq\varepsilon}{\rm rank}(A+E).
$$

\medskip\noindent Note: rank$(A,\varepsilon)\leq\,$rank$\, A$. The
rank with tolerance $\varepsilon$ is the minimum rank of $A$ under
small perturbations by matrices that have 2-norm $\leq\varepsilon$.

If $A$ has full rank, i.e., rank$\,A=p=\min\{m,n\}$, but
rank$(A,\varepsilon)<p$ for a small $\varepsilon$, then $A$ is
``nearly rank deficient''.

\subsection{Computation of the numerical rank} \label{17.18}

The rank of $A$ with tolerance $\varepsilon$ equals the number of
singular values of $A$ (counted with multiplicity) that are
\emph{greater} than $\varepsilon$:
$$
  {\rm rank}(A,\varepsilon) = \{k\colon
  \sigma_k>\varepsilon,\ \sigma_{k+1}\leq \varepsilon\}.
$$
We also note that for each $\varepsilon>0$
$$
  {\rm rank}(A^{\ast},\varepsilon)=\,{\rm rank}(A,\varepsilon),
$$
because $A$ and $A^{\ast}$ have the same singular values
(Section~\ref{UsefulRel1}).\medskip

{\footnotesize The rest of this chapter uses certain topological terms
(open and dense sets, metric). For students who are not familiar with
these terms, explanations should be given, with simple examples.}

\subsection{Metric for matrices} \label{17.15}

We can measure the ``distance'' between matrices $A,B \in \IC^{m\times
n}$ as follows:
$$
   \text{dist}(A,B) = \|A-B\|_2
$$
Then the space of matrices $\IC^{m\times n}$ becomes a {\em metric
space}. As a result, topological notions, like open sets, dense sets,
etc., apply.

\subsection{Topological properties of full rank matrices} \label{17.16a}

Full rank matrices make an open and dense subset of $\IC^{m\times
n}$.\medskip

{\footnotesize Openness means that for any full rank matrix $A$ there
is an $\varepsilon>0$ such that all perturbations $A+E$ by small
matrices $E$ with norm $\|E\|_2< \varepsilon$ have full rank. In plain
words, any full rank matrix $A$ is ``surrounded'' by full rank
matrices.

Denseness means that for any rank-deficient matrix $A$ and any
$\varepsilon>0$ there exists a small matrix $E$ with norm $\|E\|_2 <
\varepsilon$ such that $A+E$ has full rank. In other words, full
rank matrices are scattered everywhere, they are in ``every corner''
of the space $\IC^{m\times n}$. Yet in other words, every matrix is
``arbitrarily close'' to full rank matrices.}

\proof Openness follows from Section~\ref{17.18}. Indeed, for any full
rank matrix $A$ we have $\sigma_p>0$, where $p=\min\{m,n\}$. Hence
every matrix $B$ such that $\|A-B\|_2<\sigma_p$ also has full rank.

To prove denseness, let $A$ be a rank deficient matrix and let
$A=UDV^{\ast}$ be its SVD. For any $\varepsilon>0$, we construct
$D_{\varepsilon} =\varepsilon I$ and $E=UD_{\varepsilon}V^{\ast}$.
Then $\|E\|_2=\|D_{\varepsilon}\|_2=\varepsilon$ and
$$
   {\rm rank}(A+E)=\,{\rm rank}\bigl(U(D+D_{\varepsilon})V^{\ast}\bigr)
   =\,{\rm rank}(D+D_{\varepsilon})=\min\{m,n\}
$$
so $A+E$ has full rank. \qed

\subsection{Topological property of diagonalizable matrices} \label{17.16b}

Diagonalizable matrices make a dense subset of $\IC^{n\times n}$.

\proof See Exercise~\ref{ExDMdense}.

\newpage

\begin{exercise} \label{Exrankone}
Let $x \in \IC^n$ and $y\in\IC^m$. Consider the $m\times n$ matrix
defined by $A=yx^{\ast}$. \begin{itemize} \item[(a)] Show that
rank$\,A=1$. \item[(b)] Show that $\|A\|_2=\|x\|_2\|y\|_2$. \item[(c)]
Show that $\|A\|_F=\|x\|_2 \|y\|_2$ (which is the same as $\|x\|_F
\|y\|_F$).
\end{itemize}
\end{exercise}

\begin{exercise}
(JPE, September 1996) Compute the singular values of
$$
  A=\left (\begin{array}{rrr} 0 & -1.6 & 0.6 \\
    0 & 1.2 & 0.8 \\ 0 & 0 & 0 \\ 0 & 0 & 0 \end{array}\right )
$$
\end{exercise}

\begin{exercise}
(JPE, May 2003) Determine the singular value decomposition for the
matrix
$$
  A=\left (\begin{array}{rr} 3 & 2 \\ 2 & 3 \\ 2 & -2
  \end{array}\right )
$$
\end{exercise}

\begin{exercise}
Find the numerical rank with tolerance 0.9 of the matrix
$$
  A=\left (\begin{array}{rr} 3 & 2 \\
    -4 & -5 \end{array}\right )
$$
\end{exercise}

\begin{exercise}
Let $Q\in\IC^{n\times n}$ be unitary. Find all singular values of
$Q$.
\end{exercise}

\begin{exercise}
Show that if two matrices $A,B \in \IC^{n\times n}$ are unitary
equivalent, then they have the same singular values. Is the converse
true? (Prove or give a counterexample.)
\end{exercise}

\newpage

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\refstepcounter{section} \label{Ch6}

\centerline{Chapter \ref{Ch6} }\vspace{5mm}

\centerline{\Large\bf Schur Decomposition}\vspace{3mm}

\addcontentsline{toc}{section}{\ref{Ch6}. Schur Decomposition}

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\setcounter{subsection}{0} \setcounter{exercise}{0}
\setcounter{equation}{0}

Recall that every complex matrix $A\in\IC^{n\times n}$ is similar to a
Jordan matrix. In other words, for every linear operator
$\IC^n\to\IC^n$ there exists a basis in which the operator is
represented by a Jordan matrix. How can we represent linear operators
if we use orthonormal bases only? In other words, to what extend one
can simplify a complex matrix by using unitary equivalence (instead of
similarity)?

\subsection{Schur decomposition} \label{17.1}

Any matrix $A\in \IC^{n\times n}$ is unitary equivalent to an upper
triangular matrix $T$. That is, there exists a unitary matrix $Q$ such
that
$$
    A = QTQ^\ast.
$$
Note that the diagonal of $T$ consists of the eigenvalues of $A$.
Moreover, one can find $Q$ and $T$ in such a way that the eigenvalues
of $A$ appear in any given order on the diagonal of $T$.

\medskip{\footnotesize\emph{Proof}.
We use induction on $n$. For $n=1$, we just write $A=I\cdot A\cdot
I^\ast$ and note that $I$ is unitary and $A=[a_{11}]$ is upper
triangular.

Now let $n>1$. Let $\lambda$ be an eigenvalue of $A$ and let $x$ be a
unit eigenvector corresponding to $\lambda$. Let $Q_1$ be a unitary
matrix whose first column is $x$. Such a matrix exists, because there
is an ONB in $\IC^n$ whose first vector is $x$, according to
Section~\ref{ONBexist}; then $Q_1$ can be constructed as a matrix whose
columns are the vectors of that ONB.

Note that $Q_1e_1=x$, hence $Q_1^{\ast}x=e_1$. Therefore
$$
    Q_1^{\ast}AQ_1e_1=Q_1^{\ast}Ax=\lambda Q_1^{\ast}x=\lambda e_1.
$$
In other words, $e_1$ is an eigenvector of the matrix $Q_1^{\ast}AQ_1$
corresponding to the eigenvalue $\lambda$. Now the above formula
implies
$$
   Q_1^{\ast}AQ_1=\left[\begin{array}{cc}
   \lambda & w^\ast \\ 0 & B\end{array}\right]
$$
with some $w\in\IC^{n-1}$ and $B\in\IC^{(n-1)\times (n-1)}$.

By our inductive assumption, the smaller matrix $B$ has a Schur
decomposition, i.e., $\hat{Q}^{\ast}B\hat{Q}=\hat{T}$ for a unitary
matrix $\hat{Q}\in\IC^{(n-1)\times (n-1)}$ and an upper triangular
matrix $\hat{T}$. By Section~\ref{UnitaryColumnsRows}, the enlarged
$n\times n$ matrix $\begin{bmatrix} 1 & 0 \\ 0 & \hat{Q}\end{bmatrix}$
is also unitary. Thus
$$
     Q=Q_1\,\left[\begin{array}{cc} 1 &
     0 \\ 0 & \hat{Q} \end{array}\right]
$$
is a unitary matrix, too (Section~\ref{Group}). Now one can verify
directly that
\begin{align*}
      Q^{\ast}AQ &=\left[\begin{array}{cc}
        1 & 0 \\ 0 & \hat{Q}^{\ast}
          \end{array}\right]
         \left[\begin{array}{cc} \lambda & w^\ast \\ 0 & B \end{array}\right]
         \left[\begin{array}{cc} 1 & 0 \\ 0 & \hat{Q} \end{array}\right ]\\
          &=\left[\begin{array}{cc} \lambda & w^\ast \hat{Q} \\
          0 & \hat{Q}^{\ast}B\hat{Q} \end{array}\right]
          =\left[\begin{array}{cc} \lambda & w^\ast \hat{Q} \\
          0 & \hat{T} \end{array}\right].
\end{align*}
The last matrix is an $n\times n$ upper triangular matrix, as required.
\qed}

\subsection{Normal matrices} \label{17.2}

A matrix $A\in\IC^{n\times n}$ is said to be {\em normal} if
$AA^{\ast}=A^{\ast}A$.

\medskip\noindent{\footnotesize One can easily check that unitary and Hermitian matrices are
normal.}

\subsection{Normal matrices under unitary equivalence} \label{17.3}

If $A$ is normal and $Q$ unitary, then $B=Q^{\ast}AQ$ is normal. In
other words, the class of normal matrices is closed under unitary
equivalence.

\medskip{\footnotesize \emph{Proof}.
Note that $B^{\ast}=Q^{\ast}A^{\ast}Q$ and
$BB^{\ast}=Q^{\ast}AA^{\ast}Q=Q^{\ast}A^{\ast}AQ=B^{\ast}B$.\qed}

\subsection{Lemma} \label{17.4}

If $A$ is normal and upper triangular, then $A$ is diagonal.

\medskip{\footnotesize \emph{Proof}.
We use induction on $n$. For $n=1$ the lemma is trivial. Assume that
$n>1$. Let us compute the top left element of the matrix
$AA^{\ast}=A^{\ast}A$. On the one hand, it is
$$
     \sum_{i=1}^n a_{1i}\bar{a}_{1i}=
     \sum_{i=1}^n |a_{1i}|^2.
$$
On the other hand, it is just $|a_{11}|^2$. Hence,
$a_{12}=\cdots=a_{1n}=0$, i.e.,
$$
     A=\left[\begin{array}{cc} a_{11} & 0 \\ 0 & B\end{array}\right]
$$
One can easily check that $AA^{\ast}=A^{\ast}A$ implies
$BB^{\ast}=B^{\ast}B$, i.e., $B$ is a normal $(n-1)\times (n-1)$
matrix. Since $A$ is upper triangular, so is $B$. By our inductive
assumption, $B$ is diagonal, hence $A$ is diagonal, too. \qed}

\new Note: Any diagonal matrix is normal, because diagonal matrices commute.

\subsection{Theorem} \label{17.5}

A matrix $A\in\IC^{n\times n}$ is normal if and only if it is unitary
equivalent to a diagonal matrix. In that case the Schur decomposition
takes form
 \beq \label{SchurD}
   A=QDQ^{\ast}
 \eeq
where $D$ is a diagonal matrix. Whenever the Schur decomposition takes
form \eqref{SchurD}, the columns of $Q$ (Schur vectors) become
eigenvectors of $A$.


\medskip\noindent{\footnotesize \emph{Proof}.
This readily follows from Sections~\ref{17.1}--\ref{17.4}; see also
Section~\ref{11.22a}. \qed}


\subsection{Remark} \label{17.6}

Three classes of complex matrices (unitary, Hermitian, and normal) have
the same property: they are unitary equivalent to a diagonal matrix (in
other words, they admit an ONB consisting of eigenvectors).

The difference between these classes lies in restrictions on the
eigenvalues: unitary matrices have eigenvalues on the unit circle
($|\lambda|=1$), Hermitian matrices have real eigenvalues
($\lambda\in\IR$), and now normal matrices have arbitrary complex
eigenvalues.

\begin{center}
\begin{tikzpicture}
         \draw[blue,thick,->,-latex] (-4,0) -- (4,0);
         \draw[thick,black!60,->,-latex] (0,-3) -- (0,3);
         \draw[thick,green] (0,0) circle(2);
         \node[anchor=north west] at (2,0) {1};
         \node[anchor=north east] at (-2,0) {$-1$};
         \node[anchor=south east] at (0,2) {$\mathbf{i}$};
         \node[anchor=north east] at (0,-2) {$-\mathbf{i}$};
         \node[anchor=south west] at (-1.7,-0.05) {\color{blue}H$\ \ \ $e$\ \ \ $r$\ \ \ $m$\ \ \ $i$\ \ \ $t$\ \ \ $i$\ \ \ $a$\ \ \ $n};
         \node[anchor=north,rotate=45] at (-1.45,1.35) {\color{green}U};
         \node[anchor=north,rotate=30] at (-1,1.65) {\color{green}n};
         \node[anchor=north,rotate=15] at (-0.6,1.95) {\color{green}i};
         \node[anchor=north,rotate=0] at (-0.1,2.03) {\color{green}t};
         \node[anchor=north,rotate=-15] at (0.4,1.93) {\color{green}a};
         \node[anchor=north,rotate=-30] at (0.9,1.73) {\color{green}r};
         \node[anchor=north,rotate=-45] at (1.3,1.44) {\color{green}y};
         \node[rotate=35,anchor=south west] at (-1.98,-2.3) {\color{gray}\Large N$\ \ \ \ \ $O$\ \ \ \ \ $R$\ \ \ \ \ $M$\ \ \ \ \ $A$\ \ \ \ \ $L};
\end{tikzpicture}
\end{center}

\newpage

\subsection{Real Schur Decomposition} \label{17.7}

If $A\in\IR^{n\times n}$, then there exists an orthogonal matrix $Q$
such that
$$
     Q^TAQ=\left[\begin{array}{cccc}
     R_{11} & R_{12} & \cdots & R_{1m} \\
     0 & R_{22} & \cdots & R_{2m} \\
     \vdots & \vdots & \ddots & \vdots \\
     0 & 0 & \cdots & R_{mm}
     \end{array}\right]
$$
where each diagonal block $R_{ii}$ is either a $1\times 1$ matrix or a
$2\times 2$ matrix.

\medskip\noindent{\footnotesize \emph{Proof}.
We use the induction on $n$. Our induction will be somewhat unusual --
its increment will be either 1 or 2, i.e., our proof of the theorem for
any given $n$ will be based on its validity for either $n-1$ or $n-2$
(depending on the matrix $A$). Thus the basis for our induction must
consist of \emph{two} smallest values of $n$, i.e., $n=1$ and $n=2$.

For $n=1$ and $n=2$, we just write $A=I\cdot A\cdot I^\ast$ and note
that $I$ is unitary and $A$ is either a single $1\times 1$ block (if
$n=1$) or a single $2\times 2$ block (if $n=2$).

Now let $n>2$. If the matrix $A$ has a real eigenvalue, then we can
reduce the dimension by one just like in the proof in Section~\ref{17.1} and then use our inductive assumption of the
validity of the theorem for $n-1$.

If $A$ has no real eigenvalues, then by Lemma~\ref{10.11a} there is a
two-dimensional subspace $W\subset \IR^n$ invariant under $A$. Let
$\{x_1,x_2\}$ be an ONB of $W$ (one exists due to
Sect.~\ref{ONBexist}). The invariance of $W$ under $A$ implies that
$Ax_1\in W$ and $Ax_2\in W$, hence
\begin{align*}
         Ax_1 &= r_{11}\, x_1 + r_{21}\, x_2 \\
         Ax_2 &= r_{12}\, x_1 + r_{22}\, x_2
\end{align*}
with some $r_{ij}\in\IR$. Due to Sect.~\ref{ONBexist}, the ONB
$\{x_1,x_2\}$ in $W$ can be extended to an ONB $\{x_1,\ldots,x_n\}$ in
$\IR^n$. Let $\tilde{Q}$ denote the orthogonal matrix with columns
$x_1,\ldots,x_n$. Observe that $\tilde{Q}e_1=x_1$ and
$\tilde{Q}e_2=x_2$, hence $\tilde{Q}^T x_1 =e_1$ and $\tilde{Q}^T x_2
=e_2$. Therefore,
$$
   \tilde{Q}^T A\tilde{Q}e_1
   =\tilde{Q}^T Ax_1
   =\tilde{Q}^T (r_{11}\, x_1 + r_{21}\, x_2)
   =r_{11}\, e_1 + r_{21}\, e_2
$$
and similarly
$$
   \tilde{Q}^T A\tilde{Q}e_2
   =\tilde{Q}^T Ax_2
   =\tilde{Q}^T (r_{12}\, x_1 + r_{22}\, x_2)
   =r_{12}\, e_1 + r_{22}\, e_2
$$
Thus,
$$
   \tilde{Q}^T A\tilde{Q}
      =\left[\begin{array}{cc}
      R_{11} & \tilde{R}_{12} \\ 0 & \tilde{R}_{22}
      \end{array}\right]
$$
where
$$
     R_{11}
      =\left[\begin{array}{cc}
      r_{11} & r_{12} \\ r_{21} & r_{22}
      \end{array}\right]
$$
$\tilde{R}_{12}$ is some $2\times (n-2)$ matrix and $\tilde{R}_{22}$ is
some $(n-2)\times (n-2)$ matrix. Now we can apply our inductive
assumption to the $(n-2)\times (n-2)$ matrix $\tilde{R}_{22}$ and
finish the proof in the same way as the proof in Section~\ref{17.1}.
Note that we need the validity of the theorem for $n-2$, i.e., the
increment of our induction is 2 in the above case. \qed}

\newpage

\begin{exercise}
(combined from JPE, October 1990 and May 1997) Let $A\in\IC^{n\times
n}$ be a normal matrix.\begin{itemize} \item[(a)] Prove that $A-\lambda
I$ is normal for any $\lambda\in\IC$. \item[(b)] Prove that
$\|Ax\|=\|A^{\ast}x\|$ for all $x$.\item[(c)] Prove that $(\lambda,x)$
is an eigenpair of $A$ if and only if $(\bar{\lambda},x)$ is an
eigenpair of $A^{\ast}$. (Hence, $A$ and $A^{\ast}$ have the same
eigenvectors.) \end{itemize}
\end{exercise}

\begin{exercise}
(JPE, September 2002) A matrix $A\in\IC^{n\times n}$ is said to be {\em
skew Hermitian} if $A^\ast = -A$.\begin{itemize} \item[(a)] Prove that
if $A$ is skew Hermitian and B is unitary equivalent to $A$, then $B$
is also skew Hermitian.\item[(b)] Prove that the eigenvalues of a skew
Hermitian matrix are purely imaginary, i.e.\ they satisfy
$\bar{\lambda}=-\lambda$.\item[(c)] What special form does the Schur
decomposition take for a skew Hermitian matrix $A$? \end{itemize}
\end{exercise}

\begin{exercise} \label{ExDMdense}
(JPE, September 1998). Show that diagonalizable complex matrices make a
dense subset of $\IC^{n\times n}$. That is, for any $A\in\IC^{n\times
n}$ and $\varepsilon>0$ there is a diagonalizable $B\in\IC^{n\times n}$
such that $\|A-B\|_2<\varepsilon$.
\end{exercise}

\begin{exercise}
[Bonus] (JPE, May 1996). Let $T$ be a linear operator on a finite
dimensional complex inner product space $V$, and let $T^{\ast}$ be the
adjoint of $T$. Prove that $T=T^{\ast}$ if and only if $T^{\ast}T=T^2$.
\end{exercise}

\newpage

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\refstepcounter{section} \label{Ch7}

\centerline{Chapter \ref{Ch7} }\vspace{5mm}

\centerline{\Large\bf LU Decomposition}\vspace{3mm}

\addcontentsline{toc}{section}{\ref{Ch7}. LU Decomposition}

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\setcounter{subsection}{0} \setcounter{exercise}{0}
\setcounter{equation}{0}

First we review Gaussian elimination from Linear Algebra:

\subsection{Gaussian elimination}

Let $A\in\IC^{n\times n}$ be a square matrix. We will modify it
consecutively, in $n-1$ steps, as described below, to obtain an upper
triangular matrix. For the first step we denote $A^{(1)}=A$ and
$a_{ij}^{(1)}=a_{ij}$ for all $1\leq i,j \leq n$.\medskip

\textbf{Step 1}. Assume that $a_{11}^{(1)} \ne 0$. We define {\em
multipliers}
$$
     m_{i1} = a^{(1)}_{i1}/a^{(1)}_{11}\qquad
     \text{for}\ \ i=2,\ldots,n
$$
Then we replace the $i$-th row $\tilde{a}_i^{(1)}$ of the matrix
$A^{(1)}$ with $\tilde{a}_i^{(1)} - m_{i1}\tilde{a}_1^{(1)}$ for all
$i=2,\ldots,n$, and obtain a new matrix
$$
    {A}^{(2)} = \left[
     \begin{array}{cccc}
     a_{11}^{(1)} & a_{12}^{(1)} & \cdots & a_{1n}^{(1)} \\
     0 & a_{22}^{(2)} & \cdots & a_{2n}^{(2)} \\
    \vdots & \vdots& \ddots & \vdots \\
    0 & a_{n2}^{(2)} & \cdots & a_{nn}^{(2)}
    \end{array}\right]
$$
where
$$
    a_{ij}^{(2)} = a_{ij}^{(1)}-m_{i1} a_{1j}^{(1)}
    \qquad \text{for}\ \ 2\le i,j \le n
$$
Note that the first column of $A^{(2)}$ below the diagonal consists of
zeros, i.e., $a_{i1}^{(2)}=0$ for all $i=2,\ldots,n$.

\textbf{Step 2}. Assume that $a_{22}^{(2)} \ne 0$. We define {\em
multipliers}
$$
   m_{i2} = a_{i2}^{(2)}/a_{22}^{(2)}\qquad \text{for}\ \ i=3,\ldots,n
$$
Then we replace the $i$-th row $\tilde{a}_i^{(2)}$ of the matrix
$A^{(2)}$ with $\tilde{a}_i^{(2)} - m_{i2}\tilde{a}_2^{(2)}$ for all
$i=3,\ldots,n$, and obtain a new matrix
$$
    {A}^{(3)} = \left[
     \begin{array}{ccccc}
     a_{11}^{(1)} & a_{12}^{(1)} & a_{13}^{(1)} & \cdots & a_{1n}^{(1)} \\
     0 & a_{22}^{(2)} & a_{23}^{(2)} & \cdots & a_{2n}^{(2)} \\
     0 & 0 & a_{33}^{(3)} & \cdots & a_{3n}^{(2)} \\
    \vdots & \vdots & \vdots & \ddots & \vdots \\
    0 & 0 & a_{n3}^{(3)} & \cdots & a_{nn}^{(3)}
    \end{array}\right]
$$
where
$$
    a_{ij}^{(3)} = a_{ij}^{(2)}-m_{i1} a_{2j}^{(2)}
    \qquad \text{for}\ \ 3\le i,j \le n
$$
Note that the first two columns of $A^{(3)}$ below the diagonal
consists of zeros, i.e., $a_{ij}^{(3)}=0$ for $j=1,2$ and all
$i=j+1,\ldots,n$.

Then we continue this process. At each step we create zeros in one more
column below the diagonal. In $n-1$ steps all the columns below the
main diagonal will consist of zeros, hence the resulting matrix $A^{n}$
will be upper triangular, and we will denote ir by $U$:
$$
    A^{(n)}=U = \left[
    \begin{array}{ccccc}
     a_{11}^{(1)} & a_{12}^{(1)} & a_{13}^{(1)} & \cdots & a_{1n}^{(1)} \\
     0 & a_{22}^{(2)} & a_{23}^{(2)} & \cdots & a_{2n}^{(2)} \\
     0 & 0 & a_{33}^{(3)} & \cdots & a_{3n}^{(3)} \\
    \vdots & \vdots & \vdots & \ddots & \vdots \\
    0 & 0 & 0 & \cdots & a_{nn}^{(n)}
    \end{array}\right]
$$
Remember that the above procedure requires $a_{kk}^{(k)}\neq 0$ for all
$k=1,\ldots,n-1$. These numbers are called {\em pivots}.

The procedure has to stop prematurely if (and only if) one of the
pivots happens to be zero. In that case we say that the Gaussian
elimination fails.

Note that
$$
    a_{kk}^{(k)} \neq 0\qquad \text{for all}\ \ k=1,\ldots,n-1
$$
is equivalent to
$$
    a_{11}^{(1)}\cdots a_{kk}^{(k)} \neq 0\qquad \text{for all}\ \ k=1,\ldots,n-1
$$
This will be convenient for the criterion of failure below.

\subsection{Principal minors}

Let $A\in\IC^{n\times n}$. For $1\le k \le n-1$, the $k$-th principal
minor of $A$ is the $k\times k$ matrix formed by the entries in the
first $k$ rows and the first $k$ columns of $A$. In other words, $A_k$
is the top left $k\times k$ block of $A$. We denote the $k$-th
principal minor of $A$ by $A_k$. We will also denote $A_n=A$.

\subsection{Criterion of failure} \label{LUcriterion}

Gaussian elimination fails if and only if
$$
    {\rm det}\, A_k=0
    \qquad \text{for some}\ \ k=1,\ldots,n-1.
$$
This is because for each $k=1,\ldots, n$
$$
    \det A_k=a^{(1)}_{11}\cdots a^{(k)}_{kk}.
$$

Next we present the Gaussian elimination in matrix form. All our
formulas can be verified by direct inspection, so we give them without
proof.

\subsection{Gauss matrices}

For each $j=1,\ldots,n-1$ we denote
$$
    m^{(j)} = \left[\begin{array}{c}
      0\\ \vdots \\ 0 \\ m_{j+1,j} \\ \vdots \\ m_{n,j}
    \end{array}\right]
$$
and then we define the so called \emph{Gauss matrix} $G_j$ by
$$
    {G}_j = I - m^{(j)} e_j^\ast = \left[\begin{array}{cccclcc}
     1 & 0 & \cdots & 0 & \cdots & 0 & 0 \\
     0 & 1 & \cdots & 0 & \cdots & 0 & 0 \\
     \vdots & \vdots & \ddots & \vdots & \ddots & \vdots & \vdots \\
     0 & 0 & \ddots & 1 & \cdots & \vdots & \vdots \\
     \vdots & \vdots & \ddots & -m_{j+1,j} & \ddots& \vdots & \vdots\\
     \vdots & \vdots & \ddots & \vdots & \vdots & 1 & 0\\
     0 & 0 & \cdots & -m_{n,j} & \cdots & 0 & 1
      \end{array}\right].
$$

\subsection{Main matrix formulas}

For each $k=1,\ldots,n-1$ we have
$$
  A^{(k+1)} = {G}_k {A}^{(k)}
$$
and therefore
$$
    U=A^{(n)}={G}_{n-1} \cdots {G}_2 {G}_1 {A}.
$$
This can be rewritten as
$$
    A={G}_{1}^{-1}G_2^{-1} \cdots {G}_{n-1}^{-1}U.
$$
Next let us denote
$$
     L_j = I + m^{(j)} e_j^\ast = \left[\begin{array}{cccclcc}
     1 & 0 & \cdots & 0 & \cdots & 0 & 0 \\
     0 & 1 & \cdots & 0 & \cdots & 0 & 0 \\
     \vdots & \vdots & \ddots & \vdots & \ddots & \vdots & \vdots \\
     0 & 0 & \cdots & 1 & \cdots & 0 & 0 \\
     \vdots & \vdots & \ddots & m_{j+1,j} & \ddots& \vdots & \vdots\\
     \vdots & \vdots & \ddots & \vdots & \ddots & 1 & 0\\
     0 & 0 & \cdots & m_{n,j} & \cdots & 0 & 1
      \end{array}\right].
$$
For each $j=1,\ldots,n-1$ we have $L_jG_j=I$, hence ${G}^{-1}_j = L_j$,
and thus
 \beq \label{ALLLU}
    A=L_{1}L_2 \cdots L_{n-1}U.
 \eeq

\subsection{Unit lower/upper triangular matrices} \label{unitlower}

A matrix $L$ is said to be \emph{unit lower triangular} if it is lower
triangular and has ones on its main diagonal. Note that $\det L=1$. The
following slight extensions of the rules given in
Section~\ref{UpperLowerTriang} can be verified directly:

\begin{itemize}
\item If $L_1$ and $L_2$ are both unit lower triangular matrices,
    then so is their product $L_1 L_2$ and so are their inverses
    $L_1^{-1}$ and $L_2^{-1}$.
\end{itemize}


\begin{center}
\begin{tikzpicture}
  \node[anchor=west] at (-3.05,0.8) {\footnotesize Unit lower};
  \node[anchor=west] at (-3,0.5) {\footnotesize triangular};
  \node[anchor=west] at (-2.95,0.2) {\footnotesize matrices:};
  \fill[gray!40] (0,0)--(0,0.9)--(0.9,0)--cycle;
  \fill[gray!20] (0,0.9)--(0,1)--(0.1,1)--(1,0.1)--(1,0)--(0.9,0)--cycle;
  \draw[thick] (0,0) -- (0,1) -- (1,1) -- (1,0) -- cycle;
  \node[anchor=north east] at (1,1) {$0$};
  \node[anchor=north east] at (0.4,1.0) {\tiny $1$};
  \node[anchor=north east] at (0.7,0.7) {\tiny $1$};
  \node[anchor=north east] at (1.0,0.4) {\tiny $1$};
  \node[anchor=west] at (0.95,0.5) {$\times$};
  \begin{scope}[xshift=1.5cm]
  \fill[gray!40] (0,0)--(0,0.9)--(0.9,0)--cycle;
  \fill[gray!20] (0,0.9)--(0,1)--(0.1,1)--(1,0.1)--(1,0)--(0.9,0)--cycle;
  \draw[thick] (0,0) -- (0,1) -- (1,1) -- (1,0) -- cycle;
  \node[anchor=north east] at (1,1) {$0$};
  \node[anchor=north east] at (0.4,1.0) {\tiny $1$};
  \node[anchor=north east] at (0.7,0.7) {\tiny $1$};
  \node[anchor=north east] at (1.0,0.4) {\tiny $1$};
  \node[anchor=west] at (0.95,0.5) {$=$};
  \end{scope}
  \begin{scope}[xshift=3cm]
  \fill[gray!40] (0,0)--(0,0.9)--(0.9,0)--cycle;
  \fill[gray!20] (0,0.9)--(0,1)--(0.1,1)--(1,0.1)--(1,0)--(0.9,0)--cycle;
  \draw[thick] (0,0) -- (0,1) -- (1,1) -- (1,0) -- cycle;
  \node[anchor=north east] at (1,1) {$0$};
  \node[anchor=north east] at (0.4,1.0) {\tiny $1$};
  \node[anchor=north east] at (0.7,0.7) {\tiny $1$};
  \node[anchor=north east] at (1.0,0.4) {\tiny $1$};
  \end{scope}
  \begin{scope}[xshift=6cm]
  \fill[gray!40] (0,0)--(0,0.9)--(0.9,0)--cycle;
  \fill[gray!20] (0,0.9)--(0,1)--(0.1,1)--(1,0.1)--(1,0)--(0.9,0)--cycle;
  \draw[thick] (0,0) -- (0,1) -- (1,1) -- (1,0) -- cycle;
  \node[anchor=north east] at (1,1) {$0$};
  \node[anchor=north east] at (0.4,1.0) {\tiny $1$};
  \node[anchor=north east] at (0.7,0.7) {\tiny $1$};
  \node[anchor=north east] at (1.0,0.4) {\tiny $1$};
  \node[anchor=west] at (0.94,1) {\scriptsize -1};
  \node[anchor=west] at (1.07,0.5) {$=$};
  \end{scope}
  \begin{scope}[xshift=7.62cm]
  \fill[gray!40] (0,0)--(0,0.9)--(0.9,0)--cycle;
  \fill[gray!20] (0,0.9)--(0,1)--(0.1,1)--(1,0.1)--(1,0)--(0.9,0)--cycle;
  \draw[thick] (0,0) -- (0,1) -- (1,1) -- (1,0) -- cycle;
  \node[anchor=north east] at (1,1) {$0$};
  \node[anchor=north east] at (0.4,1.0) {\tiny $1$};
  \node[anchor=north east] at (0.7,0.7) {\tiny $1$};
  \node[anchor=north east] at (1.0,0.4) {\tiny $1$};
  \end{scope}
\end{tikzpicture}
\end{center}


In a similar way we can define \emph{unit upper triangular} matrices.
They obey similar rules.

\subsection{Main matrix formulas (continued)}

Note that $L_1,\ldots,L_{n-1}$ in \eqref{ALLLU} are unit lower
triangular matrices, thus their product is a unit lower triangular
matrix, which we denote by $L$. More precisely, we have
\begin{align*}
    L &= L_1L_2\cdots L_{n-1}\\
    &=(I+m^{(1)} e_1^\ast)(I+m^{(2)} e_2^\ast) \cdots
                (I+m^{(n-1)} e_{n-1}^\ast)\\
    &= I + m^{(1)} e_1^\ast + m^{(2)} e_2^\ast + \cdots + m^{(n-1)} e_{n-1}^\ast\\
   & = \left[\begin{array}{ccccl}
    1  & 0 & 0 & \cdots & 0\\
    m_{21} & 1 & 0 & \cdots & 0\\
    \vdots & \vdots & \vdots & \ddots & \vdots \\
    m_{n-1,1}& m_{n-1,2} & \cdots & 1 & 0\\
    m_{n1} & m_{n2} & \cdots & m_{n,n-1} & 1
     \end{array}\right].
\end{align*}

We summarize our matrix formulas in the following theorem:

\subsection{Theorem (LU decomposition)} \label{TmLU}

Let $A\in\IC^{n\times n}$ have non-singular principal minors up to
the order $n-1$. Then there is a decomposition
$$
      A=LU
$$
where $L$ is a unit lower triangular matrix and $U$ is an upper
triangular matrix:

\begin{center}
\begin{tikzpicture}
%  \node[anchor=west] at (-6,0.5) {$A=LU$, schematically:};
  \fill[gray!40] (0,0)--(0,1)--(1,1)--(1,0)--cycle;
  \draw[thick] (0,0) -- (0,1) -- (1,1) -- (1,0) -- cycle;
  \node[anchor=north] at (0.5,0.07) {\scriptsize $A$};
  \node[anchor=west] at (0.95,0.5) {$=$};
  \begin{scope}[xshift=1.5cm]
  \fill[gray!40] (0,0)--(0,0.9)--(0.9,0)--cycle;
  \fill[gray!20] (0,0.9)--(0,1)--(0.1,1)--(1,0.1)--(1,0)--(0.9,0)--cycle;
  \draw[thick] (0,0) -- (0,1) -- (1,1) -- (1,0) -- cycle;
  \node[anchor=north east] at (1,1) {$0$};
  \node[anchor=north east] at (0.4,1.0) {\tiny $1$};
  \node[anchor=north east] at (0.7,0.7) {\tiny $1$};
  \node[anchor=north east] at (1.0,0.4) {\tiny $1$};
  \node[anchor=north] at (0.5,0.07) {\scriptsize $L$};
  \node[anchor=west] at (0.95,0.5) {$\times$};
  \end{scope}
  \begin{scope}[xshift=3cm]
  \fill[gray!40] (0,0.9)--(0,1)--(1,1)--(1,0)--(0.9,0)--cycle;
  \draw[thick] (0,0) -- (0,1) -- (1,1) -- (1,0) -- cycle;
  \node[anchor=south west] at (0,0) {$0$};
  \node[anchor=north] at (0.5,0.07) {\scriptsize $U$};
  \end{scope}
\end{tikzpicture}
\end{center}
\vspace{-3mm}

\noindent Note that det$\,L=1$, hence
$$
   \det A=\det U=u_{11}\cdots u_{nn}.
$$
If, in addition, $A$ is non-singular, then the LU decomposition is
unique.\medskip

\noindent{\footnotesize \emph{Proof}: The existence follows from our
matrix formulas. To prove the uniqueness, we argue by way of
contradiction. Let $A=\tilde{L} \tilde{U} = L U$ be two LU
decompositions. Since $A$ is non-singular, it follows that $\tilde{U}$
is also non-singular, and hence
$$
  {L}^{-1}\tilde{L} = U\tilde{U}^{-1}.
$$
By the rules of Section~\ref{unitlower}, the inverse ${L}^{-1}$ is unit
lower triangular, so the product ${L}^{-1} \tilde{L}$ is unit lower
triangular. On the other hand, $U\tilde{U}^{-1}$ is upper triangular.
The only matrix that is both unit lower triangular and upper triangular
is the identity matrix $I$, hence
$$
  {L}^{-1}\tilde{L} = U\tilde{U}^{-1} = I.
$$
This implies $\tilde{L} = L$ and $\tilde{U}=U$. \qed}

\subsection{Forward and backward substitutions} \label{FBS}

The LU decomposition can be used to solve a system of linear equations
$$
    Ax=b,
$$
where $A\in\IC^{n\times n}$ is a given nonsingular matrix, $b\in\IC^n$
is a given vector, and and $x\in \IC^n$ is an unknown vector to be
computed.

Suppose we have an LU decomposition $A=LU$, where $L$ is lower
triangular and $U$ is upper triangular. Then we can find $x$ in two
steps.

\begin{center}
\begin{tikzpicture}
%  \node[anchor=west] at (-6,0.5) {$A=LU$, schematically:};
  \fill[gray!40] (0,0)--(0,1)--(0.1,1)--(1,0.1)--(1,0)--cycle;
  \draw[thick] (0,0) -- (0,1) -- (1,1) -- (1,0) -- cycle;
  \node[anchor=north east] at (1,1) {$0$};
  \node[anchor=north] at (0.5,0.07) {\scriptsize $L$};
  \node[anchor=west] at (0.95,0.5) {\tiny$\times$};
  \begin{scope}[xshift=1.4cm]
  \fill[gray!40] (0,0.9)--(0,1)--(1,1)--(1,0)--(0.9,0)--cycle;
  \draw[thick] (0,0) -- (0,1) -- (1,1) -- (1,0) -- cycle;
  \node[anchor=south west] at (0,0) {$0$};
  \node[anchor=north] at (0.5,0.07) {\scriptsize $U$};
  \node[anchor=west] at (0.95,0.5) {\tiny$\times$};
  \end{scope}
  \begin{scope}[xshift=2.8cm]
  \draw[thick,fill=red!20] (0,0) -- (0,1) -- (0.2,1) -- (0.2,0) -- cycle;
  \node[anchor=north] at (0.1,0.04) {\scriptsize $x$};
  \node[anchor=west] at (0.15,0.5) {$=$};
  \draw[thick,fill=gray!40] (0.7,0) -- (0.7,1) -- (0.9,1) -- (0.9,0) -- cycle;
  \node[anchor=north] at (0.8,0.07) {\scriptsize $b$};
  \end{scope}
  \node[anchor=north] at (2.18,0.1) {$\underbrace{\phantom{mmmnn}}_{y}$};
%  \node[anchor=north] at (2.5,-0.1) {\scriptsize $y$};
\end{tikzpicture}
\end{center}

\noindent{\bf Step 1}. Denote $Ux=y$ and solve the lower triangular
system $Ly = b $ for $y$ via ``\underline{forward substitution}'' (computing
$y_1,\ldots,y_n$ subsequently):
{\footnotesize \begin{align*}
  \ell_{11}y_1 &= b_1 &\Rightarrow & & y_1&=b_1/\ell_{11}\\
  \ell_{21}y_1+\ell_{22}y_2 &= b_2 &\Rightarrow & & y_2&=(b_2-\ell_{21}y_1)/\ell_{22}\\
  \ell_{31}y_1+\ell_{32}y_2+\ell_{33}y_3 &= b_3 &\Rightarrow & & y_3&=(b_3-\ell_{31}y_1-\ell_{32}y_2)/\ell_{33}\\
  \vdots &  &  & & &\quad \vdots \\
  \ell_{n1}y_1+\cdots+\ell_{nn}y_n &= b_n &\Rightarrow & & y_n&=(b_n-\ell_{n1}y_1-\cdots-\ell_{n(n-1)}y_{n-1})/\ell_{nn}
\end{align*}}
Note: if $L$ is \emph{unit} lower triangular, then
$\ell_{11}=\cdots=\ell_{nn}=1$, and therefore the divisions can be
avoided.\vspace{5mm}

\noindent {\bf Step 2}. Solve the system $Ux=y$ for $x$ via ``\underline{backward substitution}'' (computing $x_n,\ldots,x_1$ subsequently):
{\footnotesize
\begin{align*}
  u_{nn}x_n &= y_n &\Rightarrow & & x_n&=y_n/u_{nn}\\
  u_{(n-1)(n-1)}x_{n-1}+u_{(n-1)n}x_n &= y_{n-1} &\Rightarrow & & x_{n-1}&=(y_{n-1}-u_{(n-1)n}x_n)/u_{(n-1)(n-1)}\\
  \vdots &  &  & & &\quad \vdots \\
  u_{11}x_1+\cdots+u_{1n}x_n &= y_1 &\Rightarrow & & x_1&=(y_1-u_{12}x_2-\cdots-u_{1n}x_n)/u_{11}
\end{align*}}

\newpage

\subsection{Cost of computation} \label{costLU}

The cost of computation is measured in ``flops'', where a flop ({\em
fl}oating point {\em op}eration) is  an arithmetic operation (addition,
subtraction, multiplication, division, or a root extraction).

Let us estimate the cost of the LU decomposition. The cost of
computation of $A^{(2)}$ is $n-1$ divisions to compute the multipliers
and then $2n(n-1)$ flops to update the rows ($n-1$ rows with $2n$ flops
per row), i.e. total of approximately $2n^2$ flops. The computation of
$A^{(3)}$ then takes $2(n-1)^2$ flops, and so on.

Thus the total computational cost for the LU factorization is
$$
  2\bigl(n^2+(n-1)^2 + \cdots + 1^2\bigr) = \frac{2n(n+1)(2n+1)}{6} \approx
  \frac{2n^3}{3}\quad \text{flops}
$$

If one solves a system of linear equations $Ax=b$, then the LU
decomposition is followed by solving two triangular systems
(Section~\ref{FBS}). The cost to solve one triangular system is about
$n^2$ flops. So there is an additional cost of $2n^2$ flops, which is
negligible compared to $2n^3/3$. Hence, the total cost is still
$\approx 2n^3/3$.

Note that the LU decomposition takes most of the computations required
for solving a system $Ax=b$. Thus, this method is particularly well
suited to very common situations in which one is solving systems $Ax =
b$ for more than one vector $b$, but with the same matrix $A$. In that
case the LU decomposition is done just once, and then each additional
$b$ will require $\approx 2n^2$ flops.

\subsection{Computation of $A^{-1}$}

Assume that $A\in\IC^{n\times n}$ is non-singular and has non-singular
principal minors up to the order $n-1$, so that the Gaussian
elimination works. One can find the matrix $X=A^{-1}$ by solving the
system $AX=I$ for $X\in\IC^{n\times n}$. This amounts to solving $n$
systems of linear equations $Ax_k=e_k$, for $k=1,\ldots,n$, where $x_k$
stands for the $k$-th column of the matrix $X$. The computational cost
of this procedure is
$$
   \frac{2n^3}{3}+n\times 2n^2=\frac{8n^3}{3}\quad \text{flops}
$$
As a matter of fact, this is the fastest way of computing the
inverse $A^{-1}$.

\newpage

\subsection{``Manual'' solution of systems of linear equations}

One needs to remember that the LU decomposition (with the subsequent
forward and backward substitutions) is just a convenient matrix
representation of the standard and more familiar Gaussian elimination.
These two procedures are mathematically equivalent. When one solves a
system of linear equations $Ax=b$ practically (on paper), one should
definitely apply the Gaussian elimination to both $A$ and $b$, rather
than forming the L and U matrices.

\subsection{Example} \label{GaussExample}

{\footnotesize Suppose one needs to solve the following system by LU
decomposition:
$$
  \left[\begin{array}{cc} 0.01 & 2 \\ 1 & 3 \end{array}\right]
  \left[\begin{array}{l} x \\ y \end{array}\right]=
  \left[\begin{array}{l} 2 \\ 4 \end{array}\right]
$$
Then one multiplies the first row by 100 and subtracts it from the
second row. This gives
$$
  \left[\begin{array}{cc} 0.01 & 2 \\ 0 & -197 \end{array}\right]
  \left[\begin{array}{l} x \\ y \end{array}\right]=
  \left[\begin{array}{c} 2 \\ -196 \end{array}\right]
$$
Then one solves the second equation for $y$ and finds
$y=\frac{196}{197}$. Lastly, one substitutes this value for $y$ into
the first equation and finds $x=\tfrac{200}{197}$.}

\subsection{Warnings}

It is a common practice to solve systems of linear equations $Ax=b$
with a nonsingular matrix $A$ via Gaussian elimination (or,
equivalently, LU decomposition). However, this method requires that the
matrix $A$ satisfies certain conditions (Section~\ref{LUcriterion}).

It is easy to find nonsingular matrices for which Gaussian elimination
does \emph{not} work and LU decomposition does \emph{not} exist.
For example, let $A=\left [\begin{smallmatrix}0 & 1\\
1 & 0\end{smallmatrix}\right ]$. Then the Gaussian elimination fails
instantly, because the very first pivot is zero. Furthermore, for this
matrix $A$ there is no LU decomposition $A=LU$, even if we only require
that $L$ be lower triangular (not necessarily unit lower triangular);
see Exercise~\ref{ExNoLU}.

In addition, when a pivot is not exactly zero but close to zero, then
things can go wrong, too: multipliers become very large, and numerical
computations (in the sense specified later in Chapter~\ref{Ch11}) tend
to become inaccurate. One should avoid such situations by using an
appropriate \emph{pivoting strategy}. We will describe these
strategies, briefly, and illustrate them by example.

\newpage

\subsection{Partial pivoting} \label{PartialPivoting}

The idea is to avoid zero or small pivots by interchanging rows. At
every step of Gaussian elimination one looks for the largest (in
absolute value) element in the pivot column (at or below the main
diagonal). Then the row containing the largest element is
interchanged with the current row. Now the largest element is on the
main diagonal. After that the usual elimination step is performed.

Partial pivoting ensures that all the multipliers (i.e., the
elements of the matrix $L$) have absolute values less than or equal
to one.

\subsection{Example} \label{GaussExamplePP}

{\footnotesize We apply the partial pivoting strategy to
Example~\ref{GaussExample}. In the first column, the larger element
is $1$, hence we need to interchange the rows to put that element on
the diagonal:
$$
  \left[\begin{array}{cc} 1 & 3 \\ 0.01 & 2 \end{array}\right]
  \left[\begin{array}{l} x \\ y \end{array}\right]=
  \left[\begin{array}{l} 4 \\ 2 \end{array}\right].
$$
Now we multiply the first row by 0.01 and subtract it from the second
row. This gives
$$
  \left[\begin{array}{cc} 1 & 3 \\ 0 & 1.97 \end{array}\right]
  \left[\begin{array}{l} x \\ y \end{array}\right]=
  \left[\begin{array}{c} 4 \\ 1.96 \end{array}\right].
$$
Then we solve the second equation for $y$ and find $y=\frac{196}{197}$.
Lastly, we substitute this value for $y$ into the first equation and
find $x=\tfrac{200}{197}$.}

\subsection{Complete pivoting}

The method of {\em complete pivoting} involves both row and column
interchanges to make use of the largest pivot available. This method
provides additional insurance against buildup of computational errors.

\subsection{Example} \label{GaussExampleCP}

{\footnotesize We apply the complete pivoting strategy to
Example~\ref{GaussExample}. The largest element in the whole matrix
is $3$, so we need to move it to the top left position by
interchanging rows and columns:
$$
  \left[\begin{array}{cc} 3 & 1 \\ 2 & 0.01 \end{array}\right]
  \left[\begin{array}{l} y \\ x \end{array}\right]=
  \left[\begin{array}{l} 4 \\ 2 \end{array}\right].
$$
Note that interchanging columns requires permutation of the unknowns
$x$ and $y$. Next we multiply the first row by 2/3 and subtract it
from the second row. This gives
$$
  \left[\begin{array}{cc} 3 & 1 \\ 0 & -\tfrac{197}{300} \end{array}\right]
  \left[\begin{array}{l} y \\ x \end{array}\right]=
  \left[\begin{array}{c} 4 \\ -\tfrac{2}{3} \end{array}\right].
$$
Then we solve the second equation for $x$ and find
$x=\tfrac{200}{197}$. Lastly, we substitute this value for $x$ into the
first equation and find $y=\frac{196}{197}$.}


\newpage

\subsection{Diagonally dominant matrices}

{\footnotesize A matrix $A\in\IC^{n\times n}$ such that
$$
   |a_{ii}| > \sum_{j \ne i} |a_{ij}|
$$
for all $i$ is said to be {\em strictly row diagonally dominant}. If
$$
    |a_{jj}| > \sum_{i \ne j} |a_{ij}|
$$
for all $j$ the matrix is said to be {\em strictly column diagonally
dominant}.}

One can check that if a matrix $A$ is strictly row (or column)
diagonally dominant, then det$\, A\neq 0$ {\footnotesize (the proof
can be omitted for now; this fact readily follows from Gershgorin
Theorem that will be given in Chapter~\ref{Ch15})}. Since all the
principal minors of $A$ are also strictly row (or column) diagonally
dominant, we have det$\, A_k\neq 0$ for all $1\leq k< n$. Therefore
no zero pivots will be encountered during Gaussian elimination.

{\footnotesize With some extra effort one can show that if $A$ is
strictly column diagonally dominant, then all the multipliers (i.e.,
the elements of $L$) have absolute values $\leq 1$.}

\begin{exercise}
Find a nonzero matrix $A\in\IR^{2\times 2}$ that admits at least two LU
decomposition, i.e.\ $A=L_1U_1=L_2U_2$, where $L_1$ and $L_2$ are two
{\em distinct} unit lower triangular matrices and $U_1$ and $U_2$ are
two {\em distinct} upper triangular matrices.
\end{exercise}

\begin{exercise} \label{ExNoLU}
Show that the matrix $\begin{bmatrix} 0 & 1 \\ 1 & 0
\end{bmatrix}$ admits no LU decomposition, even if we only require
that $L$ be lower triangular (not necessarily unit lower
triangular).
\end{exercise}

\begin{exercise}
The spectral radius of a matrix $A\in\IC^{n\times n}$ is defined by
$$
     \rho_A=\max\{|\lambda|:\ \lambda\text{ eigenvalue of }A\}.
$$
\begin{itemize}
\item[(a)] Show that $\rho_A\leq \|A\|_2$. \item[(b)] Give an
    example of a $2\times 2$ matrix $A$ such that $\rho_A<1$ but
$\|A\|_2>100$.\item[(c)] Show that if
$$
   \lim_{n\to\infty}\|A^n\|_2 = 0,
$$
then $\rho_A<1$.
\end{itemize}
\end{exercise}

\begin{exercise}
[Bonus] In the notation of the previous problem, show that if
$\rho_A<1$, then
$$
   \lim_{n\to\infty}\|A^n\|_2 = 0.
$$
Hint: use Jordan decomposition.
\end{exercise}

\newpage

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\refstepcounter{section} \label{Ch8}

\centerline{Chapter \ref{Ch8} }\vspace{5mm}

\centerline{\Large\bf Cholesky Factorization}\vspace{3mm}

\addcontentsline{toc}{section}{\ref{Ch8}. Cholesky Factorization}

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\setcounter{subsection}{0} \setcounter{exercise}{0}
\setcounter{equation}{0}


\subsection{Theorem (LDM$^{\ast}$ Decomposition)} \label{TmLDM*}

Assume that $A\in \IC^{n\times n}$ is non-singular and has
non-singular principal minors, i.e., det$\,A_k\neq 0$ for all
$k=1,\ldots,n$. Then there are unique matrices $L,D,M$ such that
$L,M$ are unit lower triangular, $D$ is diagonal, and
\beq \label{LDM*}
   A=LDM^{\ast}
\eeq

\begin{center}
\begin{tikzpicture}
%  \node[anchor=west] at (-6,0.5) {$A=LU$, schematically:};
  \fill[gray!40] (0,0)--(0,1)--(1,1)--(1,0)--cycle;
  \draw[thick] (0,0) -- (0,1) -- (1,1) -- (1,0) -- cycle;
  \node[anchor=north] at (0.5,0.07) {\scriptsize $A$};
  \node[anchor=west] at (0.95,0.5) {$=$};
  \begin{scope}[xshift=1.5cm]
  \fill[gray!40] (0,0)--(0,0.9)--(0.9,0)--cycle;
  \fill[gray!20] (0,0.9)--(0,1)--(0.1,1)--(1,0.1)--(1,0)--(0.9,0)--cycle;
  \draw[thick] (0,0) -- (0,1) -- (1,1) -- (1,0) -- cycle;
  \node[anchor=north east] at (1,1) {$0$};
  \node[anchor=north east] at (0.4,1.0) {\tiny $1$};
  \node[anchor=north east] at (0.7,0.7) {\tiny $1$};
  \node[anchor=north east] at (1.0,0.4) {\tiny $1$};
  \node[anchor=north] at (0.5,0.07) {\scriptsize $L$};
  \node[anchor=west] at (0.95,0.5) {$\times$};
  \end{scope}
  \begin{scope}[xshift=3cm]
  \fill[gray!40] (0,0.9)--(0,1)--(0.1,1)--(1,0.1)--(1,0)--(0.9,0)--cycle;
  \draw[thick] (0,0) -- (0,1) -- (1,1) -- (1,0) -- cycle;
  \node[anchor=south west] at (0,0) {$0$};
  \node[anchor=north east] at (1,1) {$0$};
  \node[anchor=north] at (0.5,0.07) {\scriptsize $D$};
  \node[anchor=west] at (0.95,0.5) {$\times$};
  \end{scope}
  \begin{scope}[xshift=4.5cm]
  \fill[gray!40] (0,0.9)--(0,1)--(1,1)--(1,0)--(0.9,0)--cycle;
  \fill[gray!20] (0,0.9)--(0,1)--(0.1,1)--(1,0.1)--(1,0)--(0.9,0)--cycle;
  \draw[thick] (0,0) -- (0,1) -- (1,1) -- (1,0) -- cycle;
  \node[anchor=north east] at (0.4,1.0) {\tiny $1$};
  \node[anchor=north east] at (0.7,0.7) {\tiny $1$};
  \node[anchor=north east] at (1.0,0.4) {\tiny $1$};
  \node[anchor=south west] at (0,0) {$0$};
  \node[anchor=north] at (0.5,0.07) {\scriptsize $M^\ast$};
  \end{scope}
\end{tikzpicture}
\end{center}
\vspace{-3mm}

{\footnotesize \proof According to Theorem~\ref{TmLU}, there is an
LU decomposition $A=LU$. Let $u_{11},\ldots,u_{nn}$ denote the
diagonal components of $U$. Define diagonal matrix $D=\, {\rm
diag}\{u_{11},\ldots,u_{nn}\}$. Since $A$ is non-singular, $D$ is
invertible, and $D^{-1}=\, {\rm
diag}\{u^{-1}_{11},\ldots,u^{-1}_{nn}\}$. One can easily check that
$M^{\ast}=D^{-1}U$ is a unit upper triangular matrix, hence
\eqref{LDM*} exists.

To prove the uniqueness, suppose $A=LDM^{\ast}=L_1D_1M^{\ast}_1$ are
two different LDM$^{\ast}$ decompositions. Note that all the
matrices here are invertible, because $A$ is non-singular. By the
uniqueness of the LU decomposition (Theorem~\ref{TmLU}), we have
$L=L_1$, hence, $DM^{\ast}=D_1M^{\ast}_1$ and
$D^{-1}_1D=M^{\ast}_1(M^{\ast})^{-1}$. Since both $M^{\ast}$ and
$M^{\ast}_1$ are unit upper triangular, so is
$M^{\ast}_1(M^{\ast})^{-1}$ (by Sect.~\ref{unitlower}). On the other
hand, $D^{-1}_1D$ is diagonal. The only matrix that is unit lower
triangular and diagonal is the identity matrix. Thus
$M^{\ast}_1(M^{\ast})^{-1}=I$ and $D^{-1}_1D=I$, which implies
$M^\ast=M^\ast_1$ and $D=D_1$. \qed}

\subsection{Corollary (LDL$^{\ast}$ Decomposition)} \label{13.2}

Suppose $A\in \IC^{n\times n}$ satisfies the assumptions of
Theorem~\ref{TmLDM*} \emph{and} is Hermitian. Then there exist a
unique unit lower triangular matrix $L$ and a unique diagonal matrix
$D$ such that
$$
        A=LDL^{\ast}
$$
Moreover, the components of $D$ are \emph{real} numbers.

{\footnotesize \proof By Theorem~\ref{TmLDM*}, $A=LDM^{\ast}$. Now
$A=A^{\ast}=MD^{\ast}L^{\ast}$, and by the uniqueness of the
LDM$^{\ast}$ decomposition we have $L=M$ and $D=D^{\ast}$. The
latter implies that $D$ is a real matrix. \qed}

\subsection{Sylvester's Theorem} \label{TmSylvester}

Let $A\in\IC^{n\times n}$ be a Hermitian matrix. Then
$$
  A\ \ \text{is positive definite}
  \quad\Longleftrightarrow\quad
  \det A_k>0\ \ (\forall k=1,\ldots, n)
$$

{\footnotesize \proof Let $A$ be positive definite. By
Section~\ref{12.13}, $A=B^\ast B$ and $\det B\neq 0$. Hence
$$
   \det A=\det B\times\det B^{\ast}=
   \det B\times \overline{\det B}=|\det B|^2>0.
$$
One can easily check that every principal minor $A_k$ is also a
Hermitian positive definite matrix, therefore by the same argument
$\det A_k>0$.

Now let $\det A_k>0$ for all $k=1,\ldots,n$. By
Corollary~\ref{13.2}, $A=LDL^{\ast}$. Denote by $L_k$ and $D_k$ the
$k$-th principal minors of $L$ and $D$, respectively. One can easily
check that $A_k= L_kD_kL_k^{\ast}$. Since $L_k$ is \emph{unit} lower
triangular, we have $\det L_k=1$, thus $\det D_k=\det A_k>0$. Denote
$D=\,{\rm diag}\{d_1,\ldots,d_n\}$, hence $D_k=\,{\rm
diag}\{d_1,\ldots,d_k\}$. Now
$$
  \det D_k = d_1\cdots d_k>0\qquad \forall k=1,\ldots,n
$$
implies that $d_1,\ldots,d_n$ are all real and positive. Lastly,
$$
  \la Ax,x\ra=\la LDL^{\ast} x, x\ra
  =\la DL^{\ast} x,L^{\ast} x\ra= \la Dy,y\ra >0
$$
where $y=L^{\ast}x$. Note that $y\neq 0$ whenever $x\neq 0$, because
$L^\ast$ is invertible. \qed}


\subsection{Corollary} \label{13.4}

Let $A$ be a positive definite matrix. Then $a_{ii}>0$ for all
$i=1,\ldots,n$. Furthermore, let $1\leq i_1<i_2<\cdots <i_k\leq n$, and
let $A'$ be the $k\times k$ matrix formed by the intersections of the
rows and columns of $A$ with numbers $i_1,\ldots,i_k$. Then $\det
A'>0$.

{\footnotesize \proof Recall that $A$ defines an operator $\IC^n \to
\IC^n$. We can reorder the canonical basis vectors $e_1,\ldots,e_n$
in $\IC^n$ and get a new ONB $\{e_{i_1},\ldots,e_{i_k},\ldots\}$. In
this new basis, the operator is represented by a matrix $B$ such
that $B_k = A'$, i.e., $A'$ is a principal minor of a positive
definite matrix. Then we can apply Sylvester's
Theorem~\ref{TmSylvester}.\qed}

\medskip The above corollary suggests a quick way of proving that
a given matrix $A$ is \emph{not} positive definite: just spot a
non-positive diagonal component or a $2\times 2$ submatrix (with two
components on the diagonal of $A$) with a non-positive determinant,
etc.

\newpage

\subsection{Theorem (Cholesky Factorization)} \label{13.5}

Let $A\in\IC^{n\times n}$ be Hermitian and positive definite. Then
there exists a unique lower triangular matrix $G$ with real positive
diagonal entries such that\vspace{3mm}

\centerline{$A=GG^{\ast}$}

\vspace{2mm}\noindent{\footnotesize Example (zeros in matrices are not
shown):}

\vspace{3mm}\noindent{\footnotesize $\phantom{i}\quad \begin{bmatrix} 4 & 2 & 8\\ 2 & 10 & 7\\
8 & 6 & 21\end{bmatrix} = \begin{bmatrix} 2 &  & \\ 1 & 3 & \\ 4 & 2 &
1\end{bmatrix} \times \begin{bmatrix} 2 & 1 & 4\\  & 3 & 2\\  &  &
1\end{bmatrix}\qquad\phantom{m}$}
\begin{tikzpicture}[scale=1.05,baseline=2.9ex] \fill[brown!56]
(0,0)--(0,0.4)--(0.4,0.4)--(0.4,0)--cycle; \fill[brown!42]
(0.4,0)--(0.4,0.4)--(0.8,0.4)--(0.8,0)--cycle; \fill[brown!99]
(0.8,0)--(0.8,0.4)--(1.2,0.4)--(1.2,0)--cycle; \fill[brown!14]
(0,0.4)--(0,0.8)--(0.4,0.8)--(0.4,0.4)--cycle; \fill[brown!70]
(0.4,0.4)--(0.4,0.8)--(0.8,0.8)--(0.8,0.4)--cycle; \fill[brown!49]
(0.8,0.4)--(0.8,0.8)--(1.2,0.8)--(1.2,0.4)--cycle; \fill[brown!28]
(0,0.8)--(0,1.2)--(0.4,1.2)--(0.4,0.8)--cycle; \fill[brown!14]
(0.4,0.8)--(0.4,1.2)--(0.8,1.2)--(0.8,0.8)--cycle; \fill[brown!56]
(0.8,0.8)--(0.8,1.2)--(1.2,1.2)--(1.2,0.8)--cycle; \draw[gray,very
thin] (0,0)--(0,1.2)--(1.2,1.2)--(1.2,0)--cycle; \node[anchor=west] at
(1.15,0.6) {$=$};
\begin{scope}[xshift=1.8cm]
\fill[brown!40] (0,0)--(0,0.4)--(0.4,0.4)--(0.4,0)--cycle;
\fill[brown!20] (0.4,0)--(0.4,0.4)--(0.8,0.4)--(0.8,0)--cycle;
\fill[brown!10] (0.8,0)--(0.8,0.4)--(1.2,0.4)--(1.2,0)--cycle;
\fill[brown!10] (0,0.4)--(0,0.8)--(0.4,0.8)--(0.4,0.4)--cycle;
\fill[brown!30] (0.4,0.4)--(0.4,0.8)--(0.8,0.8)--(0.8,0.4)--cycle;
\fill[gray!0] (0.8,0.4)--(0.8,0.8)--(1.2,0.8)--(1.2,0.4)--cycle;
\fill[brown!20] (0,0.8)--(0,1.2)--(0.4,1.2)--(0.4,0.8)--cycle;
\fill[gray!0] (0.4,0.8)--(0.4,1.2)--(0.8,1.2)--(0.8,0.8)--cycle;
\fill[gray!0] (0.8,0.8)--(0.8,1.2)--(1.2,1.2)--(1.2,0.8)--cycle;
\draw[gray,very thin] (0,0)--(0,1.2)--(1.2,1.2)--(1.2,0)--cycle;
\node[anchor=west] at (1.15,0.6) {$\times$};
\end{scope}
\begin{scope}[xshift=3.6cm]
\fill[gray!0]   (0,0)--(0,0.4)--(0.4,0.4)--(0.4,0)--cycle;
\fill[gray!0] (0.4,0)--(0.4,0.4)--(0.8,0.4)--(0.8,0)--cycle;
\fill[brown!10] (0.8,0)--(0.8,0.4)--(1.2,0.4)--(1.2,0)--cycle;
\fill[gray!0] (0,0.4)--(0,0.8)--(0.4,0.8)--(0.4,0.4)--cycle;
\fill[brown!30] (0.4,0.4)--(0.4,0.8)--(0.8,0.8)--(0.8,0.4)--cycle;
\fill[brown!20] (0.8,0.4)--(0.8,0.8)--(1.2,0.8)--(1.2,0.4)--cycle;
\fill[brown!20] (0,0.8)--(0,1.2)--(0.4,1.2)--(0.4,0.8)--cycle;
\fill[brown!10] (0.4,0.8)--(0.4,1.2)--(0.8,1.2)--(0.8,0.8)--cycle;
\fill[brown!40] (0.8,0.8)--(0.8,1.2)--(1.2,1.2)--(1.2,0.8)--cycle;
\draw[gray,very thin] (0,0)--(0,1.2)--(1.2,1.2)--(1.2,0)--cycle;
\end{scope}
\end{tikzpicture}

{\footnotesize \proof By Corollary~\ref{13.2}, the matrix $A$ has an
LDL$^\ast$ decomposition $A=LDL^{\ast}$. Denote $D=\, {\rm
diag}\{d_1,\ldots,d_n\}$. As it was shown in the proof of Sylvester's
Theorem~\ref{TmSylvester}, all $d_i$'s are real and positive numbers.
Thus we can form another diagonal matrix with real positive diagonal
entries as follows: $D_{1/2}=\, {\rm
diag}\{\sqrt{d_1},\ldots,\sqrt{d_n}\}$. Then $D=D^{1/2}D^{1/2}$ and
setting $G=LD^{1/2}$ readily gives $A=GG^{\ast}$. One can easily check
that the diagonal entries of $G$ are $\sqrt{d_1}, \ldots,\sqrt{d_n}$,
so they are real positive numbers, as required.

To prove the uniqueness, suppose
$A=GG^{\ast}=\tilde{G}\tilde{G}^{\ast}$ are two different Cholesky
factorizations. Then $\tilde{G}^{-1}G=\tilde{G}^{\ast}(G^{\ast})^{-1}$.
This is the equality of a lower triangular matrix and an upper
triangular one; hence both matrices are diagonal:
$$
   \tilde{G}^{-1}G=\tilde{G}^{\ast}(G^{\ast})^{-1}=D'
   =\,{\rm diag}\{d_1',\ldots,d_n'\}.
$$
Since $A$ is positive definite, it must be non-singular, hence all the
matrices in our formulas must be invertible. Note that
$\tilde{G}=G(D')^{-1}$, which implies
$$
   \tilde{g}_{ii} = g_{ii}/d_i'\qquad\forall\ \ i=1,\ldots,n.
$$
Similarly, $\tilde{G}^\ast=G^\ast D'$, which implies
$$
 \tilde{g}_{ii} = g_{ii}d_i'\qquad \forall\ \ i=1,\ldots,n.
$$
Multiplying the above two formulas gives $\tilde{g}_{ii}^2 = g_{ii}^2$,
and since these are real positive numbers, we get $\tilde{g}_{ii} =
g_{ii}$. Therefore $d_i'=1$ and $D'=I$. This implies $\tilde{G}=G$.
\qed}

\subsection{Algorithm for Cholesky factorization} \label{13.6}

{\footnotesize Here we outline the algorithm for computing the matrix
$G=(g_{ij})$ from the matrix $A=(a_{ij})$. For simplicity, we will
restrict our presentation to the \textbf{real} matrices
$A\in\IR^{n\times n}$. Note that $G$ is lower triangular, i.e.,
$g_{ij}=0$ for $i<j$, hence,
$$
   a_{ij}=\sum_{k=1}^{\min\{i,j\}}g_{ik}g_{jk}.
$$
Setting $i=j=1$ gives $a_{11}=g_{11}^2$, hence
$$
  g_{11}=\sqrt{a_{11}}.
$$
Next, for $2\leq i\leq n$ we have $a_{i1}=g_{i1}g_{11}$, hence
$$
      g_{i1}=a_{i1}/g_{11}\ \ \ \ \ \ i=2,\ldots,n.
$$
This gives the first column of $G$. Now, inductively,
assume that we already have the first $j-1$ columns
of $G$. Then $a_{jj}=\sum_{k=1}^j g_{jk}^2$, hence
$$
   g_{jj}=\sqrt{a_{jj}-\sum_{k=1}^{j-1}g_{jk}^2}.
$$
Next, for $j+1\leq i\leq n$ we have $a_{ij}=
\sum_{k=1}^jg_{ik}g_{jk}$, hence
$$
   g_{ij}=\frac{1}{g_{jj}}\biggl( a_{ij}-\sum_{k=1}^{j-1}g_{ik}g_{jk}\biggr).
$$
}

\subsection{Cost of computation} \label{13.7}

{\footnotesize The computation of $g_{ij}$ takes $\approx 2j$ flops for
each $i=j,\ldots,n$, so the total is
$$
   \sum_{j=1}^n 2j(n-j)\approx 2n\frac{n^2}{2}-
   2\,\frac{n^3}{3}=\frac{n^3}{3}
$$
Recall that the LU decomposition takes about $2n^3/3$ flops, so the
Cholesky factorization is nearly twice as fast. It is also more stable
than the LU decomposition, in the sense precisely defined below, in
Chapter~\ref{Ch13}.}

\subsection{Criterion of positive definiteness} \label{13.8}

The above algorithm can be used to verify that a given real symmetric
matrix, $A$, is positive definite. Whenever all the square root
extractions in Section~\ref{13.6} are possible and give non zero
numbers, i.e. whenever
$$
   a_{11}>0\ \ \ \ {\rm and}\qquad
   a_{jj}-\sum_{k=1}^{j-1}g_{jk}^2>0\quad \forall j\geq 2
$$
the matrix $A$ is positive definite. If at least one of the above
quantities is \emph{not} positive, then $A$ is not positive definite.

\begin{exercise}
(JPE, May 1994) Let $A\in{\mathbb R}^{n\times n}$ be given, symmetric
and positive definite. Define $A_0=A$, and consider the sequence of
matrices defined by
$$
   A_k=G_kG_k^t\ \ \ \ \ {\rm and}\ \ \ \ \ A_{k+1}=G_k^tG_k
$$
where $A_k=G_kG_k^t$ is the Cholesky factorization for $A_k$. Prove
that the $A_k$ all have the same eigenvalues.
\end{exercise}


\newpage

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\refstepcounter{section} \label{Ch9}

\centerline{Chapter \ref{Ch9} }\vspace{5mm}

\centerline{\Large\bf QR Decomposition}\vspace{3mm}

\addcontentsline{toc}{section}{\ref{Ch9}. QR Decomposition}

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\setcounter{subsection}{0} \setcounter{exercise}{0}
\setcounter{equation}{0}


\subsection{Gram-Schmidt orthogonalization (revisited)}
\label{QR1}

Let $\{v_1,\ldots,v_n\}$ be a linearly independent set of vectors in
$\IC^m$ ($m \geq n$).

\noindent The Gram-Schmidt orthogonalization described in
Section~\ref{GramSchmidt} gives an orthonormal set of vectors
$\{u_1,\ldots,u_n\}$ in $\IC^m$, and the equations
\eqref{GSstep1}--\eqref{GS} can be written as
\begin{align} \label{GSforQR}
     v_1 &= r_{11}u_1, \nonumber\\
     v_2 &= r_{12}u_1 + r_{22}u_2, \nonumber \\
     v_3 &= r_{13}u_1 + r_{23}u_2 + r_{33}u_3, \\
     &\quad \dots \nonumber \\
     v_n &= r_{1n}u_1 + r_{2n}u_2 + \cdots + r_{n-1,n}u_{n-1}+r_{nn}u_n \nonumber
\end{align}
where $r_{ip}=\la v_p,u_i\ra$ for $i<p\leq n$ and $r_{pp}=\|w_{p}\|$.
Note that $r_{pp}>0$.

Let us assemble a matrix $A \in \IC^{m\times n}$ whose columns are the
vectors $v_1,\ldots, v_n$, a matrix $\hat{Q}$ whose columns are the
vectors $u_1,\ldots,u_n$:
$$
  A=   \left[\!\!\!
   \begin{array}{c} \phantom{\vdots} \\ v_1 \\ \phantom{\vdots} \end{array}
   \!\!\!\right | \left . \!\!\!\!
   \begin{array}{c} \phantom{\vdots} \\ v_2 \\ \phantom{\vdots} \end{array}
   \!\!\!\right | \left . \!\!\!\!
   \begin{array}{c} \phantom{\vdots} \\ \cdots \\ \phantom{\vdots} \end{array}
   \!\!\!\right | \left . \!\!\!\!
   \begin{array}{c} \phantom{\vdots} \\ v_n \\ \phantom{\vdots} \end{array}
   \!\!\!\right ]
   \qquad\text{and}\qquad
  \hat{Q}=   \left[\!\!\!
   \begin{array}{c} \phantom{\vdots} \\ u_1 \\ \phantom{\vdots} \end{array}
   \!\!\!\right | \left . \!\!\!\!
   \begin{array}{c} \phantom{\vdots} \\ u_2 \\ \phantom{\vdots} \end{array}
   \!\!\!\right | \left . \!\!\!\!
   \begin{array}{c} \phantom{\vdots} \\ \cdots \\ \phantom{\vdots} \end{array}
   \!\!\!\right | \left . \!\!\!\!
   \begin{array}{c} \phantom{\vdots} \\ u_n \\ \phantom{\vdots} \end{array}
   \!\!\!\right ].
$$
Let $\hat{R}\in \IC^{n\times n}$ be an upper triangular matrix defined
as follows:
$$
     \hat{R}=\left[\begin{array}{cccc}
     r_{11} & r_{12} & \cdots & r_{1n} \\
     0 & r_{22} & \cdots & r_{2n} \\
     \vdots & \vdots & \ddots & \vdots \\
     0 & 0 & \cdots & r_{nn}
     \end{array}\right].
$$
Then the above relations \eqref{GSforQR} can be written in matrix form:
\beq \label{AhQhR}
   A = \hat{Q}\hat{R}
\eeq
One can verify the equivalence of \eqref{GSforQR} and \eqref{AhQhR} by
direct inspection.

\subsection{Linearly dependent case} \label{QR2}

{\footnotesize Let us relax our assumption that $v_1,\ldots,v_n$ are
linearly independent. Thus let $\{v_1,\ldots,v_n\}$ be an
\emph{arbitrary} set of vectors in $\IC^m$. Then the Gram-Schmidt
procedure \ref{GramSchmidt} can be adjusted as follows.

For some $p\geq 1$ we now may have $v_p=0$ or, more generally,
$$
  v_p \in\, {\rm span}\{v_1,\ldots,v_{p-1}\}.
$$
This implies $v_p \in\, {\rm span}\{u_1,\ldots,u_{p-1}\}$, therefore
$w_p=0$, in the notation of Section~\ref{GramSchmidt}. Now the vector
$u_p= w_p/\|w_p\|$ cannot be determined.

Instead, we can choose $u_p$ as an arbitrary unit vector orthogonal to
$u_1,\ldots,u_{p-1}$ (we know that such vectors exist due to
Sect.~\ref{ONBexist}). After such a ``loosely constrained'' selection
of $u_p$ we continue the Gram-Schmidt orthogonalization as described in
Section~\ref{GramSchmidt}. Every time we encounter a linear dependence
$v_p \in\, {\rm span}\{v_1,\ldots,v_{p-1}\}$, we select $u_p$
``frivolously'' as specified above. In that case, the respective
equation
$$
     v_p = r_{1p}u_1 + \cdots + r_{p-1,p}u_{p-1}+r_{pp}u_p
$$
for $v_p$, as given in \eqref{GSforQR}, will still hold, but with
$r_{pp}=0$ instead of $r_{pp}>0$.}

\subsection{Extension of $\hat{Q}$ to $Q$} \label{QR3}

The vectors $u_1,\ldots,u_n$ make an orthonormal set in $\IC^m$ that
can be extended to an ONB $\{u_1,\ldots,u_m\}$, according to
Sect.~\ref{ONBexist}. Now we can form a larger, square matrix
{\footnotesize
$$
    Q=   \left[\!\!\!
   \begin{array}{c} \phantom{i} \\ \phantom{\vdots} \\ u_1 \\ \phantom{i} \\ \phantom{\vdots} \end{array}
   \!\!\!\right | \left . \!\!\!\!
   \begin{array}{c} \phantom{i} \\ \phantom{\vdots} \\ u_2 \\ \phantom{i} \\ \phantom{\vdots} \end{array}
   \!\!\!\right | \left . \!\!\!\!
   \begin{array}{c} \phantom{i} \\ \phantom{\vdots} \\ \cdots \\ \phantom{i} \\ \phantom{\vdots} \end{array}
   \!\!\!\right | \left . \!\!\!\!
   \begin{array}{c} \phantom{i} \\ \phantom{\vdots} \\ u_n \\ \phantom{i} \\ \phantom{\vdots} \end{array}
   \!\!\!\right | \left . \!\!\!\!
   \begin{array}{c} \phantom{i} \\ \phantom{\vdots} \\ \cdots \\ \phantom{i} \\ \phantom{\vdots} \end{array}
   \!\!\!\right | \left . \!\!\!\!
   \begin{array}{c} \phantom{i} \\ \phantom{\vdots} \\ u_m \\ \phantom{i} \\ \phantom{\vdots} \end{array}
   \!\!\!\right ] \in \IC^{m\times m}
$$}
which is \emph{unitary}, according to Sect.~\ref{CharIsom3}. By adding
$m-n$ rows consisting of zeros to the bottom of the matrix $\hat{R}$ we
get a larger matrix
{\footnotesize
$$
     R=\left[\begin{array}{cccc}
     r_{11} & r_{12} & \cdots & r_{1n} \\
     0 & r_{22} & \cdots & r_{2n} \\
     \vdots & \vdots & \ddots & \vdots \\
     0 & 0 & \cdots & r_{nn}\\
     0 & 0 & \cdots & 0\\
    \vdots & \vdots & \ddots & \vdots \\
     0 & 0 & \cdots & 0
     \end{array}\right] \in \IC^{m\times n}.
$$}
We still call $R$ an ``upper triangular matrix''. Now the relation
\eqref{AhQhR} can be written as
\beq \label{AQR}
         A=QR.
\eeq
The equivalence of \eqref{AhQhR} and \eqref{AQR} can be verified by
direct inspection.

\newpage

We summarize the previous constructions as follows:

\subsection{Theorem (QR decomposition)}
\label{QR4}

For any matrix $A\in\IC^{m\times n}$ with $m\geq n$ there exist a
unitary matrix $Q\in\IC^{m\times m}$ and an upper triangular matrix
$R\in\IC^{m\times n}$ such that
\beq \label{QR}
   A=QR.
\eeq
This is called \emph{(full) QR decomposition} of $A$.

The first (left) $n$ columns of the matrix $Q$ form a smaller matrix
$\hat{Q} \in \IC^{m\times n}$ (whose columns are orthonormal vectors in
$\IC^m$) and the first (upper) $n$ rows of the matrix $R$ form a square
upper triangular matrix $\hat{R}$ so that
\beq \label{reducedQR}
        A=\hat{Q}\hat{R}.
\eeq
This is called \emph{reduced (``skinny'') QR decomposition} of $A$.

\begin{center}
\begin{tikzpicture}
  \draw[thick,fill=gray!30] (0,-0.5) -- (0,1) -- (1,1) -- (1,-0.5) -- cycle;
  \node[anchor=west] at (0.95,0.5) {$=$};
  \node[anchor=north] at (0.5,-0.61) {$A$};
  \begin{scope}[xshift=1.5cm]
  \draw[thick,fill=gray!30] (0,-0.5) -- (0,1) -- (1.5,1) -- (1.5,-0.5) -- cycle;
  \node[anchor=west] at (1.45,0.5) {$\times$};
  \node[anchor=north] at (0.5,-0.61) {$Q$};
  \end{scope}
  \begin{scope}[xshift=3.5cm]
  \fill[gray!30] (0,0.9)--(0,1)--(1,1)--(1,0)--(0.9,0)--cycle;
  \draw[thick] (0,-0.5) -- (0,1) -- (1,1) -- (1,-0.5) -- cycle;
  \node[anchor=west] at (0.95,0.5) {$=$};
  \node[anchor=south west] at (0.2,-0.4) {$0$};
  \node[anchor=north] at (0.5,-0.61) {$R$};
  \end{scope}
  \begin{scope}[xshift=5cm]
  \draw[dashed,fill=gray!10] (0,-0.5) -- (0,1) -- (1.5,1) -- (1.5,-0.5) -- cycle;
  \draw[thick,fill=gray!30] (0,-0.5) -- (0,1) -- (1,1) -- (1,-0.5) -- cycle;
  \node[anchor=west] at (1.45,0.5) {$\times$};
  \node[anchor=north] at (0.5,-0.5) {$\hat{Q}$};
  \end{scope}
  \begin{scope}[xshift=7cm]
  \fill[gray!30] (0,0.9)--(0,1)--(1,1)--(1,0)--(0.9,0)--cycle;
  \draw[dashed] (0,-0.5) -- (0,1) -- (1,1) -- (1,-0.5) -- cycle;
  \draw[thick] (0,0) -- (0,1) -- (1,1) -- (1,0) -- cycle;
  \node[anchor=south west] at (0,0) {$0$};
  \node[anchor=north] at (0.5,-0.5) {$\hat{R}$};
  \end{scope}
\end{tikzpicture}
\end{center}
\vspace{-5mm}

{\footnotesize \proof Given a matrix $A\in\IC^{m\times n}$, we
disassemble it into column vectors $v_1,\ldots,v_n \in \IC^m$, and then
apply all the constructions described in
Sections~\ref{QR1}--\ref{QR3}.\qed}

\subsection{Real QR decomposition} \label{QR5}

Suppose $A\in\IR^{m\times n}$ is a real matrix. Then all the
constructions of Sections~\ref{QR1}--\ref{QR3} can be done with real
matrices only, and they produce real matrices $Q$ and $R$. In
particular, $Q$ will be an orthogonal matrix.

\subsection{Positivity of the diagonal of $R$} \label{QR5aaa}

The QR decomposition can be constructed so that the diagonal components
of $R$ ($\hat{R}$) are real non-negative numbers:
$$
   r_{ii} \geq 0\qquad \forall i=1,\ldots,n.
$$
If $A$ has full rank (i.e., rank$\,A=n$), then the columns of $A$ are
linearly independent. In that case $\hat{R}$ is invertible, and one can
construct $R$ (and $\hat{R}$) so that its diagonal components are real
positive numbers:
$$
   r_{ii} > 0\qquad \forall i=1,\ldots,n.
$$

\subsection{Uniqueness of the QR decomposition} \label{QR6}

If $A$ has full rank (i.e., rank$\,A=n$), then the matrix $\hat{R}$
with real positive diagonal entries ($r_{ii}>0$) is unique. The
corresponding matrix $\hat{Q}$ is unique, too.

{\footnotesize \proof Let $A=\hat{Q}\hat{R}=\hat{Q}_1\hat{R}_1$ be two
different reduced QR decompositions with the specified properties. Then
$$
  A^{\ast}A = \hat{R}^{\ast}\hat{R}
$$
because $\hat{Q}^\ast\hat{Q} = I$ is the identity matrix (this follows
from the columns of $\hat{Q}$ being orthonormal vectors). Similarly,
$$
  A^{\ast}A = \hat{R}^{\ast}_1\hat{R}_1.
$$
Thus we get two Cholesky factorizations for the same matrix $A^\ast A$.
But this matrix is positive definite (due to Section~\ref{16.8}), hence its Cholesky factorization must be unique (cf.\ Section~\ref{13.5}).
This implies $\hat{R}=\hat{R}_1$. Lastly,
$\hat{Q}_1=\hat{Q}\hat{R}\hat{R}_1^{-1}=\hat{Q}$. \qed}

\subsection{Cost of QR} \label{QR7}

{\footnotesize In order to compute the reduced QR decomposition
\eqref{reducedQR}, one needs to apply Gram-Schmidt orthogonalization to
the $n$ columns of the matrix $A$ and compute the $n$ columns
$u_1,\ldots,u_n$ of the matrix $\hat{Q}$. The vector $u_p$ is found by
$$
   w_{p}=v_{p}-\sum_{i=1}^{p-1}\la v_{p},u_i\ra\, u_i,
   \quad \text{and} \quad
   u_p=\frac{w_p}{\|w_p\|},
$$
see Section~\ref{GramSchmidt}. Here each scalar product $\la
v_p,u_i\ra$ requires $m$ multiplications and $m$ additions, and then
subtracting every term $\la v_p,u_i\ra\, u_i$ from $v_p$ requires $m$
multiplication and $m$ subtractions, for each $i=1,\ldots,p$. The total
is $4mp$ flops. The subsequent computation of $\|w_{p+1}\|$ and then
$u_{p+1}$ requires $3m$ flops, which is a relatively small number, and
we ignore it. The total flop count is $\sum_{p=1}^n 4mp \approx 2mn^2$
flops, i.e., the QR takes
$$
    \boxed{\phantom\int 2mn^2\qquad{\rm flops}\phantom\int}
$$
}

\subsection{Cost of SVD} \label{SVDcom}

{\footnotesize There is no finite algorithms for the computation of SVD
described in Chapter~\ref{Ch5} (except for small matrices, see
Remark~\ref{ComSVD}). The reason will be discussed later in
Chapter~\ref{Ch15}. In practice, the SVD is computed by special
iterative algorithms. The computation of the reduced SVD requires
approximately
$$
    \boxed{\phantom\int2mn^2 + 11n^3\qquad{\rm flops}\phantom\int}
$$
}

\subsection{Modified Gram-Schmidt orthogonalization} \label{MGS}

The algorithm from Section~\ref{GramSchmidt} is often called {\em
classical} Gram-Schmidt orthogonalization, as opposed to the {\em
modified} Gram-Schmidt orthogonalization we present next. Given a
linearly independent set of vectors $\{v_1,\ldots,v_n\}$ in $\IC^m$, we
denote $v_i^{(1)}=v_i$ for $i=1,\ldots,n$, then compute
$$
           u_1=v_1^{(1)}/\|v_1^{(1)}\|,
$$
and then modify all the remaining vectors by the rule
$$
    v_i^{(2)} = v_i^{(1)} - \la v_i^{(1)},u_1 \ra\, u_1
    \qquad \text{for}\ \ 1<i\leq n.
$$
After that, inductively, for each $p\geq 2$ we compute
$$
   u_p=v_p^{(p)}/\|v_p^{(p)}\|,
$$
and then modify all the remaining vectors by the rule
$$
    v_i^{(p+1)} = v_i^{(p)} - \la v_i^{(p)},u_p \ra\, u_p
    \qquad \text{for}\ \ p<i\leq n.
$$
The modified and classical Gram-Schmidt methods produce the same
orthonormal basis $\{u_1,\ldots,u_n\}$ (i.e., these two methods are
mathematically equivalent). They are based on a different logic,
though.

The classical Gram-Schmidt computes $u_p$ by making the current vector
$v_p$ orthogonal to all the previously constructed vectors
$u_1,\ldots,u_{p-1}$, without touching the remaining vectors
$v_{p+1},\ldots,v_n$. The amount of work \underline{increases} as $p$
grows from 1 to $n$. This is a ``lazy man schedule'' - do as little as
possible and leave the rest of the work ``for later''.

The modified Gram-Schmidt computes $u_p$ and makes all the remaining
vectors $v_{p+1},\ldots,v_n$ orthogonal to it. Once this is done, the
remaining vectors will be in the orthogonal complement to the subspace
span$\{u_1,\ldots,u_p\}$ and there is no need to involve the previously
constructed vectors anymore. The amount of work \underline{decreases}
as $p$ grows from 1 to $n$. This is an ``industrious man schedule'' -
do as much as possible \emph{now} and reduce the workload.

Overall, both methods require the same amount of flops. But the
modified Gram-Scmidt has an important advantage that it gives more
accurate numerical results in computer calculations; see programming
assignment.



\newpage

\centerline{Classical Gram-Schmidt:}\bigskip

\begin{tikzpicture}
  \matrix(V)[matrix of math nodes,column sep=-3mm,row sep=2mm]
  {
  w_1 & = & v_1 & & & & &[9mm] u_1=\tfrac{w_1}{\|w_1\|} \\
  w_2 & = & v_2 & - \la v_2,u_1\ra u_1 & & & & u_2=\tfrac{w_2}{\|w_2\|} \\
  w_3 & = & v_3 & - \la v_3,u_1\ra u_1 &  - \la v_3,u_2\ra u_2 & & & u_3=\tfrac{w_3}{\|w_3\|} \\
  w_4 & = & v_4 & - \la v_4,u_1\ra u_1 &  - \la v_4,u_2\ra u_2 &  - \la v_4,u_3\ra u_3 & & u_4=\tfrac{w_4}{\|w_4\|} \\
   & \vdots  &  &  & \vdots  &  & & \vdots \\
  w_p & = & v_p & - \la v_p,u_1\ra u_1 &  - \la v_p,u_2\ra u_2 &  -\cdots & - \la v_p,u_{p-1}\ra u_{p-1} & u_p=\tfrac{w_p}{\|w_4\|} \\
   & \vdots  &  &  & \vdots  &  & \vdots & \vdots \\
          };
\begin{pgfonlayer}{background}
   \draw[->,-latex,line width=0.3mm,color=blue!40]
          (V-1-3.east) -- ($(V-1-3.east)!0.55!(V-1-8.west)$);
   \draw[line width=0.3mm,color=blue!40]
          ($(V-1-3.east)!0.5!(V-1-8.west)$) -- (V-1-8.west);
   \draw[olive,rounded corners,fill=olive!15]
          ($(V-1-8.north west)+(0.1,-0.05)$) rectangle ($(V-1-8.south east)-(0.1,-0.05)$);
   \draw[->,-latex,line width=0.3mm,color=blue!40]
          (V-1-8.west) -- ($(V-1-8.west)!0.55!(V-2-4.east)$);
   \draw[line width=0.3mm,color=blue!40]
          ($(V-1-8.west)!0.5!(V-2-4.east)$) -- (V-2-4.east);
   \draw[red,rounded corners,fill=red!15]
          ($(V-2-3.north west)+(0.1,0.1)$) rectangle ($(V-2-4.south east)-(0.1,0)$);
   \draw[->,-latex,line width=0.3mm,color=blue!40]
          (V-2-4.east) -- ($(V-2-4.east)!0.55!(V-2-8.west)$);
   \draw[line width=0.3mm,color=blue!40]
          ($(V-2-4.east)!0.5!(V-2-8.west)$) -- (V-2-8.west);
   \draw[olive,rounded corners,fill=olive!15]
          ($(V-2-8.north west)+(0.1,-0.05)$) rectangle ($(V-2-8.south east)-(0.1,-0.05)$);
   \draw[->,-latex,line width=0.3mm,color=blue!40]
          (V-2-8.west) -- ($(V-2-8.west)!0.55!(V-3-5.east)$);
   \draw[line width=0.3mm,color=blue!40]
          ($(V-2-8.west)!0.5!(V-3-5.east)$) -- (V-3-5.east);
   \draw[red,rounded corners,fill=red!15]
          ($(V-3-3.north west)+(0.1,0.1)$) rectangle ($(V-3-5.south east)-(0.1,0)$);
   \draw[->,-latex,line width=0.3mm,color=blue!40]
          (V-3-5.east) -- ($(V-3-5.east)!0.55!(V-3-8.west)$);
   \draw[line width=0.3mm,color=blue!40]
          ($(V-3-5.east)!0.5!(V-3-8.west)$) -- (V-3-8.west);
   \draw[olive,rounded corners,fill=olive!15]
          ($(V-3-8.north west)+(0.1,-0.05)$) rectangle ($(V-3-8.south east)-(0.1,-0.05)$);
   \draw[->,-latex,line width=0.3mm,color=blue!40]
          (V-3-8.west) -- ($(V-3-8.west)!0.55!(V-4-6.east)$);
   \draw[line width=0.3mm,color=blue!40]
          ($(V-3-8.west)!0.5!(V-4-6.east)$) -- (V-4-6.east);
   \draw[red,rounded corners,fill=red!15]
          ($(V-4-3.north west)+(0.1,0.1)$) rectangle ($(V-4-6.south east)-(0.1,0)$);
   \draw[->,-latex,line width=0.3mm,color=blue!40]
          (V-4-6.east) -- ($(V-4-6.east)!0.55!(V-4-8.west)$);
   \draw[line width=0.3mm,color=blue!40]
          ($(V-4-6.east)!0.5!(V-4-8.west)$) -- (V-4-8.west);
   \draw[olive,rounded corners,fill=olive!15]
          ($(V-4-8.north west)+(0.1,-0.05)$) rectangle ($(V-4-8.south east)-(0.1,-0.05)$);
   \draw[->,-latex,line width=0.3mm,color=blue!40]
          (V-4-8.west) -- ($(V-4-8.west)!0.2!(V-6-5.east)$);
   \draw[dashed,line width=0.3mm,color=blue!40]
          ($(V-4-8.west)!0.15!(V-6-5.east)$) -- ($(V-4-8.west)!0.4!(V-6-5.east)$);
   \draw[red,rounded corners,fill=red!15]
          ($(V-6-3.north west)+(0.1,0.1)$) rectangle ($(V-6-7.south east)-(0.1,0)$);
   \draw[->,-latex,line width=0.3mm,color=blue!40]
           ($(V-6-7.east)!0.1!(V-6-8.west)$) -- ($(V-6-7.east)!0.9!(V-6-8.west)$);
   \draw[olive,rounded corners,fill=olive!15]
          ($(V-6-8.north west)+(0.1,-0.05)$) rectangle ($(V-6-8.south east)-(0.1,-0.05)$);
   \draw[->,-latex,line width=0.3mm,color=blue!40]
          (V-6-8.west) -- ($(V-6-8.west)!0.3!(V-7-7.south east)$);
   \draw[dashed,line width=0.3mm,color=blue!40]
          (V-6-8.west) -- ($(V-6-8.west)!0.6!(V-7-7.south east)$);
\end{pgfonlayer}
\end{tikzpicture}

\begin{center}
The amount of work increases at each step (the red rows grow longer)
\end{center}

\bigskip\bigskip\centerline{Modified Gram-Schmidt:}\bigskip

\begin{tikzpicture}
  \matrix(V)[matrix of math nodes,column sep=-3mm,row sep=2mm]
  {
  w_1 & = & v_1 & & & & &[9mm] u_1=\tfrac{w_1}{\|w_1\|} \\
  w_2 & = & v_2 & - \la v_2,u_1\ra u_1 & & & & u_2=\tfrac{w_2}{\|w_2\|} \\
  w_3 & = & v_3 & - \la v_3,u_1\ra u_1 &  - \la v_3,u_2\ra u_2 & & & u_3=\tfrac{w_3}{\|w_3\|} \\
  w_4 & = & v_4 & - \la v_4,u_1\ra u_1 &  - \la v_4,u_2\ra u_2 &  - \la v_4,u_3\ra u_3 & & u_4=\tfrac{w_4}{\|w_4\|} \\
   & \vdots  &  &  & \vdots  &  & & \vdots \\
  w_p & = & v_p & - \la v_p,u_1\ra u_1 &  - \la v_p,u_2\ra u_2 &  -\cdots & - \la v_p,u_{p-1}\ra u_{p-1} & u_p=\tfrac{w_p}{\|w_4\|} \\
   & \vdots  &  &  & \vdots  &  & \vdots & \vdots \\
          };
\begin{pgfonlayer}{background}
   \draw[->,-latex,line width=0.3mm,color=blue!40]
          (V-1-3.east) -- ($(V-1-3.east)!0.55!(V-1-8.west)$);
   \draw[line width=0.3mm,color=blue!40]
          ($(V-1-3.east)!0.5!(V-1-8.west)$) -- (V-1-8.west);
   \draw[olive,rounded corners,fill=olive!15]
          ($(V-1-8.north west)+(0.1,-0.05)$) rectangle ($(V-1-8.south east)-(0.1,-0.05)$);
   \draw[->,-latex,line width=0.3mm,color=blue!40]
          (V-1-8.west) -- ($(V-1-8.west)!0.55!(V-2-4.east)$);
   \draw[line width=0.3mm,color=blue!40]
          ($(V-1-8.west)!0.5!(V-2-4.east)$) -- (V-2-4.east);
   \draw[red,rounded corners,fill=red!15]
          ($(V-2-3.north west)+(0.1,0.1)$) rectangle ($(V-6-4.south east)-(0.2,1.2)$);
   \draw[->,-latex,line width=0.3mm,color=blue!40]
          (V-2-4.east) -- ($(V-2-4.east)!0.55!(V-2-8.west)$);
   \draw[line width=0.3mm,color=blue!40]
          ($(V-2-4.east)!0.5!(V-2-8.west)$) -- (V-2-8.west);
   \draw[olive,rounded corners,fill=olive!15]
          ($(V-2-8.north west)+(0.1,-0.05)$) rectangle ($(V-2-8.south east)-(0.1,-0.05)$);
   \draw[->,-latex,line width=0.3mm,color=blue!40]
          (V-2-8.west) -- ($(V-2-8.west)!0.55!(V-3-5.east)$);
   \draw[line width=0.3mm,color=blue!40]
          ($(V-2-8.west)!0.5!(V-3-5.east)$) -- (V-3-5.east);
   \draw[red,rounded corners,fill=red!15]
          ($(V-3-5.north west)+(0.2,0.1)$) rectangle ($(V-6-5.south east)-(0.2,1.2)$);
   \draw[->,-latex,line width=0.3mm,color=blue!40]
          (V-3-5.east) -- ($(V-3-5.east)!0.55!(V-3-8.west)$);
   \draw[line width=0.3mm,color=blue!40]
          ($(V-3-5.east)!0.5!(V-3-8.west)$) -- (V-3-8.west);
   \draw[olive,rounded corners,fill=olive!15]
          ($(V-3-8.north west)+(0.1,-0.05)$) rectangle ($(V-3-8.south east)-(0.1,-0.05)$);
   \draw[->,-latex,line width=0.3mm,color=blue!40]
          (V-3-8.west) -- ($(V-3-8.west)!0.55!(V-4-6.east)$);
   \draw[line width=0.3mm,color=blue!40]
          ($(V-3-8.west)!0.5!(V-4-6.east)$) -- (V-4-6.east);
   \draw[red,rounded corners,fill=red!15]
          ($(V-4-6.north west)+(0.2,0.1)$) rectangle ($(V-4-6.south east)-(0.2,3.2)$);
   \draw[->,-latex,line width=0.3mm,color=blue!40]
          (V-4-6.east) -- ($(V-4-6.east)!0.55!(V-4-8.west)$);
   \draw[line width=0.3mm,color=blue!40]
          ($(V-4-6.east)!0.5!(V-4-8.west)$) -- (V-4-8.west);
   \draw[olive,rounded corners,fill=olive!15]
          ($(V-4-8.north west)+(0.1,-0.05)$) rectangle ($(V-4-8.south east)-(0.1,-0.05)$);
   \draw[->,-latex,line width=0.3mm,color=blue!40]
          (V-4-8.west) -- ($(V-4-8.west)!0.2!(V-6-5.east)$);
   \draw[dashed,line width=0.3mm,color=blue!40]
          ($(V-4-8.west)!0.15!(V-6-5.east)$) -- ($(V-4-8.west)!0.4!(V-6-5.east)$);
   \draw[red,rounded corners,fill=red!15]
          ($(V-6-7.north west)+(0.2,0.1)$) rectangle ($(V-6-7.south east)-(0.1,1.2)$);
   \draw[->,-latex,line width=0.3mm,color=blue!40]
           ($(V-6-7.east)!0.1!(V-6-8.west)$) -- ($(V-6-7.east)!0.9!(V-6-8.west)$);
   \draw[olive,rounded corners,fill=olive!15]
          ($(V-6-8.north west)+(0.1,-0.05)$) rectangle ($(V-6-8.south east)-(0.1,-0.05)$);
   \draw[->,-latex,line width=0.3mm,color=blue!40]
          (V-6-8.west) -- ($(V-6-8.west)!0.2!(V-7-7.south east)$);
   \draw[dashed,line width=0.3mm,color=blue!40]
          (V-6-8.west) -- ($(V-6-8.west)!0.4!(V-7-7.south east)$);
\end{pgfonlayer}
\end{tikzpicture}

\begin{center}
The amount of work decreases at each step (the red columns get shorter)
\end{center}

\begin{exercise}
(JPE, September 2002) Consider three vectors
$$
  v_1=\begin{pmatrix} 1 \\ \epsilon \\ 0 \\ 0 \end{pmatrix},
  \quad
  v_2=\begin{pmatrix} 1 \\ 0 \\ \epsilon \\ 0 \end{pmatrix},
  \quad
  v_3=\begin{pmatrix} 1 \\ 0 \\ 0 \\ \epsilon \end{pmatrix}.
$$
where $\epsilon\ll 1$.
\begin{itemize}
\item[(a)] Use the classical Gram-Schmidt method to compute 3
    \textbf{orthonormal} vectors $q_1, q_2, q_3$, making the
    approximation that $1+\epsilon^2\approx 1$ (that is, replace
    any term containing $\epsilon^2$ or smaller with zero,
    \textbf{but} retain terms containing $\epsilon$). Are $q_i$
    ($i=1,2,3$) pairwise orthogonal? If not, why not? \item[(b)]
    Repeat (a) using the modified Gram-Schmidt orthogonalization
    process. Are the $q_i$($i=1,2,3$) pairwise orthogonal? If not,
why not?
\end{itemize}
\end{exercise}

\newpage

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\refstepcounter{section} \label{Ch10}

\centerline{Chapter \ref{Ch10} }\vspace{5mm}

\centerline{\Large\bf Least Squares}\vspace{3mm}

\addcontentsline{toc}{section}{\ref{Ch10}. Least Squares}

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\setcounter{subsection}{0} \setcounter{exercise}{0}
\setcounter{equation}{0}

\subsection{Definition}
\label{16.1}

A system of linear equations $Ax=b$ with $A\in\IC^{m\times n}$, $x\in
\IC^n$ and $b\in\IC^m$, is said to be {\em overdetermined} if $m>n$.
Since there are more equations than unknowns, the system usually has no
solutions; so we will write it as $Ax \approx b$.


\subsection{Conditions for existence and uniqueness of a solution}
\label{16.1a}

{\small Recall that every matrix $A\in\IC^{m\times n}$ defines a linear
transformation $\IC^n\to\IC^m$. Since it takes a smaller space into a
larger one, its range is a proper subset of $\IC^m$.

\begin{center}
\begin{tikzpicture}
 \draw[thick,green,fill=green!20] (0,0) ellipse (1.5 and 0.375);
 \node[anchor=north] at (0,-0.375) {$\IC^n$};
 \draw[dashed,green,fill=green!15] (7,0) ellipse (1.5 and 0.375);
 \node[anchor=north east,rotate=-10] at (7,-0.375) {\tiny ${\rm Range}\, A$};
 \draw[thick,gray] (7,0) ellipse (1.5 and 0.8);
 \node[anchor=north west] at (8.4,-0.2) {$\IC^m$};
 \draw[line width=1mm,gray,->,-latex] (2.5,0) -- (4.5,0);
 \node[anchor=south] at (3.5,0) {$A$};
\end{tikzpicture}
\end{center}
\vspace{-3mm}

\noindent It is clear that the equation $Ax=b$ has a solution if and
only if $b\in\,{\rm Range}\,A$. In the latter case the solution is
unique if and only if our linear transformation is injective. The
latter occurs if and only if Ker$\, A=\{0\}$ (cf.\
Section~\ref{SurjectInject}).}

\subsection{Least squares solution} \label{16.5}

Let $Ax\approx b$ be an overdetermined linear system. A vector
$x\in\IC^n$ that minimizes the function
$$
        E(x)=\|b-Ax\|_2
$$
is called a {\em least squares solution} of $Ax\approx b$. The vector
$r=b-Ax$ is called the {\em residual vector} and $\|r\|_2$ the {\em
residual norm}.

\subsection{Normal equations} \label{16.7}

Let $Ax \approx b$ be an overdetermined linear system. The linear
system
\beq \label{normalEq}
        A^{\ast}Ax=A^{\ast}b
\eeq
is called the {\em system of normal equations} associated with
$Ax\approx b$.\smallskip

{\small Note that $A^\ast A\in \IC^{n\times n}$, hence \eqref{normalEq}
is an ordinary system of linear equations (not an overdetermined
system).}

\subsection{Theorem} \label{16.9}

Let $Ax\approx b$ be an overdetermined linear system. Then
\begin{itemize}
\item[(a)] A vector $x$ minimizes $E(x)=\|b-Ax\|_2$ if and only if
it is an exact solution of the system $Ax=\hat{b}$, where $\hat{b}$ is
the orthogonal projection of $b$ onto Range$\, A$.
\item[(b)] A vector $x$ minimizing $E(x)$ always exists. It is
    unique if and only if $A$ has full rank (equivalently, Ker$\,
    A=\{0\}$).
\item[(c)] A vector $x$ minimizes $E(x)$ if and only if it is
a solution of the system of normal equations $A^{\ast}Ax
=A^{\ast}b$.
\end{itemize}

{\footnotesize {\em Proof. }Denote $W=\,{\rm Range}\, A$. We have an
orthogonal decomposition $\IC^m=W \oplus W^\perp$, in particular
$b=\hat{b}+r$, where $\hat{b}\in W$ and $r\in W^\perp$ are uniquely
determined by $b$. Now Pythagorean Theorem~\ref{Pythagoras} gives
\begin{align*}
  [E(x)]^2 = \|b-Ax\|_2^2 &=\|\underbrace{b-\hat{b}}_{=r\in W^\perp}
   +\underbrace{\hat{b}-Ax}_{\in W}\|^2 \\
   &= \|r\|_2^2 + \|\hat{b}-Ax\|_2^2\geq \|r\|_2^2
\end{align*}
Hence, $\min_x E(x)=\|r\|_2$ is attained whenever $Ax=\hat{b}$. Since
$\hat{b}\in{\rm Range}\, A$, there is always a vector $x\in\IC^n$ such
that $Ax=\hat{b}$. The vector $x$ is unique whenever the map $A\colon
\IC^n \to \IC^m$ is injective, i.e., whenever Ker$\, A=\{0\}$, i.e.,
whenever $A$ has full rank. This proves (a) and (b).

To prove (c), recall that $({\rm Range}\, A)^\perp =\,{\rm Ker}\,
A^{\ast}$ ({\footnotesize by Section~\ref{11.7}}), therefore
$r=b-\hat{b}\in\, {\rm Ker}\, A^\ast$. Moreover, $b-Ax \in\, {\rm
Ker}\, A^\ast$ if and only if $Ax=\hat{b}$, because $\hat{b}$ and $r$
are uniquely determined by $b$. Now
$$
  x\text{ minimizes }E(x)\ \
  \Leftrightarrow\ \
  Ax=\hat{b}\ \
  \Leftrightarrow\ \
  Ax-b \in\, {\rm Ker}\, A^\ast\ \
  \Leftrightarrow\ \
  A^\ast Ax=A^\ast b
$$
The proof is complete. \hfill $\square$ \vspace{-5mm}}

\begin{center}
\begin{tikzpicture}
 \draw[thick,green,fill=green!20] (0,0) ellipse (2 and 0.5);
 \node[anchor=north] at (0,-0.5) {$\IC^n$};
 \draw[dashed,green,fill=green!15] (7,0) ellipse (2 and 0.5);
 \draw[thick,->,-latex,red] (7,0) -- +(1,1);
 \node[anchor=west] at (8,1) {$b$};
 \draw[thick,->,-latex,teal] (7,0) -- +(1,0);
 \node[anchor=west] at (8,0) {$\hat{b}$};
 \path[fill=blue!15] (6.9,-2) -- (7,0) -- (7.1,-2);
 \path[fill=blue!15] (6.9,2) -- (7,0) -- (7.1,2);
 \draw[thick,->,-latex,blue] (7,0) -- +(0,1);
 \node[anchor=east] at (7,0.5) {$r$};
 \node[anchor=north east,rotate=-10] at (6.9,-0.5) {\tiny $W\!={\rm Range}\, A$};
 \node[anchor=east] at (7,1.5) {\tiny $W^\perp$};
 \draw[thick,red] (7,0) circle(2);
 \node[anchor=north west] at (8.41,-1.41) {$\IC^m$};
 \draw[line width=1mm,gray,->,-latex] (2.5,0) -- (4.5,0);
 \node[anchor=south] at (3.5,0) {$A$};
 \node at (12,0) {};
\end{tikzpicture}
\end{center}

\newpage

Next we give examples that lead to overdetermined systems and least
squares problems.

\subsection{Linear least squares fit} \label{16.2}

Let $(x_i,y_i)$, $1\leq i\leq m$, be points in the $xy$ plane. For any
straight line $y=a_0+a_1x$ one defines the ``combined distance'' from
that line to the given points by
$$
        E(a_0,a_1)=\biggl[\sum_{i=1}^m (a_0+a_1x_i-y_i)^2\biggr]^{1/2}
$$
The line $y=a_0+a_1x$ that minimizes the function $E(a_0,a_1)$ is
called the {\em least squares fit} to the points $(x_i,y_i)$. This
is a basic tool in statistics. Let
$$
    A=\left[\begin{array}{cc} 1 & x_1 \\ 1 & x_2 \\
     \vdots & \vdots \\ 1 & x_m \end{array}\right]
   \ \ \ \ {\bf x}=\left[\begin{array}{c} a_0 \\ a_1 \end{array}\right]
   \ \ \ \ {\bf b}=\left[\begin{array}{c} y_1 \\ y_2 \\ \vdots \\
    y_m\end{array}\right]
$$
Then
$$
        E(a_0,a_1)=\|{\bf b}-A{\bf x}\|_2
$$
Hence the least squares fit is equivalent to the least squares problem
$A{\bf x}\approx {\bf b}$.

\subsection{Polynomial least squares fit} \label{16.27}

Generalizing \ref{16.2}, one can fit a polynomial
$y=p(x)=a_0+a_1x+\cdots +a_nx^n$ of degree $n\geq 2$ to a set of data
points $(x_i,y_i)$, $1\leq i\leq m$, with $m>n+1$. The least squares
fit is based on minimizing the function
$$
   E(a_0,\ldots,a_n)=
   \biggl[\sum_{i=1}^m\Big (a_0+a_1x_i+
   \cdots +a_nx_i^n-y_i\Big )^2\biggl]^{1/2}
$$
This is equivalent to the least squares problem for an overdetermined
linear system
$$
   a_0+a_1x_i+\cdots +a_nx_i^n \approx y_i\ \ \ \ \ 1\leq i\leq m
$$
in which $a_0,\ldots,a_n$ are unknowns.\bigskip

%\begin{mdframed}[hidealllines=true,backgroundcolor=green!10]
%\end{mdframed}

\newpage

\subsection{Continuous least squares fit} \label{16.28}

{\footnotesize Instead of fitting a polynomial $p(x) \in P_n(\IR)$ of
degree $n$ to a discrete data set $(x_i,y_i)$ one can fit it to a
continuous function $y=f(x)$ on $[0,1]$. The least squares fit is based
on minimization of
$$
   E(a_0,\ldots,a_n)=
   \biggl[\int_0^1|f(x)-p(x)|^2\, dx\biggr]^{1/2}
$$
The solution to this problem is the orthogonal projection of $f(x)$
onto $P_n(\IR)$.

To find the solution, we use the basis $\{1,x,\ldots,x^n\}$ in
$P_n(\IR)$. Then $a_0,\ldots,a_n$ can be found by solving the system of
equations (analogous to normal equations)
$$
     \sum_{j=0}^n a_j\la x^j,x^i\ra =
     \la f,x^i\ra \ \ \ \ \ \ \ 1\leq i\leq n.
$$
The matrix of coefficients here is
$$
     \la x^j,x^i\ra =\int_0^1 x^{i+j}\, dx=\frac{1}{1+i+j}
$$
for $0\leq i,j\leq n$.}

\new Next we present three computer algorithms for solving the least square
problem.

\subsection{Algorithm 1, based on normal equations} \label{A1}

{\footnotesize This is the simplest one:

\medskip\noindent 1.  Form the matrix $A^{\ast}A$ and the vector $A^{\ast}b$.\\
2. Compute the Cholesky factorization $A^{\ast}A=GG^{\ast}$.\\ 3. Solve
the lower-triangular system $Gz=A^{\ast}b$ for $z$ by forward
substitution \ref{FBS}.\\ 4. Solve the upper-triangular system
$G^{\ast}x=z$ for $x$ by backward substitution \ref{FBS}.

\medskip\noindent The cost of this algorithm is dominated by Steps 1 and 2.
Because the matrix $A^\ast A$ is symmetric, its computation requires
$mn(n+1)\approx mn^2$ flops. The computation of $A^{\ast}b$ requires
only $2mn$ flops, a relatively small amount which we can ignore. The
Cholesky factorization takes $n^3/3$ flops (cf.\ Section~\ref{13.7}).
Hence the total cost is
$$
    \boxed{\phantom\int\approx mn^2+\tfrac 13
    n^3\qquad{\rm flops}\phantom\int}
$$
}

\noindent \textbf{Note:} the above method is a \emph{computer}
algorithm. For manual work (on paper), solving the least squares
problem by using normal equations means the following: you compute the
matrix $A^{\ast}A$ and the vector $A^{\ast}b$ and then solve the system
$A^{\ast}Ax=A^{\ast}b$ for $x$ any way you want (by Gaussian
elimination, for example).

\subsection{Algorithm 2, based on QR decomposition} \label{A2}

{\footnotesize Using the reduced QR decomposition (Section~\ref{QR5})
allows us to rewrite the system of normal equations as
$$
    \hat{R}^\ast\hat{Q}^{\ast}\hat{Q}\hat{R}x
    = \hat{R}^\ast\hat{Q}^\ast b
$$
If $A$ has full rank, the matrix $\hat{R}^\ast$ is nonsingular and
we cancel it out. Also, since the columns of $\hat{Q}$ are
orthonormal vectors, $\hat{Q}^{\ast}\hat{Q}=I$. Hence
$$
   \hat{R}x=\hat{Q}^{\ast}b
$$
This suggests the following algorithm:

\new 1. Compute the reduced QR decomposition $A=\hat{Q}\hat{R}$.\\
2. Compute the vector $\hat{Q}^{\ast}b$.\\ 3. Solve the
upper-triangular system $\hat{R}x=\hat{Q}^{\ast}b$ for $x$ by backward
substitution \ref{FBS}.

\new The cost of this algorithm is dominated by Step 1, the
reduced QR factorization, which requires
$$
    \boxed{\phantom\int \approx 2mn^2\qquad
     {\rm flops}\phantom\int}
$$
see Section~\ref{QR7}. This is approximately twice as much as
Algorithm~1 takes.}

\subsection{Algorithm 3, based on SVD} \label{A3}

{\footnotesize Using the reduced SVD (Section~\ref{SVDred}) allows us
to rewrite the system of normal equations as
$$
    V\hat{D}\hat{U}^\ast\hat{U}\hat{D}V^{\ast}x
    = V\hat{D}\hat{U}^\ast b
$$
The matrix $V$ is unitary, and we cancel it out. If $A$ has full rank,
the matrix $\hat{D}$ is nonsingular and we cancel it, too. Since the
columns of $\hat{U}$ are orthonormal vectors,
$\hat{U}^{\ast}\hat{U}=I$. Hence
$$
   \hat{D}V^{\ast}x=\hat{U}^{\ast}b
$$
This suggests the following algorithm:

\new 1. Compute the reduced SVD decomposition $A=\hat{U}\hat{D}V^{\ast}$.\\
2. Compute the vector $\hat{U}^{\ast}b$.\\ 3. Solve the diagonal system
$\hat{D}z=\hat{U}^{\ast}b$ for $z$. \\ 4. Compute $x=Vz$.

\new The cost of this algorithm is dominated by step 1, the
reduced SVD decomposition, which requires
$$
    \boxed{\phantom\int \approx 2mn^2+11n^3
    \qquad {\rm flops}\phantom\int}
$$
see Section~\ref{SVDcom}. This is approximately the same amount as in
Algorithm 2 for $m\gg n$, but for $n\approx m$ this algorithm is much
more expensive. }

Algorithm 1 is the simplest and the cheapest, but it often gives
inaccurate results in numerical calculations. Algorithms 2 and 3 are
more complicated and expensive (in terms of flops), but usually give
more accurate numerical results, for the reasons we learn in the next
chapters.

\subsection{Rank deficient matrix $A$} \label{Adefic}

If rank$\,A<n$, then the least squares solution is not unique: the set
of solutions is $\{x\in \IC^n \colon Ax = \hat{b}\}$, which is a line
or a plane in $\IC^n$ parallel to Ker$\,A$. In this case the ``best''
solution is the one of minimal norm:
$$
      Ax_{\text{best}}=\hat{b}\quad\text{and}\quad
      \|x_{\text{best}}\|_2\leq\|x\|_2\quad
      \forall x\colon Ax=\hat{b}
$$
Algorithms 1 and 2 fail to find any solution for a rank deficient
matrix $A$. On the contrary, Algorithm~3 easily finds the minimal norm
solution as follows.

When solving the diagonal system $\hat{D}z=\hat{U}^{\ast}b$ for $z$, we
just set $z_i=0$ whenever $d_{ii}=0$ (in that case
$(\hat{U}^{\ast}b)_i=0$ automatically). This obviously gives a minimal
norm vector $z$. Since $\|x\|_2 = \|Vz\|_2 = \|z\|_2$, we get a minimal
norm vector $x$ as well.

\begin{exercise}
(JPE, September 1997) Let
$$
  A=\left [\begin{array}{rr} 3 & 3 \\ 0 & 4 \\ 4 & -1 \end{array}\right
  ],\ \ \ \ {\rm and}\ \ \ \
  b=\left [\begin{array}{r} 2 \\ -2 \\ 1\end{array}\right ]
$$
Use the Gram-Schmidt process to find an orthonormal basis for the
column space of $A$. Factor $A$ into a product $QR$ where
$Q\in\IR^{3\times 2}$ has an orthonormal set of column vectors and
$R\in\IR^{2\times 2}$ is upper triangular. Solve the least squares
problem $Ax=b$. Compute the norm of the residual vector, $\|r\|$.
\end{exercise}

\begin{exercise}
(JPE, May 1998) Given the data (0,1), (3,4) and (6,5), use a QR
factorization technique to find the best least squares fit by a
linear function. Also, solve the problem via the system of normal
equations.
\end{exercise}


\newpage

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\refstepcounter{section} \label{Ch11}

\centerline{Chapter \ref{Ch11} }\vspace{5mm}

\centerline{\Large\bf Machine Arithmetic}\vspace{3mm}

\addcontentsline{toc}{section}{\ref{Ch11}. Machine Arithmetic}

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\setcounter{subsection}{0} \setcounter{exercise}{0}
\setcounter{equation}{0}

\subsection{Decimal number system} \label{14.1}

In our decimal system, natural numbers are represented by a sequence of
digits. For example, $582=5\cdot 10^2 + 8\cdot 10+2$. Generally,
$$
    a_n\cdots a_1a_0=10^na_n+\cdots +10a_1+a_0,
$$
where $a_i\in\{0, 1, 2, 3, 4, 5, 6, 7, 8, 9\}$ are digits.
Fractional numbers require an additional {\em fractional part}:
$$
     f=.d_1d_2\ldots d_t\ldots=
     10^{-1}d_1+10^{-2}d_2+\cdots+10^{-t}d_t+\cdots
$$
which may be finite or infinite.

\subsection{Floating point representation} \label{14.2}

Alternatively, any real number can be written as a product of a
fractional part with a sign and a power of ten:
$$
     \bigl(\pm .d_1d_2\ldots d_t\ldots\bigr)\times 10^e
$$
where $d_i$ are decimal digits and $e\in{\mathbb Z}$ is an integer. For
example,
\beq \label{582}
   58.2=0.582\times 10^2=0.0582\times 10^3,\quad\text{etc.}
\eeq
This is called {\em floating point} representation of decimal numbers.
The part $.d_1d_2\ldots d_t\ldots$ is called {\em mantissa} and $e$ is
called {\em exponent}. By changing the exponent $e$ with a fixed
mantissa $.d_1d_2\ldots d_t\ldots$ we can move (``float'') the decimal
point, for example $0.582\times 10^2=58.2$ and $0.582\times 10^1=5.82$.

\newpage

\subsection{Normalized floating point representation}
\label{14.3}

To avoid unnecessary multiple representations of the same number, such
as in \eqref{582}, we can require that $d_1\neq 0$. We say the floating
point representation is {\em normalized} if $d_1\neq 0$. Then
$0.582\times 10^2$ is the only normalized representation of the number
58.2.

For every positive real $r>0$ there is a unique integer $e\in{\mathbb
Z}$ such that $f =10^{-e}r\in [0.1,1)$. Then $r=f\times 10^e$ is the
normalized representation of $r$. For most of real numbers, the
normalized representation is unique, however, there are exceptions,
such as
$$
    \bigl(.9999\ldots\bigr)\times 10^0 =
     \bigl(.1000\ldots\bigr)\times 10^1,
$$
both representing the number $r=1$. In such cases one of the two
representations has a finite fractional part and the other has an
infinite fractional part with trailing nines.


\subsection{Binary number system} \label{14.4}

In the binary number system, the base is 2 (instead of 10), and there
are only two digits: 0 and 1. Any natural number $N$ can be written, in
the binary system, as a sequence of binary digits:
$$
    N=(a_n\cdots a_1a_0)_2=2^na_n+\cdots +2a_1+a_0
$$
where $a_i \in \{0,1\}$. For example, $5=101_2$, $11=1011_2$,
$64=1000000_2$, etc. Binary system, due to its simplicity, is used by
all computers. In the modern computer world, the word {\em bit} means
{\em b}inary dig{\em it}.

Fractional numbers require an additional {\em fractional part}:
$$
     f=(.d_1d_2\ldots d_t\ldots)_2=
     2^{-1}d_1+2^{-2}d_2+\cdots+2^{-t}d_t+\cdots
$$
which may be finite or infinite. For example, $0.5 = 0.1_2$,
$0.625=0.101_2$, and
$$
   0.6 = (0.10011001100110011\ldots)_2
$$
(the blocks $00$ and $11$ alternate indefinitely). The floating point
representation of real numbers in the binary system is given by
$$
     r=\bigl(\pm .d_1d_2\ldots d_t\ldots\bigr)_2\times 2^e
$$
where $.d_1d_2\ldots d_t\ldots$ is called {\em mantissa} and
$e\in{\mathbb Z}$ is called {\em exponent}. Again, we say that the
above representation is normalized if $d_1\neq 0$, this ensures
uniqueness for almost all real numbers. Note that $d_1 \neq 0$ implies
$d_1=1$, i.e., every normalized binary representation begins with a
one.

\newpage

\subsection{Other number systems} \label{14.5}

We can use a number system with any fixed base $\beta\geq 2$. By
analogy with Sections~\ref{14.1} and \ref{14.4}, any natural number $N$
can be written, in that system, as a sequence of digits:
$$
    N=(a_n\cdots a_1a_0)_{\beta}=\beta^na_n+\cdots +\beta a_1+a_0
$$
where $a_i\in\{0, 1,\ldots,\beta-1\}$ are digits. Fractional
numbers require an additional {\em fractional part}:
$$
     f=.d_1d_2\ldots d_t\ldots=
     \beta^{-1}d_1+\beta^{-2}d_2+\cdots+\beta^{-t}d_t+\cdots
$$
which may be finite or infinite. The floating point representation of
real numbers in the system with base $\beta$ is given by
$$
     r=\bigl(\pm .d_1d_2\ldots d_t\ldots\bigr)_{\beta}\times \beta^e
$$
where $.d_1d_2\ldots d_t\ldots$ is called {\em mantissa} and
$e\in{\mathbb Z}$ is called {\em exponent}. Again, we say that the
above representation is normalized if $d_1\neq 0$, this ensures
uniqueness for almost all real numbers.\medskip

{\footnotesize In the real world, computers can only handle a certain
fixed number of digits in their electronic memory. For the same reason,
possible values of the exponent $e$ are always limited to a certain
fixed interval. This motivates our next definition.}

\subsection{Machine number systems (an abstract version)} \label{14.6}

A \emph{machine number system} is specified by four integers,
$(\beta,t,L,U)$, where\medskip

\noindent$\phantom{m}\qquad$ $\beta\geq 2$ is the base\\
$\phantom{m}\qquad$ $t\geq 1$ is the length of the mantissa\\
$\phantom{m}\qquad$ $L\in\IZ$ is the minimal value for the exponent $e$\\
$\phantom{m}\qquad$ $U\in\IZ$ is the maximal value for the exponent $e$
{\footnotesize (of course, $L\leq U$)}.\medskip

\noindent Real numbers in a machine system are represented by
\beq \label{MachineSystem}
     r=\bigl(\pm .d_1d_2\ldots d_t\bigr)_{\beta}\times \beta^e,
     \qquad L\leq e\leq U
\eeq
The representation must be normalized, i.e., $d_1\neq 0$.\medskip

\subsection{Basic properties of machine systems}
\label{BasicPropMachineSystems}

{\footnotesize Suppose we use a machine system with parameters
$(\beta,t,L,U)$. There are finitely many real numbers that can be
represented by \eqref{MachineSystem}. More precisely, there are
$$
   2(\beta-1)\beta^{t-1}(U-L+1)
$$
such numbers. The largest machine number is
$$
  M=\bigl(.(\beta-1)\beta\ldots \beta\bigr)_{\beta}\times \beta^U
  =\beta^U(1-\beta^{-t}).
$$
The smallest positive machine number is
$$
   m=\bigl(.10\ldots 0\bigr)_{\beta}\times \beta^L
   =\beta^{L-1}.
$$
Note that zero cannot be represented in the above format, since we
require $d_1\neq 0$. For this reason, every real machine systems
includes a few special numbers, like zero, that have to be represented
differently. Other ``special numbers'' are $+\infty$ and $-\infty$.}

\subsection{Two standard machine systems}\label{14.8}

{\footnotesize Most modern computers conform to the IEEE floating-point
standard (ANSI/IEEE Standard 754-1985), which specifies two machine
systems:\begin{itemize}\item[\textcircled{\scriptsize I}] {\em Single
precision} is defined by $\beta=2$, $t=24$, $L=-125$ and
$U=128$.\item[\textcircled{\scriptsize II}] {\em Double precision} is
defined by $\beta=2$, $t=53$, $L=-1021$ and $U=1024$.
\end{itemize}
}

\subsection{Rounding rules} \label{14.9}

A machine system with parameters $(\beta,t,L,U)$ provides
\emph{exact} representation for finitely many real numbers. Other
real numbers have to be \emph{approximated} by machine numbers.
Suppose $x\neq 0$ be a real number with normalized floating point
representation
$$
   x=\bigl(\pm 0.d_1d_2\ldots\bigr)_{\beta}\times \beta^e
$$
where the number of digits may be finite or infinite.

{\footnotesize If $e>U$ or $e<L$, then $x$ cannot be properly
represented in the machine system (it is either ``too large'' or ``too
small''). If $e<L$, then $x$ is usually converted to the special number
zero. If $e>U$, then $x$ is usually converted to the special number
$+\infty$ or $-\infty$.}

If $e\in [L,U]$ is within the proper range, then the mantissa of $x$
has to be reduced to $t$ digits (if it is longer than that or
infinite). There are two standard versions of such reductions:
\begin{itemize} \item[(a)] keep the first $t$ digits and \textbf{chop off} the
rest; \item[(b)] \textbf{round off} to the nearest available, i.e.
use the rules
$$
  \left \{ \begin{array}{ll}
  .d_1\ldots d_t & {\rm if}\ \ d_{t+1}<\beta/2  \\
  .d_1\ldots d_t+.0\ldots 01 \ \ \ & {\rm if}\ \ d_{t+1}\geq\beta/2
      \end{array}\right .
$$
\end{itemize}


\subsection{Relative errors} \label{14.9a}

Let $x$ be a real number and $x_c$ its \emph{computer
representation} in a machine system with parameters $(\beta,t,L,U)$,
as described above. {\footnotesize We will always assume that $x$ is
neither too big nor too small, i.e., its exponent $e$ is within the
proper range $L\leq e\leq U$.} How accurately does $x_c$ represent
$x$? (How close is $x_c$ to $x$?)

The \emph{absolute error} of the computer representation, i.e.,
$x_c-x$, may be quite large when the exponent $e$ is big. It is more
customary to describe the accuracy in terms of the \emph{relative
error} $(x_c-x)/x$ as follows:
$$
     \frac{x_c-x}{x}=\varepsilon
       \ \ \ \ {\rm or} \ \ \ \
     x_c=x(1+\varepsilon).
$$
It is easy to see that the maximal possible value of $|\varepsilon|$ is
$$
     |\varepsilon| \leq {\bu}=\left\{\begin{array}{rl}
 \beta^{1-t}\ \ \ &
 {\rm for}\ {\rm chopped}\ {\rm arithmetic}\ {\rm (a)} \\
 \frac 12\beta^{1-t}\ \ \ &
 {\rm for}\ {\rm rounded}\ {\rm arithmetic}\ {\rm (b)}
      \end{array}\right .
$$
The number $\bu$ is called {\em unit roundoff} or {\em machine
epsilon}.

\subsection{Machine epsilon} \label{MachineEpsilon}

Note that the machine epsilon $\bu$ is \emph{not} the smallest positive
number $m$ represented by the given machine system (cf.\
Sect.~\ref{BasicPropMachineSystems}).

One can describe $\bu$ as the smallest positive number $\varepsilon>0$
such that $(1+\varepsilon)_c \neq 1$. In other words, $\bu$ is the
smallest positive value that, when added to one, yields a result
different from one.

In more practical terms, $\bu$ tells us how many accurate digits
machine numbers can carry. If $\bu\sim 10^{-p}$, then any machine
number $x_c$ carries at most $p$ accurate decimal digits.

{\footnotesize For example, suppose for a given machine system $\bu\sim
10^{-7}$ and a machine number $x_c$ representing some real number $x$
has value 35.41879236 (when printed on paper or displayed on computer
screen). Then we can say that $x \approx 35.41879$, and the digits of
$x$ beyond 9 cannot be determined. In particular, the digits 236 in the
printed value of $x_c$ are meaningless (they are ``trash'' to be
discarded).}

\subsection{Machine epsilon for the two standard machine systems} \label{14.10}

\textcircled{\scriptsize I} For the IEEE floating-point \emph{single
precision} standard with chopped arithmetic ${\bf u}=2^{-23}\approx
1.2\times 10^{-7}$. In other words, approximately 7 decimal digits are
accurate. \medskip

\textcircled{\scriptsize II} For the IEEE floating-point \emph{double
precision} standard with chopped arithmetic ${\bf u}=2^{-52}\approx
2.2\times 10^{-16}$. In other words, approximately 16 decimal digits
are accurate.

\newpage

In the next two examples, we will solve systems of linear equations by
using the rules of a machine system. In other words, we will pretend
that we are \emph{computers}. This will help us understand how real
computers work.

\subsection{Example} \label{14.11}

{\footnotesize Let us solve the system of equations
$$
  \left[\begin{array}{cc} 0.01 & 2 \\ 1 & 3 \end{array}\right]
  \left[\begin{array}{l} x \\ y \end{array}\right]=
  \left[\begin{array}{l} 2 \\ 4 \end{array}\right]
$$
The exact solution was found in Section~\ref{GaussExample}:
$$
   x=\tfrac{200}{197}\approx 1.015
   \qquad\text{and}\qquad
   y=\tfrac{196}{197}\approx 0.995.
$$
Now let us solve this system by using chopped arithmetic with base
$\beta=10$ and $t=2$ (i.e., our mantissa will be always limited to
two decimal digits).

First we use Gaussian elimination (without pivoting). Multiplying the
first equation by 100 and subtracting it from the second gives
$-197y=-196$. Both numbers 197 and 196 are three digit long, so we must
chop the third digit off. This gives $-190y=-190$, hence $y=1$.
Substituting $y=1$ into the first equation gives $0.01x=2-2=0$, hence
$x=0$. Thus our computed solution is
$$
   x_c=0 \qquad\text{and}\qquad y_c=1.
$$
The relative error in $x$ is $(x_c-x)/x=-1$, i.e., the computed
$x_c$ is 100\% off mark!

Let us increase the length of mantissa to $t=3$ and repeat the
calculations. This gives
$$
   x_c=2 \qquad\text{and}\qquad y_c=0.994.
$$
The relative error in $x$ is now $(x_c-x)/x=0.97$, i.e., the
computed $x_c$ is 97\% off mark. Not much of improvement... We
postpone the explanation until Chapter~\ref{Ch13}.

Let us now apply partial pivoting (Section~\ref{PartialPivoting}).
First we interchange the rows:
$$
  \left[\begin{array}{cc} 1 & 3 \\ 0.01 & 2 \end{array}\right]
  \left[\begin{array}{l} x \\ y \end{array}\right]=
  \left[\begin{array}{l} 4 \\ 2 \end{array}\right].
$$
Now we multiply the first equation by 0.01 and subtract it from the
second to get $1.97y=1.96$. With $t=2$, we must chop off the third
digit in both numbers: $1.9y=1.9$, hence $y=1$. Substituting $y=1$
into the fits equation gives $x+3=4$, hence $x=1$, i.e.,
$$
   x_c=1 \qquad\text{and}\qquad y_c=1.
$$
This is a great improvement over the first two solutions (without
pivoting). The relative error in $x$ is now $(x_c-x)/x=-0.015$, so
the computed $x_c$ is only 1.5\% off. The relative error in $y$ is
even smaller (about 0.005).

\newpage

\begin{center}
{\footnotesize \framebox{
\begin{tabular}{ccccc}
 & & & & \\
  Machine arithmetic & & Exact arithmetic: & & Machine arithmetic \\
  with $t=2$: & & & & with $t=3$:\\
 & & $\begin{aligned} 0.01x+2y&=2\\ x+3y&=4 \end{aligned}$ & & \\
 & & $\Downarrow$ & & \\
 $\begin{aligned} 0.01x+2y&=2\\ -190y=&-190 \end{aligned}$
 & $\stackrel{\tiny \text{\tiny Chopping off}}{\xleftarrow{\hspace*{1.2cm}}}$ &
 $\begin{aligned} 0.01x+2y&=2\\ -197y&=-196 \end{aligned}$ &  & \\
 $\Downarrow$ & & $\Downarrow$ & & \\
 $\begin{aligned} 0.01x+2y&=2\\ y&=1 \end{aligned}$ & &
 $\begin{aligned} 0.01x&+2y=2\\ y&=\tfrac{196}{197}\approx 0.9949 \end{aligned}$
 & $\stackrel{\tiny \text{\tiny Chopping off}}{\xrightarrow{\hspace*{1.2cm}}}$ &
 $\begin{aligned} 0.01x+2y&=2\\ y&=0.994 \end{aligned}$ \\
 $\Downarrow$ & & $\Downarrow$ & & $\Downarrow$ \\
 $\begin{aligned} 0.01x&=2-2=0\\ y&=1 \end{aligned}$ &
 $\begin{aligned}\textcircled{\tiny !!}\\ \phantom{mm}\end{aligned}$ &
 $\begin{aligned} 0.01x&=2-\tfrac{392}{197}=\tfrac{2}{197}\\ y&=\tfrac{196}{197} \end{aligned}$ &
 $\begin{aligned}\textcircled{\tiny !!}\\ \phantom{mm}\end{aligned}$ &
 $\begin{aligned} 0.01x&=2-1.98\xcancel{8}=0.02\\ y&=0.994 \end{aligned}$ \\
 $\Downarrow$ & & $\Downarrow$ & & $\Downarrow$ \\
 $\begin{aligned} x&=0\\ y&=1 \end{aligned}$ & &
 $\begin{aligned} x&=\tfrac{200}{197}\approx 1.0152\\ y&=\tfrac{196}{197}\approx 0.9949 \end{aligned}$ & &
 $\begin{aligned} x&=2\\ y&=0.994 \end{aligned}$ \\
 & & & & \\
\end{tabular}}
}

Example \ref{14.11} without pivoting.\label{ExMachine}
\end{center}

$\phantom{m}$\vspace{5mm}

\begin{center}
{\footnotesize \framebox{
\begin{tabular}{ccccc}
 & & & & \\
  Machine arithmetic & & Exact arithmetic: & & Machine arithmetic \\
  with $t=2$: & & & & with $t=3$:\\
 & & $\begin{aligned} x+3y&=4\\ 0.01x+2y&=2 \end{aligned}$ & & \\
 & & $\Downarrow$ & & \\
 $\begin{aligned} x+3y&=4 \\ 1.9y&=1.9 \end{aligned}$
 & $\stackrel{\tiny \text{\tiny Chopping off}}{\xleftarrow{\hspace*{1.4cm}}}$ &
 $\begin{aligned} x+3y&=4\\ 1.97y&=1.96 \end{aligned}$ &  & \\
 $\Downarrow$ & & $\Downarrow$ & & \\
 $\begin{aligned} x+3y&=4\\ y&=1 \end{aligned}$ & &
 $\begin{aligned} &x+3y=4\\ &y=\tfrac{196}{197}\approx 0.9949 \end{aligned}$
 & $\stackrel{\tiny \text{\tiny Chopping off}}{\xrightarrow{\hspace*{1.4cm}}}$ &
 $\begin{aligned} x+3y&=4\\ y&=0.994 \end{aligned}$ \\
 $\Downarrow$ & & $\Downarrow$ & & $\Downarrow$ \\
 $\begin{aligned} x&=4-3=1\\ y&=1 \end{aligned}$ & &
 $\begin{aligned} x&=4-\tfrac{588}{197}=\tfrac{200}{197}\\ y&=\tfrac{196}{197} \end{aligned}$ & &
 $\begin{aligned} x&=4-2.98\xcancel{2}=1.02\\ y&=0.994 \end{aligned}$ \\
 & & & & \\
\end{tabular}}
}

Example \ref{14.11} with partial pivoting.
\end{center}

\newpage

Now let us continue the partial pivoting with $t=3$. We can keep
three digits, so $1.97y=1.96$ gives $y=1.96/1.97\approx 0.9949$,
which we have to reduce to $y=0.994$. Substituting this value of $y$
into the first equation gives $x+2.982=4$, which we have to reduce
to $x+2.98=4$, hence $x=1.02$. So now
$$
   x_c=1.02 \qquad\text{and}\qquad y_c=0.994.
$$
The relative error in $x$ is $(x_c-x)/x=0.0047$, less than 0.5\%.

The table below shows the relative error of the numerical solution
$x_c$ by Gaussian elimination with pivoting and different lengths of
mantissa. We see that the relative error is roughly proportional to
the ``typical'' round-off error $10^{-t}$, with a factor of about 2
to 5. We can hardly expect a better accuracy.

\begin{center}
\begin{tabular}{|c|c|c|c|}
\hline  & relative error & typical error & factor \\
\hline  $t=2$ & $1.5\times 10^{-2}$  & $10^{-2}$ & $1.5$ \\
\hline  $t=3$ & $4.7\times 10^{-3}$  & $10^{-3}$ & $4.7$ \\
\hline  $t=4$ & $2.2\times 10^{-4}$  & $10^{-4}$ & $2.2$ \\
\hline
\end{tabular}\vspace*{0.2cm}
\end{center}

\noindent Conclusions: Gaussian elimination without pivoting may
lead to catastrophic errors, which will remain unexplained until
Chapter~\ref{Ch13}. Pivoting is more reliable -- it seems to provide
nearly maximum possible accuracy here. But see the next example...}

\subsection{Example} \label{14.11a}

{\footnotesize Let us solve another system of equations:
$$
  \left[\begin{array}{cc} 3 & 1 \\ 1 & 0.35 \end{array}\right]
  \left[\begin{array}{l} x \\ y \end{array}\right]=
  \left[\begin{array}{c} 5 \\ 1.7 \end{array}\right]
$$
The exact solution here is
$$
   x=1
   \qquad\text{and}\qquad
   y=2.
$$
The largest coefficient, 3, is at the top left corner already, so
pivoting (partial or complete) would not change anything.

Solving this system in chopped arithmetic with $\beta=10$ and $t=2$
gives $x_c=0$ and $y_c=5$, which is 150\% off. Increasing the length
of the mantissa to $t=3$ gives $x_c=0.883$ and $y_c=2.35$, so the
relative error is 17\%. With $t=4$, we obtain $x_c=0.987$ and
$y_c=2.039$, now the relative error is 2\%. The table below shows
that the relative error of the numerical solutions is roughly
proportional to the typical round-off error $10^{-t}$, but with a
big factor fluctuating around 150 or 200.

\begin{center}
\begin{tabular}{|c|c|c|c|}
\hline  & relative error & typical error & factor \\
\hline  $t=2$ & $1.5\times 10^{-0}$  & $10^{-2}$ & $150$ \\
\hline  $t=3$ & $1.7\times 10^{-1}$  & $10^{-3}$ & $170$ \\
\hline  $t=4$ & $2.0\times 10^{-2}$  & $10^{-4}$ & $200$ \\
\hline
\end{tabular}\vspace*{0.2cm}
\end{center}

\noindent We postpone a complete analysis of the above two examples
until Chapter~\ref{Ch13}.}

\subsection{Computational errors} \label{14.12}

Let $x$ and $y$ be two real numbers represented in a machine system by
$x_c$ and $y_c$, respectively. An arithmetic operation $x\ast y$ (where
$\ast$ stands for one of the four basic operations: $+,-,\times,\div$)
is performed by a computer in the following way. The computer first
finds $x_c\ast y_c$ exactly and then represents that number in its
machine system. The result is $z=(x_c\ast y_c)_c$.

{\footnotesize Note that, generally, $z$ is different from $(x\ast
y)_c$, which is the machine representation of the exact result $x\ast
y$. Hence, $z$ is not necessarily the best representation for $x\ast
y$. In other words, the computer makes additional round off errors at
each arithmetic operation.}

Assuming that $x_c=x(1+\varepsilon_1)$ and $y_c=y(1+\varepsilon_2)$ we
have
$$
    (x_c\ast y_c)_c=(x_c\ast y_c)\,(1+\varepsilon_3)=
     [\,x(1+\varepsilon_1)\,]\ast [\,y(1+\varepsilon_2)\,]
     \,(1+\varepsilon_3)
$$
where $|\varepsilon_1|,|\varepsilon_2|,|\varepsilon_3|\leq\bf u$.

\subsection{Multiplication and division} \label{14.13}

For multiplication, we have
$$
   z=xy(1+\varepsilon_1)(1+\varepsilon_2)(1+\varepsilon_3)\approx
       xy(1+\varepsilon_1+\varepsilon_2+\varepsilon_3)
$$
(here we ignore higher order terms like $\varepsilon_1\varepsilon_2$),
so the relative error is (approximately) bounded by $3{\bf u}$. A
similar estimate can be made for division:
$$
   z=\frac{x(1+\varepsilon_1)(1+\varepsilon_3)}{y(1+\varepsilon_2)}\approx
       \frac xy\,(1+\varepsilon_1-\varepsilon_2+\varepsilon_3).
$$
{\footnotesize Note: we used Taylor expansion
$$
  \frac{1}{1+\varepsilon_2} = 1-\varepsilon_2+\varepsilon_2^2-\varepsilon_2^3+\cdots
$$
and again ignored higher order terms.}

Thus again the relative error is (approximately) bounded by $3{\bf
u}$.\medskip

Conclusion: machine multiplication and machine division magnify
relative errors by a factor of three, at most.

\newpage

\subsection{Addition and subtraction} \label{14.14}

For addition, we have
$$
    z=(x+y+x\varepsilon_1+y\varepsilon_2)(1+\varepsilon_3)=
    (x+y)\Bigl(1+\frac{x\varepsilon_1+y\varepsilon_2}{x+y}\Bigr)
    (1+\varepsilon_3).
$$
\vspace{-1mm}Again ignoring higher order terms we can bound the
relative error of $z$ by
$$
      \frac{|x|+|y|}{|x+y|}\,{\bf u} + {\bf u}.
$$
Thus, the operation of addition magnifies relative errors by a factor
of
$$
      \frac{|x|+|y|}{|x+y|}+1.
$$
Similar estimates can be made for subtraction $x-y$: it magnifies
relative errors by a factor
$$\vspace{-1mm}
      \frac{|x|+|y|}{|x-y|}+1.
$$
Hence the addition and subtraction magnify relative errors by a
variable factor which depends on $x$ and $y$. This factor may be
arbitrarily large if $x+y\approx 0$ for addition or $x-y\approx 0$ for
subtraction. This phenomenon is known as {\em catastrophic
cancelation}. {\footnotesize It occurred in our Example~\ref{14.11}
when we solved it without pivoting (see the line marked with double
exclamation signs on page~\pageref{ExMachine})}.

\begin{exercise}
(JPE, September 1993). Solve the system
$$
  \left (\begin{array}{ll} 0.001 & 1.00 \\ 1.00 & 2.00 \end{array}\right )
  \left (\begin{array}{l} x \\ y \end{array}\right )=
  \left (\begin{array}{l} 1.00 \\ 3.00 \end{array}\right )
$$
using the LU decomposition with and without partial pivoting and
chopped arithmetic with base $\beta=10$ and $t=3$ (i.e., work with a
three digit mantissa). Obtain computed solutions $(x_c,y_c)$ in both
cases. Find the exact solution, compare, make comments.
\end{exercise}

\begin{exercise}
\vspace{-1mm} (JPE, May 2003). Consider the system
$$
  \left (\begin{array}{ll} \varepsilon & 1 \\ 2 & 1 \end{array}\right )
  \left (\begin{array}{l} x \\ y \end{array}\right )=
  \left (\begin{array}{l} 1 \\ 0 \end{array}\right )
$$
Assume that $|\varepsilon|\ll 1$. Solve the system by using the LU
decomposition with and without partial pivoting and adopting the
following rounding off models (at all stages of the computation!):
\begin{align*}
 a+b\varepsilon &= a \qquad\text{(for } a\neq 0\text{)},\\
 a+b/\varepsilon &= b/\varepsilon \qquad\text{(for } b\neq
 0\text{)}.
\end{align*}
Find the exact solution, compare, make comments.
\end{exercise}

\newpage

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\refstepcounter{section} \label{Ch12}

\centerline{Chapter \ref{Ch12} }\vspace{5mm}

\centerline{\Large\bf Condition Numbers}\vspace{3mm}

\addcontentsline{toc}{section}{\ref{Ch12}. Condition Numbers}

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\setcounter{subsection}{0} \setcounter{exercise}{0}
\setcounter{equation}{0}

{\footnotesize
\subsection{Introduction} \label{14.00}

We have seen in Chapter~\ref{Ch11} that not all real numbers can be
precisely encoded in a given machine system. In most instances there is
a difference between the exact value of a real number $x$ and its
computer representation $x_c$ (this difference is called \emph{error}
of representation). We have also seen that numerical calculations with
finite precision (limited length of mantissa) tend to magnify errors,
sometimes dramatically. In this chapter our object of study will be the
factor by which errors are magnified (that factor will be called
\emph{condition number}).

Also we have seen that in numerical analysis it is customary to deal
with \emph{relative errors}, given by $|x_c-x|/x$, rather than absolute
errors, $|x_c-x|$. A relative error $\sim 10^{-k}$ means that the value
of $x_c$, when recorded in decimal format, has $k$ accurate digits
(recall Sect.~\ref{MachineEpsilon}). We note that it is the order of
magnitude of a relative error that matters, not its exact value. For
example, relative errors $0.0001$ and $0.0002$ are nearly the same, in
practical terms: both are of order $10^{-4}$, hence both tell us that
we can trust approximately four decimal digits in the recorded value of
$x_c$. }

\subsection{Computational process as a function} \label{14.0}

Any computational process transforms given numbers (\emph{input}) into
computed numbers (\emph{output}).\medskip

{\footnotesize In Example~\ref{14.11} we were given the coefficients of
a matrix (0.01, 2, 1, and 3) and the components of a right hand side
vector (2 and 4). The results of our computation were two numbers:
$x=\tfrac{200}{197}$ and $y=\tfrac{196}{197}$.

\begin{center}
\begin{tabular}{ccccc}
  $\begin{aligned}\phantom{mmm}&\text{Input}\\0.01,\ &2,\ 1,\ 3,\ 2,\ 4\end{aligned}$ &
  ${\xrightarrow{\hspace*{1cm}}}$ &
  \framebox{$\begin{aligned}&\text{computational}\\
  &\phantom{mn}\text{process}\end{aligned}$} &
  ${\xrightarrow{\hspace*{1cm}}}$ &
  $\qquad\begin{aligned}\ &\text{Output}\\&\tfrac{200}{197},\ \tfrac{196}{197}\end{aligned}$ \\
\end{tabular}
\end{center}

\noindent Given numbers can be viewed as components of a vector, and
computed numbers -- as components of another vector. Thus every
computational process is a transformation of a vector space into
another vector space. In our Example~\ref{14.11}, it was $\IR^6 \to
\IR^2$.}\medskip

In other words, every computational process can be viewed as a function
$f\colon V\to W$, where $V$ and $W$ are vector spaces.

\subsection{Condition number of a function} \label{14.15}

Let $V$ and $W$ be two \emph{normed} vector spaces, and $f\colon V\to
W$ a function. The {\em condition number} $\kappa$ of $f$ at a point
$x\in V$ is defined by
$$
   \kappa = \kappa(f,x) = \lim_{\delta\to 0}
   \sup_{\|\Delta x\|\leq\delta}
   \left ( \frac{\|\Delta f\|}{\|f\|}
   \bigg/ \frac{\|\Delta x\|}{\|x\|} \right )
$$
where $\Delta f = f(x+\Delta x) - f(x)$. This is the maximal factor by
which small relative errors are magnified by $f$ in the vicinity of the
point $x$.

{\footnotesize The condition number characterizes the {\em sensitivity}
of the output, $f(x)$, to small perturbations in the input, $x$.}

\subsection{Lemma} \label{14.15a}

Let $\IC^n\to\IC^n$ be a linear operator defined by a nonsingular
matrix $A\in\IC^{n\times n}$, and let $\|\cdot\|$ be a norm on $\IC^n$.
Then for every $x\in\IC^n$
$$
   \kappa(A,x)\leq \|A\|\,\|A^{-1}\|
   \qquad{\rm and}\qquad
   \sup_{x\neq 0}\kappa(A,x) = \|A\|\,\|A^{-1}\|
$$
where $\|A\|$ denotes the induced matrix norm.

{\footnotesize \proof Let $y=Ax$ and note that $\Delta y=A(x+\Delta
x)-Ax=A(\Delta x)$. Now
$$
  \kappa(A,x)=\sup_{\Delta x\neq 0}
  \frac{\|A(\Delta x)\|}{\|\Delta x\|}
  \frac{\|x\|}{\|Ax\|}
  =\|A\|\frac{\|x\|}{\|Ax\|}
$$
which is defined for all $x\neq 0$. Next
$$
  \sup_{x\neq 0} \kappa(A,x) =
  \|A\|\,\sup_{x\neq 0} \frac{\|x\|}{\|Ax\|}=
  \|A\|\, \sup_{y\neq 0} \frac{\|A^{-1}y\|}{\|y\|} = \|A\|\,\|A^{-1}\|
$$
(note that $x\neq 0\ \Longleftrightarrow\ y\neq 0$, because $A$ is
invertible). \qed}


\subsection{Condition number of a matrix} \label{15.2}

For a nonsingular matrix $A\in\IC^{n\times n}$, the condition number
with respect to a given matrix norm $\|\cdot \|$ is defined by
\beq \label{kappadef}
    \kappa(A)=\|A\|\,\|A^{-1}\|
\eeq
We denote by $\kappa_1(A)$, $\kappa_2(A)$, $\kappa_{\infty}(A)$ the
condition numbers with respect to the 1-norm, 2-norm, and
$\infty$-norm, respectively.

\subsection{Main theorem for the condition number of a matrix} \label{15.3}

Let $A\in\IC^{n\times n}$ be a nonsingular matrix and
\begin{align*}
     Ax &= b \tag{1} \\
    (A+\Delta A)(x+\Delta x)&=b+\Delta b \tag{2}
\end{align*}
Assume that $\|\Delta A\|$ is small so that $\|\Delta A\|\,
\|A^{-1}\|<1$. Then
$$
   \frac{\|\Delta x\|}{\|x\|}\leq
   \frac{\kappa(A)}{1-\kappa(A)\frac{\|\Delta A\|}{\|A\|}}
   \biggl(\frac{\|\Delta A\|}{\|A\|}+\frac{\|\Delta b\|}{\|b\|}\biggr).
$$

{\footnotesize \proof Expanding out the second equation (2),
subtracting from it the first equation (1), and premultiplying by
$A^{-1}$ gives
$$
   \Delta x=-A^{-1}\Delta A(x+\Delta x)+A^{-1}\Delta b.
$$
Taking norms and using Section~\ref{InducedInequalities} and the
triangle inequality give
$$
   \|\Delta x\|\leq \|A^{-1}\|\, \|\Delta A\|\Big (\|x\|+\|\Delta x\|\Big )
         +\|A^{-1}\|\, \|\Delta b\|.
$$
Note that $\|b\|\leq \|A\|\, \|x\|$, hence the above inequality
rearranges to
$$
   \Big (1-\|A^{-1}\|\, \|\Delta A\|\Big )\|\Delta x\|\leq
    \left (\|A^{-1}\|\, \|\Delta A\|+\|A^{-1}\|\, \|A\|
      \frac{\|\Delta b\|}{\|b\|}\right )\|x\|.
$$
Recall that $\|\Delta A\|\, \|A^{-1}\|<1$, so the first factor above is
positive. The theorem now follows immediately. \qed}

\new \emph{Note}: The smaller the condition number $\kappa(A)$, the
smaller upper bound on the relative error $\frac{\|\Delta x\|}{\|x\|}$
we get. The value of $\kappa(A)$ thus characterizes the {\em
sensitivity} of the solution of the linear system $Ax=b$ to small
perturbations of both $A$ and $b$.

\new {\em Interpretation}. Let $Ax=b$ be a system of
linear equations to be solved numerically. A computer represents $A$ by
$A_c=A+\Delta A$ and $b$ by $b_c=b+\Delta b$. Assume that the computer
finds the exact solution $x_c$ of the perturbed system, i.e., $x_c$
satisfies $A_cx_c=b_c$. Denote by $\Delta x= x_c-x$ the resulting
error, where $x$ denotes the exact solution of the true system $Ax=b$.
Then the relative error $\|\Delta x\|/\|x\|$ can be estimated by
Theorem~\ref{15.3}.

\subsection{Corollary}

Consider the problem of solving a system of linear equations $Ax=b$
with a nonsingular matrix $A$. Then:
\begin{itemize} \item[(a)] If we fix $A$ and vary $b$, we get a map
$f_A\colon b\mapsto x$. The condition number of $f_A$ satisfies
$$
   \kappa(f_A,b)\leq \kappa(A)
   \quad{\rm and}\quad
   \sup_{b\in\IC^n}\kappa(A,b) = \kappa(A).
$$
\item[(b)] If we fix $b$ and vary $A$, we get a map $f_b\colon
    A\mapsto x$. The condition number of $f_b$ satisfies
$$
   \kappa(f_b,A) \leq \kappa(A).
$$
\end{itemize}

{\footnotesize \proof These are particular cases of Theorem~\ref{15.3}:
in part (a) we set $\Delta A=0$, and in pat (b) we set $\Delta b=0$.
Then both inequalities easily follow from Theorem~\ref{15.3}. To prove
the equality in (a), note that $x=A^{-1}b$ and apply
Lemma~\ref{14.15a}. Incidentally, we note that $\kappa(A^{-1}) =
\kappa(A)$ for any nonsingular matrix $A$. \qed}

\new {\em Remark}. In part (b), we actually have equality
$\kappa(f_b,A)=\kappa(A)$, but the proof is beyond the scope of our
course {\footnotesize (it can be found in the textbook, page~95)}.

\subsection{Corollary} \label{15.4}

Assume that in Theorem~\ref{15.3} we have $\|\Delta A\|\leq {\bf
u}\|A\|$ and $\|\Delta b\|\leq {\bf u}\|b\|$, i.e., the matrix $A$ and
the vector $b$ are represented with the best possible machine accuracy.
Then
\beq \label{kappaAu}
   \frac{\|\Delta x\|}{\|x\|}\leq
   \frac{2{\bf u}\kappa(A)}{1-{\bf u}\kappa(A)}.
\eeq
provided ${\bf u}\kappa(A) \ll 1$. \medskip

{\footnotesize One should note that the formula \eqref{kappaAu} is
practically meaningful only if ${\bf u}\kappa(A) \ll 1$, otherwise
$\|\Delta x\|$ would be comparable to $\|x\|$ so the computed solution
could not be trusted at all (in particular, estimating its relative
error is rather pointless).

On the other hand, if ${\bf u}\kappa(A) \ll 1$, the formula
\eqref{kappaAu} can be written simply (though a bit inaccurately) as
\beq \label{kappaAu2}
  \frac{\|\Delta x\|}{\|x\|}\lesssim 2{\bf u}\kappa(A).
\eeq.}

\newpage

\subsection{Practical interpretation} \label{15.5}

Assume that ${\bf u}\approx 10^{-p}$, i.e., the machine system provides
$p$ accurate digits. Now if $\kappa(A)\approx 10^q$ with $q<p$, then
$\|\Delta x\|/\|x\|\leq 10^{-(p-q)}$, i.e., the numerical solution
provides $p-q$ accurate digits.\medskip

{\footnotesize We note again, as in Sect.~\ref{14.00}, that only the
order of magnitude of $\kappa(A)$ matters, not its exact value. For
instance, there is little difference between $\kappa(A)=100$ and
$\kappa(A)=200$, both are of order $10^2$.}\medskip

Linear systems $Ax=b$ with small $\kappa(A)$ ($\sim 1$, $10$, $10^2$)
are often called {\em well-conditioned}. Those with large $\kappa(A)$
($\sim 10^3$, $10^4$, etc.) are called {\em ill-conditioned}. Their
numerical solutions are unreliable and should be avoided.

\subsection{Maxmag and minmag}

The \emph{maximum} and \emph{minimum magnification} by a matrix $A$ are
defined by
$$
   {\rm maxmag}(A) = \max_{x\neq 0}\frac{\|Ax\|}{\|x\|}
   =\max_{\|x\|=1} \|Ax\|
$$
and
$$
   {\rm minmag}(A) = \min_{x\neq 0}\frac{\|Ax\|}{\|x\|}
   =\min_{\|x\|=1} \|Ax\|.
$$

Note that
$$
   {\rm maxmag}(A) = \|A\|.
$$

For singular matrices, ${\rm minmag}(A)=0$. For nonsingular matrices
$$
   {\rm minmag}(A) = \min_{y\neq 0}\frac{\|y\|}{\|A^{-1}y\|}
   =\frac{1}{\max_{y\neq 0} \frac{\|A^{-1}y\|}{\|y\|}} = \frac{1}{\|A^{-1}\|}.
$$
Therefore
\beq \label{kappamaxmin}
   \kappa(A) = \frac{{\rm maxmag}(A)}{{\rm minmag}(A)}
   =\frac{\sup_{\|x\|=1}\|Ax\|}{\inf_{\|x\|=1}\|Ax\|}
\eeq

The formula \eqref{kappamaxmin} makes it reasonable to say that for
singular matrices $\kappa(A)=\infty$. That is, the singularity is
the extreme case of ill-conditioning. Reversing the point of view,
we can say that ill-conditioned matrices are ``nearly singular''.
This relation is made precise in Section~\ref{15.5a} below.

\newpage

{\footnotesize For the 2-norm, the above relations can be described
geometrically with the help of unit sphere defined by
\eqref{UnitSphere}. The linear transformation $A\colon \IC^n \to
\IC^n$ maps the unit sphere ${\mathbb S}_1$ onto an ellipsoid. The
longest semiaxis of the latter is ${\rm maxmag}(A)$ and its shortest
semiaxis is ${\rm minmag}(A)$.

\begin{center}
\begin{tikzpicture}
         \draw[thin] (-1.3,0)--(1.3,0);
         \draw[thin] (0,-1.2)--(0,1.3);
         \draw[thin] (0,0) circle(1);
         \node[anchor=south west] at (0.6,0.6) {\footnotesize $\mathbb S_1$};
         \draw[thick,->,-latex] (0,0)--(0.97,0.17);
         \draw[thick,->,-latex] (0,0)--(-0.17,0.97);
         \draw[line width=0.8mm,gray,->,-latex] (2,0)--(4,0);
         \node[anchor=south] at (3,-0.05) {$A$};
         \begin{scope}[xshift=6cm]
         \draw[thin] (-1.5,0)--(1.5,0);
         \draw[thin] (0,-1.2)--(0,1.3);
         \draw[thin,rotate=-20] (0,0) ellipse(1.4 and 0.7);
         \draw[thick,->,-latex,rotate=-20] (0,0)--(1.4,0);
         \node[anchor=west] at (1.2,-0.5) {\footnotesize maxmag$(A)$};
         \draw[thick,->,-latex,rotate=-20] (0,0)--(0,0.7);
         \node[anchor=south] at (1,0.6) {\footnotesize minmag$(A)$};
         \end{scope}
\end{tikzpicture}
\end{center}
Thus, $\kappa_2(A)$ shows how much the linear map $A\colon \IC^n \to
\IC^n$ ``distorts'' the unit sphere ${\mathbb S}_1$.}


\subsection{Properties of the condition numbers} \label{15.6}

\begin{enumerate}
\item If $a_j$ denotes the $j$-th column of $A$, then
    $\kappa(A)\geq \|a_j\|/\|a_i\|$
\item $\kappa(A)\geq 1$ and $\kappa(I)=1$
\item $\kappa_2(A)=1$ if and only if $A$ is a multiple of a unitary
    matrix.
\item For any unitary matrix $Q$,
   $$
        \kappa_2(QA)=\kappa_2(AQ)=\kappa_2(A)
   $$
\item If $D=\, {\rm diag}\{d_1,\dots,d_n\}$ then
   $$
       \kappa_2(D)=\kappa_1(D)=\kappa_{\infty}(D)=
       \frac{\max_{1\leq i\leq n}|d_i|}{\min_{1\leq i\leq n}|d_i|}
   $$
\item If $A$ is Hermitian with eigenvalues
    $\lambda_1,\ldots,\lambda_n$, then
   $$
      \kappa_2(A)=\frac{\max_i|\lambda_i|}{\min_i|\lambda_i|}
   $$
\item If $\sigma_1\geq\sigma_2\geq\cdots\geq\sigma_n$ denote the
    singular values of $A$, then
   $$
       \kappa_2(A) = \sigma_1/\sigma_n
   $$
\item We have
   $$
       [\kappa_2(A)]^2=\kappa_2(A^\ast A)=\kappa_2(AA^\ast)=
       \frac{\lambda_{\max}(A^\ast A)}{\lambda_{\min}(A^\ast A)}
   $$
\item We have $\kappa_2(A)=\kappa_2(A^\ast)$.
\end{enumerate}


\subsection{Closeness to a singular matrix}\label{15.5a}

We have
$$
   \min\biggl\{\frac{\|A-A_s\|_2}{\|A\|_2}\colon\
   A_s\ \text{is singular}\biggr\}=
    \frac{1}{\kappa_2(A)}.
$$
In other words, $1/\kappa_2(A)$ is the relative distance from $A$ to
the nearest singular matrix.\medskip

{\footnotesize Proof: This follows from Section~\ref{17.20} and
Section~\ref{15.6} (part 7). \qed }

\subsection{A posteriori error analysis using the residual} \label{15.8}

In practice the exact solution $x$ of the system $Ax=b$ is rarely
known. If it is unknown, how can we determine the relative error
$\frac{\|x_c-x\|}{\|x\|}$ of a numerical solution $x_c$?

One way to do that is to compute the {\em residual vector} $r=Ax_c-b$
and see if $r=0$. Of course, $r=0$ would imply that $x_c$ is the exact
solution, and we can hardly expect that. Now our intuition tells us
that if $r$ is small, the numerical solution $x_c$ must be good. But
how small should $r$ be?

Note that $Ax_c=b+r$, so $x_c$ is the \emph{exact} solution of a
perturbed system
$$
   A(\underbrace{x+\Delta x}_{x_c})=b+\underbrace{\Delta b}_{r}
$$
with $\Delta b=r$. Now Theorem~\ref{15.3} with $\Delta A=0$ implies
that
$$
   \frac{\|x_c-x\|}{\|x\|}\leq \kappa(A)\frac{\|r\|}{\|b\|}
$$
In practical terms: if $A$ is well conditioned, then the smallness of
$\frac{\|r\|}{\|b\|}$ ensures the smallness of the relative error
$\frac{\|x_c-x\|}{\|x\|}$. However, if $A$ is ill-conditioned, such a
conclusion cannot be made: the smallness of $r$ does not guarantee a
good accuracy of $x_c$.

{\footnotesize
\subsection{Extension to rectangular matrices} \label{ExtKappaRect}

For rectangular matrices $A\in \IC^{m\times n}$ ($m\neq n$) the inverse
$A^{-1}$ is not defined, hence the formula \eqref{kappadef} does not
apply. Instead, the formula \eqref{kappamaxmin} can be used as a
\emph{definition} of the condition number:
\beq \label{kapparectangular}
   \kappa(A) \stackrel{\rm def}{=} \frac{{\rm maxmag}(A)}{{\rm minmag}(A)}
   =\frac{\sup_{\|x\|=1}\|Ax\|}{\inf_{\|x\|=1}\|Ax\|}.
\eeq
Note that if $m<n$, then $A$ must have a non-trivial kernel (i.e.,
$Ax=0$ for some $x\neq 0$); thus the denominator in
\eqref{kapparectangular} turns zero. So the definition
\eqref{kapparectangular} only applies if $m>n$. Then one can
\emph{prove} (see Exercise~\ref{ExKappaRect}) that
$$
   [\kappa_2(A)]^2 = \kappa_2(A^\ast A) =
   \frac{\lambda_{\max}(A^\ast A)}{\lambda_{\min}(A^\ast A)}.
$$}

\begin{exercise}
(JPE, September 1997). Show that, given a matrix $A\in\IR^{n\times n}$,
one can choose vectors $b$ and $\Delta b$ so that if
$$
       Ax=b
$$
$$
      A(x+\Delta x)=b+\Delta b
$$
then
$$
    \frac{||\Delta x||_2}{||x||_2}=\kappa_2(A)
    \frac{||\Delta b||_2}{||b||_2}
$$
Explain the significance of this result for the `optimal' role of
condition numbers in the sensitivity analysis
of linear systems. \\
(Hint: use SVD to show that it is enough to consider the case where $A$
is a diagonal matrix.)
\end{exercise}

\begin{exercise}
(JPE, combined May 1997 and May 2008)\begin{itemize} \item[(a)] Compute
the condition numbers $\kappa_1$, $\kappa_2$ and $\kappa_{\infty}$ for
the matrix
$$
  A=\left ( \begin{array}{cc} 1 & 2 \\ 1.01 & 2 \end{array}\right )
$$
\item[(b)] Show that for every non-singular $2\times 2$ matrix $A$ we
    have $\kappa_1(A) = \kappa_{\infty}(A)$.
    \end{itemize}
\end{exercise}

\begin{exercise}
(JPE, September 2002). Consider a linear system $Ax=b$. Let $x^\ast$ be
the exact solution, and let $x_c$ be some computed approximate
solution. Let $e=x^\ast-x_c$ be the error and $r=b-Ax_c$ the residual
for $x_c$. Show that
$$
    \frac{1}{\kappa(A)}\,\frac{\|r\|}{\|b\|}
    \leq\frac{\|e\|}{\|x^\ast\|}
    \leq\kappa(A)\,\frac{\|r\|}{\|b\|}
$$
Interpret the above inequality for $\kappa(A)$ close to $1$ and for
$\kappa(A)$ large.
\end{exercise}

\begin{exercise}
Prove properties 6 and 7 of condition numbers listed in
Section~\ref{15.6}.
\end{exercise}

\begin{exercise} \label{ExKappaRect}
Suppose the condition number of a rectangular matrix $A \in
\IC^{m\times n}$ with $m>n$ is defined by \eqref{kapparectangular}.
Prove that
$$
   [\kappa_2(A)]^2 = \kappa_2(A^\ast A) =
   \frac{\lambda_{\max}(A^\ast A)}{\lambda_{\min}(A^\ast A)}
$$
\end{exercise}


\newpage

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\refstepcounter{section} \label{Ch13}

\centerline{Chapter \ref{Ch13} }\vspace{5mm}

\centerline{\Large\bf Numerical Stability}\vspace{3mm}

\addcontentsline{toc}{section}{\ref{Ch13}. Numerical Stability}

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\setcounter{subsection}{0} \setcounter{exercise}{0}
\setcounter{equation}{0}

\subsection{Introduction} \label{15.9}

When a system of linear equations $Ax=b$ is solved numerically, the
computer only knows machine representations $A_c$ and $b_c$ of $A$ and
$b$. Then, at best, it can find $x_c=A_c^{-1}b_c$, instead of
$x=A^{-1}b$. According to \ref{kappaAu2}, the resulting relative error
will be
\beq \label{E1}
     E_1= \frac{\|x_c-x\|}{\|x\|}\lesssim 2{\bf u}\kappa(A)
\eeq
This may be bad enough already when the matrix $A$ is ill-conditioned.
However, in reality things appear to be even worse, since the computer
does \emph{not} evaluate $x_c=A_c^{-1}b_c$ precisely (apart from
trivial cases). The computer executes a certain sequence of arithmetic
operations (a program) designed to solve the system $Ax=b$. As the
program runs, more and more round-off errors are made at each step, and
the errors compound toward the end.

As a result, the computer returns a vector $\hat{x}_c$ \emph{different
from} $x_c=A_c^{-1}b_c$, i.e., the actual output $\hat{x}_c$ is
\emph{not} the solution of the perturbed system $A_cx_c=b_c$. Since
errors are made in the process of computation, we expect to arrive at
$\hat{x}_c \neq x_c$, therefore $A_c\hat{x}_c\neq b_c$. The actual
output $\hat{x}_c$ depends on how many computational errors are made
and how big they are. Those errors depend not only on the machine
system but even more on the algorithm that is used to solve the system
$Ax=b$. See the diagram below.

Of course, we do not expect the final relative error
\beq \label{E2}
      E_2= \frac{\|\hat{x}_c-x\|}{\|x\|}
\eeq
be smaller than $E_1$, but we hope that it will not be much larger
either. In other words, a good algorithm should not magnify the errors
caused already by conditioning. If this is the case, the algorithm is
said to be {\em stable}.

\subsection{Stable algorithms (definition)}

An algorithm for solving a system of linear equations $Ax=b$ is said to
be {\em stable} (or \emph{numerically stable}) if
\beq \label{Estable}
      \frac{\|\hat{x}_c-x\|}{\|x\|} \leq C{\bf u}\kappa(A)
\eeq
where $C>0$ is a constant. More precisely, $C$ must be independent of
$A,b$ and the machine system, but it may depend on the size of the
matrix, $n$.

We see that for stable algorithms, the actual relative error $E_2$ is
of the same order of magnitude as the `idealistic' relative error
$E_1$, thus a stable algorithm does not magnify the errors caused
already by conditioning.

\begin{center}
\begin{tikzpicture}
         \node[anchor=north] at (-5,0) {\tiny Idealistic};
         \node[anchor=north] at (-5,-0.2) {\tiny solution $x_c$};
         \node[anchor=north] at (-5,-0.4) {\tiny in the smaller};
         \node[anchor=north] at (-5,-0.6) {\tiny{\color{blue} (blue)} region};
         \node[anchor=north] at (-5,-1.5) {\tiny Computed};
         \node[anchor=north] at (-5,-1.7) {\tiny solution $\hat{x}_c$};
         \node[anchor=north] at (-5,-1.9) {\tiny in the larger};
         \node[anchor=north] at (-5,-2.1) {\tiny{\color{green} (green)} region};
         \fill[green!50] (0,0) circle(0.2);
         \fill[blue] (0,0) circle(0.1);
         \node[anchor=north] at (0,-0.3) {\tiny Well conditioned problem};
         \node[anchor=north] at (0,-0.5) {\tiny Stable algorithm};
         \begin{scope}[xshift=6cm]
         \fill[green!50] (0,0) ellipse(0.9 and 0.2);
         \fill[blue] (0,0) ellipse(0.7 and 0.1);
         \node[anchor=north] at (0,-0.3) {\tiny Ill conditioned problem};
         \node[anchor=north] at (0,-0.5) {\tiny Stable algorithm};
         \end{scope}
         \begin{scope}[yshift=-2cm]
         \fill[green!50] (0,0) ellipse(0.9 and 0.2);
         \fill[blue] (0,0) ellipse(0.1 and 0.1);
         \node[anchor=north] at (0,-0.3) {\tiny Well conditioned problem};
         \node[anchor=north] at (0,-0.5) {\tiny Unstable algorithm};
         \end{scope}
         \begin{scope}[yshift=-2cm,xshift=6cm]
         \fill[green!50] (0,0) ellipse(1.6 and 0.4);
         \fill[blue] (0,0) ellipse(0.7 and 0.1);
         \node[anchor=north] at (0,-0.3) {\tiny Ill conditioned problem};
         \node[anchor=north] at (0,-0.5) {\tiny Unstable algorithm};
         \end{scope}
\end{tikzpicture}
\end{center}

\subsection{Backward error analysis} \label{15.10}

We have seen in Section~\ref{15.8} that the computed solution $x_c$ to
a system $Ax=b$ can be represented as the \emph{exact} solution of a
perturbed system $A(x+\Delta x)=b+\Delta b$. This is the idea of
backward error analysis: for each computed solution $\hat{x}_c$ find a
perturbed matrix, $\hat{A}=A+\Delta A$, such that
$$
   (A+\Delta A)\hat{x}_c=b_c.
$$
After that Theorem~\ref{15.3} can be used to estimate the final
relative error $E_2= \frac{\|\hat{x}_c-x\|}{\|x\|}$ in terms of the
relative error $\frac{\|\Delta A\|}{\|A\|}$ of the perturbation of
$A$.\medskip

We call $A+\Delta A$ {\em virtual} matrix, since it is neither given
nor computed numerically, its existence is the result of a logical
analysis. Moreover, it is far from being unique. One wants to find a
virtual matrix $A+\Delta A$ as close to $A$ as possible, i.e., to make
$\|\Delta A\|$ as small as possible.

\setlength{\unitlength}{0.02in}

\begin{figure}
\begin{picture}(300,100)(0,0)

\put(80,80){\vector(1,0){120}} \put(80,50){\vector(1,0){120}}
\put(80,50){\vector(4,-1){120}} \put(80,20){\line(1,0){10}}
\put(100,20){\line(1,0){10}} \put(120,20){\line(1,0){10}}
\put(140,20){\line(1,0){10}} \put(160,20){\line(1,0){10}}
\put(180,20){\vector(1,0){10}}

\put(77,80){\circle*{2}} \put(77,50){\circle*{2}}
\put(77,20){\circle{2}} \put(203,80){\circle*{2}}
\put(203,50){\circle*{2}} \put(203,20){\circle*{2}}

\put(45,83){exact $A,b$} \put(37,53){machine $A_c,b_c$}
\put(37,23){`virtual' $\hat{A}_c,b_c$} \put(199,83){exact $x$}
\put(199,53){`idealistic' $x_c$} \put(199,23){computed $\hat{x}_c$}
\put(115,83){exact solution} \put(115,53){exact solution}
\put(115,13){exact solution} \put(136,36){computer}
\put(164,30){algorithm}

\end{picture}
%\caption{The scheme of round-off error analysis.}
\end{figure}

\newpage

\subsection{Backward stable algorithms (definition)}

An algorithm for solving a system of linear equations $Ax=b$ is said to
be {\em backward stable} if there exists a virtual matrix $A+\Delta A$
such that
$$
      \frac{\|\Delta A\|}{\|A\|} \leq C{\bf u}
$$
where $C>0$ is a constant. More precisely, $C$ must be independent of
$A,b$ and the machine system, but it may depend on the size of the
matrix, $n$.

\subsection{Theorem}

Every backward stable algorithm is stable.

\proof By Theorem~\ref{15.3},
$$
   \frac{\|\hat{x}_c-x\|}{\|x\|}\leq
   \frac{\kappa(A)}{1-\kappa(A)\frac{\|\Delta A\|}{\|A\|}}
   \frac{\|\Delta A\|}{\|A\|} \lesssim C{\bf u}\kappa(A).
   \qquad\square
$$

The proofs of stability (or instability) of algorithms in linear
algebra are quite involved. We only present relevant facts here,
without proofs.

\subsection{Theorem (without proof)}
\label{15.11}

If one uses LU decomposition $A=LU$ for solving a system $Ax=b$, then
there is a virtual matrix $A+\Delta A$ such that
$$
  \|\Delta A\| \leq C \|L\|\, \|U\| {\bf u},
$$
where $C>0$ is a constant independent of $A$ and the machine system
(but it may depend on the size of the matrix, $n$). Thus, the LU
algorithm is unstable, its accuracy deteriorates when $\|L\|\,\|U\|$ is
large.

\newpage

Theorem~\ref{15.11} may \emph{explain} why numerical solutions of
particular systems of equations happen to be inaccurate.

\subsection{Example} \label{15.11a}

In Example \ref{14.11}, the LU decomposition (without pivoting) is
$$
  \left[\begin{array}{cc} 0.01 & 2 \\ 1 & 3 \end{array}\right]
 =\left[\begin{array}{cc} 1 & 0 \\ 100 & 1 \end{array}\right]\,
  \left[\begin{array}{cc} 0.01 & 2 \\ 0 & -197 \end{array}\right]
$$
hence
$$
  \|L\|_\infty\,\|U\|_\infty = 101\cdot 197 \sim 10^4.
$$
This explains the huge errors of the corresponding numerical
solutions that we observed.

\subsection{Further facts (without proofs)}

\begin{itemize}\item[(a)]
Applying partial pivoting ensures that the entries of $L$ are
uniformly bounded: $|L_{ij}|\leq 1$. Also, it is observed in
practice that in \emph{most cases} $\|U\|\leq C\|A\|$, hence the
partial pivoting algorithm is \emph{usually} stable. There are,
however, some unusual cases where partial pivoting becomes
unstable. Therefore its accuracy cannot be guaranteed. \item[(b)]
The LU decomposition with complete pivoting is always stable, in
this case one can \emph{prove} that $\|U\|\leq C\|A\|$.
\item[(c)] The Cholesky factorization $A=GG^{\ast}$ of a positive
    definite matrix $A$ is a particular form of the LU
    decomposition, so the above analysis applies. In this case, we
    know that
$$
       a_{ii}=\sum_{j=1}^i g_{ij}^2
$$
see Section~\ref{13.6}. Thus, one can easily prove that $\|G\|\leq
C\|A\|^{1/2}$, hence the Cholesky factorization is always
numerically stable.
\end{itemize}

\noindent These facts may \emph{explain} the accuracy of numerical
solutions of particular systems of equations, as illustrated by the
following examples.

\newpage

\subsection{Example} \label{15.11b}

In Example \ref{14.11}, the matrix
$$
  \begin{bmatrix}0.01 & 2 \\ 1 & 3\end{bmatrix}
$$
has singular values $\sigma_1=3.7037$ and $\sigma_2=0.5319$, hence its
condition number is $\kappa_2(A)= \sigma_1/\sigma_2 = 6.96$. This
explains a moderate factor ($\leq 5$) by which relative errors of the
numerical solutions are related to the minimal error $10^{-t}$ when we
used a stable algorithm with pivoting in Example~\ref{14.11}.

\subsection{Example} \label{15.11c}

In Example \ref{14.11a}, the matrix
$$
  \begin{bmatrix}3 & 1 \\ 1 & 0.35\end{bmatrix}
$$
has singular values $\sigma_1=3.33$ and $\sigma_2=0.0150$, hence its
condition number is $\kappa_2(A)= \sigma_1/\sigma_2 =222$. This
explains a large factor (up to 200) by which relative errors of the
numerical solutions are related to the minimal error $10^{-t}$ in
Example~\ref{14.11a}, even though we used a stable algorithm (the LU
decomposition with complete pivoting).

Remember that a stable algorithm {\em should not increase} errors
already caused by conditioning, but it cannot cancel them out.

\begin{exercise}
(JPE, September 2004) Compute the LU decomposition $A=LU$ for the
matrix
$$
  A = \left[\begin{array}{cc} 0.01 & 2 \\ 1 & 3 \end{array}\right]
$$
Compute $\|L\|_{\infty} \|U\|_{\infty}$. What does this imply about the
numerical stability of solving a system of linear equations $Ax=y$ by
LU decomposition without pivoting?
\end{exercise}

\newpage

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\refstepcounter{section} \label{Ch14}

\centerline{Chapter \ref{Ch14} }\vspace{5mm}

\centerline{\Large\bf Numerically Stable Least Squares}\vspace{3mm}

\addcontentsline{toc}{section}{\ref{Ch14}. Numerically Stable Least
Squares}

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\setcounter{subsection}{0} \setcounter{exercise}{0}
\setcounter{equation}{0}

In Chapter~\ref{Ch10} we discussed overdetermined systems of equations
$$
        Ax \approx b
$$
where $A\in\IC^{m\times n}$ with $m\geq n$ and presented three
algorithms for computing the least squares solution $x$: (i) based on
normal equations, (ii) based on QR decomposition, and (iii) based on
SVD.

\subsection{Normal equations revisited} \label{16.10a}

Algorithm \ref{A1} via normal equations  has many advantages. It is the
simplest and most elegant algorithm for the least squares problem. It
should be used whenever the computations are precise, in particular if
one solves the problem manually (on paper).

For machine computations, algorithm \ref{A1} seems to be good, too: it
is at least twice as cheap (in terms of flops) as the other two
algorithms, so it is the fastest one. It has been used in computer
programming almost exclusively before 1970s, when computers were slow
and the top speed was high priority.

However, machine computations involve round-off errors. And it has been
gradually noticed that Algorithm \ref{A1} based on normal equations
tends to magnify errors and produce inaccurate results to the extent
that was eventually deemed unacceptable. As of now, normal equations
are almost entirely abandoned and used only to demonstrate how bad
things can be.

The loss of accuracy can be best understood in terms of the condition
number. If $A$ is the given matrix, then we have seen
(Section~\ref{ExtKappaRect}) that
$$
      \kappa_2(A^{\ast}A)=[\kappa_2(A)]^2.
$$
If the matrix $A$ is ill-conditioned (i.e., its condition number is
large), then the matrix $A^\ast A$ will have a much larger condition
number.

More precisely, suppose that our machine arithmetic has precision
$\bu=10^{-p}$ (defined in Section~\ref{MachineEpsilon}) and let
$\kappa_2(A)\sim 10^{q}$ for some $q>0$. Then a numerical solution of
the least squares problem should produce $p-q$ accurate decimal digits.
However, solving the problem by normal equations would involve the
matrix $A^\ast A$ whose condition number is
$$
   \kappa_2(A^{\ast}A)=[\kappa_2(A)]^2\sim 10^{2q}.
$$
Thus the resulting numerical solution would have only $p-2q$ accurate
decimal digits: we lose an extra $q$ accurate digits.

Such a loss of accuracy may be disastrous. Suppose we use single
precision ($p=7$) and our matrix $A$ is just mildly ill-conditioned,
with $\kappa_2(A)\sim 10^3$ (recall Section~\ref{15.5}). Then solving
the problem via normal equations we would give us just $7-2\cdot 3=1$
(one!) accurate digit, instead of $7-3=4$ that can be theoretically
expected for the given matrix.

Another extreme example: suppose
$$
    A=\left[\begin{array}{cc} 1 & 1 \\ \varepsilon & 0 \\
     0 & \varepsilon \end{array}\right]
$$
where $\varepsilon\approx \frac 12\, \sqrt{\bu}$. This matrix has
condition number
$$
   \kappa_2(A) =\frac{\sqrt{2+\varepsilon^2}}{\varepsilon}
   \approx \frac{2}{\varepsilon},
$$
so we can expect the numerical solution to have relative error $\sim
\bu/\varepsilon \sim \sqrt{\bu}$, which is not too bad (in single
precision we expect 3-4 accurate digits, and in double precision --
eight accurate digits).

However, normal equations require the construction of the matrix
$$
   A^\ast A=\left[\begin{array}{cc} 1+\varepsilon^2 & 1 \\
        1 & 1+\varepsilon^2 \end{array}\right]
$$
Since $\varepsilon^2< \bu$, the matrix $A^\ast A$ will be stored in the
computer memory as $\left [
\begin{smallmatrix}1 & 1\\ 1 & 1\end{smallmatrix} \right ]$ (the
additional term $\varepsilon^2$ would disappear due to round-off;
recall Section~\ref{15.5}). The matrix $\left [\begin{smallmatrix}1 &
1\\ 1 & 1\end{smallmatrix} \right ]$ is singular, so the algorithm via
normal equations would just fail (the computer program would crush).

Still, it is possible to find a reasonably good numerical solution in
the last example, with relative error $\sim \sqrt{\bu}$, if one uses
more elaborate algorithms.

\subsection{QR-based algorithm revisited} \label{16.10b}

Let us examine the accuracy of the QR-based Algorithm \ref{A2}.

Due to Section~\ref{15.6} (part 4), $\kappa_2(A)=\kappa_2(QA)$ for
any unitary matrix $Q\in\IC^{m\times m}$, hence
$$
   \kappa_2(R)=\kappa_2(Q^\ast A) =\kappa_2(A).
$$
Therefore, this method is safe, regarding the conditioning of the
problem, no additional loss of accuracy occurs due to conditioning.

The other aspect of numerical algorithms is stability, described in
Chapter~\ref{Ch13}. The actual error may greatly depend on a particular
sequence of computations producing the matrices $Q$ and $R$.

So far we have mentioned two specific procedures for constructing $Q$
and $R$: the classical Gram-Schmidt (Sect.~\ref{GramSchmidt}) and the
modified Gram-Schmidt (Sect.~\ref{MGS}). It turns out that the
classical Gram-Schmidt is {\em unstable}, hence it leads to
unpredictable round-off errors in numerical computations. The modified
Gram-Schmidt is \emph{stable} and generally produces accurate results
(see programming assignment).

However, there are better algorithms for computing the QR
decomposition: those are based on reflection and rotation matrices. We
will learn them in this chapter.

\subsection{SVD-based algorithm revisited} \label{16.10c}

Lastly we briefly examine the accuracy of the SVD-based Algorithm
\ref{A3}.

Let $A=UDV^\ast$ be an SVD of $A$. Due to Section~\ref{15.6} (part
4), $\kappa_2(A)=\kappa_2(UA) =\kappa_2(AV)$ for any unitary matrices
$U\in\IC^{m\times m}$ and $V\in\IC^{n\times n}$ hence
$$
   \kappa_2(D)=\kappa_2(U^\ast AV) =\kappa_2(A).
$$
Therefore, this method is just as safe, regarding the conditioning of
the problem, as the QR-based Algorithm \ref{A2}.

However, Algorithm \ref{A3} requires the computation of an SVD for the
matrix $A$, for which no simple algorithm exists (the computation of
SVD is beyond the scope of this course). Standard software packages
(like MATLAB) use only stable and highly accurate algorithms for the
SVD computation. If one uses a standard software package, the SVD-based
method \ref{A3} is as good as the QR-based method \ref{A2} (see
programming assignment).

Just a minor note: the SVD-based method \ref{A3} is more reliable when
the matrix $A$ is almost singular or exactly singular; cf.\
Section~\ref{Adefic}.

\subsection{Hyperplanes} \label{16.11}

Let $V$ be a finite dimensional vector space. A subspace $W\subset V$
is called a {\em hyperplane} if dim$\, W=\,$dim$\,V-1$. If $V$ is an
inner product space and $W\subset V$ a hyperplane, then
$$
  {\rm dim}\,W^{\perp}=1.
$$
If $x\in W^\perp$ is a nonzero vector, then $W^\perp =\,{\rm
span}\{x\}$. In that case
$$
   W = \bigl(W^\perp\bigr)^\perp = \bigl({\rm span}\{x\}\bigr)^\perp.
$$
Hence any hyperplane $W$ is completely determined by a non-zero vector
$x$ orthogonal to it.

\subsection{Reflections} \label{16.11b}

Let $W\subset V$ be a hyperplane in an inner product space $V$. Recall
that $V = W \oplus W^\perp$  (Sect.~\ref{ODS}). Thus for any vector
$v\in V$ we have a unique decomposition $v=w+w'$, where $w\in W$ and
$w'\in W^{\perp}$.

The linear operator $P$ on $V$ defined by
$$
   Pv=P(w+w')=w-w'
$$
is called a {\em reflection} (or {\em reflector}) across the hyperplane
$W$.

\begin{center}
\begin{tikzpicture}
  \fill[yellow!30] (-2,-0.4)--(-1.2,0.4)--(3.5,0.4)--(2.7,-0.4)--cycle;
  \node[anchor=south west] at (-1.66,-0.42) {\footnotesize\color{brown} $W$};
  \draw[black!70,->,-latex] (-0.1,0)--(1.8,0) node[right]{\scriptsize $w$};
  \draw[black,->,-latex] (-0.1,0)--(1.8,0.8) node[right]{\scriptsize $v$};
  \draw[black!70,->,-latex] (-0.1,0)--(-0.1,0.8) node[left]{\scriptsize $w'$};
  \draw[black!70,dotted] (1.8,0.8)--(1.8,0);
  \draw[blue!70,dotted] (1.8,0)--(1.8,-0.8);
  \draw[black!70,dotted] (-0.1,0.8)--(1.8,0.8);
  \draw[red,->,-latex] (-0.1,0)--(-0.1,0.3) node[below left]{\scriptsize $x$};
  \draw[blue,->,-latex] (-0.1,0)--(1.8,-0.8) node[right]{\scriptsize $P(v)$};
  \draw[blue!70,->,-latex] (-0.1,0)--(-0.1,-0.8) node[left]{\scriptsize $-w'$};
  \draw[blue!70,dotted] (-0.1,-0.8)--(1.8,-0.8);
\end{tikzpicture}
\end{center}

{\footnotesize Note: $P(w)=w$ for all $w\in W$, so $P$ is an identity
on $W$. Also, $P(w')=-w'$ for all $w'\in W^\perp$, so $P$ negates
vectors orthogonal to $W$.

Since $W$ is completely determined by a nonzero vector $x$ orthogonal
to $W$, the reflector can be fully described in terms of $x$.}

\subsection{Householder reflection matrices} \label{16.12}

Let $x\neq 0$ be a nonzero vector in $\IC^n$. The $n\times n$ matrix
$$
     P=I-2\,\frac{x x^\ast}{x^\ast x}
     =I-2\,\frac{x x^\ast}{\|x\|^2}
$$
is called the {\em Householder reflection matrix} (or \emph{Householder
reflector}) corresponding to $x$.

{\footnotesize Obviously, $P$ is unchanged if $x$ is replaced by $cx$
for any $c\neq 0$.}

\subsection{Basic properties of Householder reflectors} \label{16.13}

Let $P$ be the reflector matrix corresponding to a vector $x\neq 0$.
Then
\begin{itemize}\item[(a)] $Px=-x$. \item[(b)] $Py=y$ whenever $\la
y,x\ra =0$. \item[(c)] $P$ is Hermitian (in the real case it is
symmetric). \item[(d)] $P$ is unitary (in the real case it is
orthogonal). \item[(e)] $P$ is involution, i.e. $P^2=I$.
\end{itemize}

{\footnotesize \emph{Proof}. Direct calculation. \qed}

\subsection{Theorem} \label{16.14}

Let $y\in\IC^n$. Choose a scalar $\sigma\in\IC$ so that
\beq \label{sigmay}
 |\sigma|=\|y\|
 \qquad\text{and}\qquad
 \sigma\cdot\la e_1,y\ra\in\IR.
\eeq
Suppose that $x=y+\sigma e_1\neq 0$. Let $P=I-2x x^\ast/\|x\|^2$ be the
corresponding Householder reflector. Then $Py=-\sigma e_1$.

\medskip\noindent{\footnotesize \emph{Proof}. By direct calculation,
we have
$$
  \la y-\sigma e_1,y+\sigma e_1\ra =\|y\|^2-\sigma\la
  e_1,y\ra+\bar{\sigma}\la y,e_1\ra-|\sigma|^2=0
$$
where we used both identities of \eqref{sigmay}. Now
\begin{align*}
   \text{\ref{16.13}(a) implies:}&\ &P(y+\sigma e_1)&=-y-\sigma e_1\\
   \text{\ref{16.13}(b) implies:}&\ &P(y-\sigma e_1)&=\phantom{-}y-\sigma e_1
\end{align*}
Adding these two equations proves the theorem. \qed}

{\footnotesize
\subsection{Remarks} \label{16.15}

\begin{itemize}\item[(a)]
To choose $\sigma$ in Theorem~\ref{16.14}, write a polar
representation for $\la e_1,y\ra=re^{i\theta}$ and then set
$\sigma=\pm \|y\|e^{-i\theta}$. \item[(b)] In the real case, we
have $\la e_1,y\ra\in\IR$, and one can just set $\sigma=\pm \|y\|$.
For machine calculation with round-off errors, the better choice is
$$
     \sigma = \begin{cases} +\|y\| &\mbox{if }\ \la e_1,y\ra\geq 0\\
     -\|y\| &\mbox{if }\ \la e_1,y\ra<0 \end{cases}.
$$
Then the computation of the vector $x=y+\sigma e_1$ will be
numerically stable, there will be no danger of catastrophic
cancelation.
\item[(c)]It is geometrically obvious that for any two vectors
    $x,y\in\IR^n$ with equal 2-norms, $\|x\|_2=\|y\|_2$, there is a
    reflector $P$ that takes $x$ to $y$. In the complex space
    $\IC^n$, this is \emph{not true}: for generic vectors $x,y \in
    \IC^n$ with equal 2-norms there is no reflector that takes $x$
    to $y$. But according to Theorem~\ref{16.14}, one can always
    find a reflector that takes $x$ to $cy$ with some scalar
    $c\in\IC$.
\end{itemize}
}

\subsection{Corollary} \label{16.16}

For any vector $y \in \IC^n$ there is a scalar $\sigma$ (which was
defined in \ref{16.14} and specified in \ref{16.15}) and a matrix $P$,
which is either a reflector or the identity ($P=I$), such that
$Py=-\sigma e_1$.

\medskip{\footnotesize \emph{Proof}. Apply Theorem~\ref{16.14} in the
case $y+\sigma e_1\neq 0$ and set $P=I$ otherwise. \qed}

\subsection{QR Decomposition via Householder reflectors} \label{16.17}

For any $A\in\IC^{m\times n}$ with $m\geq n$ there is a QR
decomposition with a unitary matrix $Q\in\IC^{m\times m}$ that is a
product of Householder reflectors. The number of those reflectors is
$\leq n$.

\medskip\noindent{\footnotesize \emph{Proof}.
We use induction on $n$. Let $n=1$, so that $A$ is just one column
$m$-vector. By Corollary~\ref{16.16} there is a matrix $P$ (a reflector
or identity) such that $PA=-\sigma e_1$ with a scalar $\sigma\in\IC$.
Hence, $A=PR$ where $R=-\sigma e_1$ is am upper triangular matrix
$m\times 1$. This provides the basis for our induction.

Now let $n\geq 1$. Let $a_1$ denote the first column of $A$. Again, by
Corollary~\ref{16.16} there is a matrix $P$ (a reflector or identity)
such that $Pa_1=-\sigma e_1$. Hence,
$$
    PA=\begin{bmatrix*}[r] -\sigma &
     w^\ast \\ 0 & B\end{bmatrix*}
$$
where $w\in\IC^{n-1}$ and $B\in\IC^{(m-1)\times (n-1)}$. By the
inductive assumption, there is a unitary matrix $Q_1\in\IC^{(m-1)\times
(m-1)}$ and an upper triangular matrix $R_1\in\IC^{(m-1)\times (n-1)}$
such that $B=Q_1R_1$. Now we expand $Q_1$ to a unitary $m\times m$
matrix as follows:
$$
     Q_2=\begin{bmatrix} 1 &
     0 \\ 0 & Q_1 \end{bmatrix}
$$
By Section~\ref{UnitaryColumnsRows}, the matrix $Q_2$ is unitary
whenever $Q_1$ is. Furthermore, if $Q_1$ is a product of $\leq n-1$
reflectors, then the same is true for $Q_2$. Now one can easily check
that $PA=Q_2R$ where
$$
     R=\begin{bmatrix*}[r] -\sigma & w^\ast \\
      0 & R_1 \end{bmatrix*}
$$
is an upper triangular matrix. Hence, $A=QR$ with $Q=PQ_2$. \qed}

\subsection{Givens rotation matrices} \label{16.24}

Let $1\leq p<q\leq m$ and $\theta\in [0,2\pi)$. The matrix
$G=G_{p,q,\theta}$ defined by
$$
  g_{pp}=\cos\theta,\ \ g_{pq}=\sin\theta,\ \
  g_{qp}=-\sin\theta,\ \ g_{qq}=\cos\theta,
$$
and $g_{ij}=\delta_{ij}$ otherwise is called a {\em Givens rotation
matrix} (or {\em Givens rotator}): {\footnotesize
    $$
       G=   \left[\begin{array}{ccccccccccc}
                    1 & & & & & & & & & & \\
                    & \ddots & & & & & & & & & \\
                    & & 1 & & & & & & & & \\
                    & & & c & & & & s & & & \\
                    & & & & 1 & & & & & & \\
                    & & & & & \ddots & & & & & \\
                    & & & & & & 1 & & & & \\
                    & & & -s & & & & c & & & \\
                    & & & & & & & & 1 & & \\
                    & & & & & & & & & \ddots & \\
                    & & & & & & & & & & 1
   \end{array}\right]\begin{array}{c} \phantom{\ddots} \\ \phantom{\ddots} \\ \phantom{\ddots} \\ \leftarrow p \\
       \phantom{1} \\ \phantom{\ddots} \\ \phantom{1} \\ \leftarrow q \\ \phantom{1} \\ \phantom{1} \\
       \phantom{\ddots} \\ \phantom{1} \end{array}
   $$
   $$
       \quad \uparrow \qquad\qquad\qquad\ \  \uparrow\qquad
    $$
    $$
       \quad p \qquad\qquad\qquad\ \ q \qquad
    $$}
(here we use convenient shorthand notation $c=\cos\theta$ and
$s=\sin\theta$).\medskip

It is easy to check that $G$ is an orthogonal matrix (recall
Sect.~\ref{UnitaryColumnsRows}).

\subsection{Geometric description of Givens rotators}

Note that the vector $y=Gx$ has components
\begin{align*}
   y_p &= \phantom{-}x_p\cos\theta + x_q\sin\theta\\
   y_q &= -x_p\sin\theta + x_q\cos\theta\\
   y_i &= \phantom{-}x_i\qquad\qquad\qquad\qquad\forall\ i\notin\{p,q\}
\end{align*}
Hence the matrix $G$ defines a clockwise rotation through the angle
$\theta$ of the $x_px_q$ coordinate plane in $\IR^m$ with all the other
coordinates left unchanged.

\newpage

\subsection{QR decomposition via Givens rotators} \label{16.25}

For any $A\in\IR^{m\times n}$ with $m\geq n$ there is a QR
decomposition with an orthogonal matrix $Q\in\IR^{m\times m}$ that is a
product of Givens rotators.

\medskip\noindent{\footnotesize \emph{Proof}.
Let $a_p$ be the leftmost column of $A$ that contains a nonzero entry
below the main diagonal, i.e.,
$$
   p=\min\{j\geq 1\colon a_{ij}\neq 0\quad\text{for some}\ \ i>j\}
$$
(if such $p$ does not exist, the matrix $A$ is already upper
triangular).

Let $a_{qp}\neq 0$ be one of the nonzero entries in the column $a_p$
below the main diagonal, i.e., satisfying $q>p$. Consider the matrix
$A'=GA$ where $G=G_{p,q,\theta}$ is a Givens rotator. Note that $p$ and
$q$ are already selected, but we are still free to choose $\theta$.

The structure of the matrices $A=(a_{ij})$ and $A'=(a_{ij}')$ can be
revealed by examining the following diagram:
    $$
       \left[\begin{array}{ccccccccccc}
                    1 & & & & & & & & & & \\
                    & \ddots & & & & & & & & & \\
                    & & 1 & & & & & & & & \\
                    & & & c & & & & s & & & \\
                    & & & & 1 & & & & & & \\
                    & & & & & \ddots & & & & & \\
                    & & & & & & 1 & & & & \\
                    & & & -s & & & & c & & & \\
                    & & & & & & & & 1 & & \\
                    & & & & & & & & & \ddots & \\
                    & & & & & & & & & & 1
       \end{array}\right]
       \left[\begin{array}{ccccccccccc}
                    * & \cdots  &  & & & & & & & & \\
                    0 & \ddots & & & & & & & & & \\
                    0 & 0 & * & * & & & & & & & \\
                    0 & \cdots & 0 & a_{pp} &  & & &  & & & \\
                    0 & \cdots & 0 & * &  & & & & & & \\
                    \vdots & \ddots & \vdots & \vdots &  &  & & & & & \\
                    0 & \cdots & 0 & * & &  &  & & & & \\
                    0 & \cdots & 0 & a_{qp} & & & &  & & & \\
                    0 & \cdots & 0 & * & & & & &  & & \\
                    \vdots & \ddots & \vdots & \vdots & & & & & & & \\
                    0 & \cdots & 0 & * & \cdots & & & & & &
       \end{array}\right]
$$
$$
      = \left[\begin{array}{ccccccccccc}
                    * & \cdots  &  & & & & & & & & \\
                    0 & \ddots & & & & & & & & & \\
                    0 & 0 & * & * & & & & & & & \\
                    0 & \cdots & 0 & a_{pp}' &  & & &  & & & \\
                    0 & \cdots & 0 & * &  & & & & & & \\
                    \vdots & \ddots & \vdots & \vdots &  &  & & & & & \\
                    0 & \cdots & 0 & * & &  &  & & & & \\
                    0 & \cdots & 0 & a_{qp}' & & & &  & & & \\
                    0 & \cdots & 0 & * & & & & &  & & \\
                    \vdots & \ddots & \vdots & \vdots & & & & & & & \\
                    0 & \cdots & 0 & * & \cdots & & & & & &
       \end{array}\right]
$$
The following facts can be verified by direct inspection:
\begin{itemize} \item[(a)] The first $p-1$ columns of $A'$
are the same as those of $A$, hence they are still zero below the
main diagonal.
\item[(b)] In the $p$-th column, only the components $a_{pp}'$ and
    $a_{qp}'$ will be different from the corresponding components
    of $A$ (hence, if $A$ had some zeros in the $p$-th column below
    the main diagonal, those will propagate to $A'$). \item[(c)]
    The subdiagonal component $a_{qp}'$ of $A'$ is given by
$$
   a_{qp}'=-a_{pp}\sin\theta+a_{qp}\cos\theta.
$$
\end{itemize}
Our goal is to make $a_{qp}'=0$. We can achieve this by using our
freedom to choose $\theta$. The following choice will do it:
\beq \label{sincosGivens}
    \cos\theta=\frac{a_{pp}}{\sqrt{a_{pp}^2+a_{qp}^2}}
    \qquad\text{and}\qquad
    \sin\theta=\frac{a_{qp}}{\sqrt{a_{pp}^2+a_{qp}^2}}.
\eeq
Note that one never actually evaluates the angle $\theta$, since
$G_{p,q,\theta}$ only contains $\cos\theta$ and $\sin\theta$, and these
are given by the above formulas.

In this way we eliminate one nonzero element $a_{qp}$ below the main
diagonal. Working from left to right, one can convert $A$ into an upper
triangular matrix $\tilde{G}A=R$ where
$$
  \tilde{G} = G_{p_k,q_k,\theta_k}\cdots G_{p_1,q_1,\theta_1}
$$
will be a product of Givens rotators. Each nonzero element of $A$ below
the main diagonal requires one multiplication by a rotator. Then we get
$A=QR$ with an orthogonal matrix
$$
  Q=\tilde{G}^T = G_{p_1,q_1,\theta_1}^T\cdots G_{p_k,q_k,\theta_k}^T.
$$
It remains to note that $G_{p,q,\theta}^T = G_{p,q,-\theta}$, so it is
a Givens rotator, too. \qed}

\subsection{Cost of QR via Givens rotators} \label{16.25a}

The evaluation of $\cos\theta$ and $\sin\theta$ by \eqref{sincosGivens}
takes 6 flops (the square root extraction is counted here as one flop).
The subsequent multiplication of $A$ by $G_{p,q,\theta}$ takes $6n$
flops. Thus, the elimination of one nonzero subdiagonal component of
$A$ takes $6n+6$ flops. If $A$ originally has $k$ nonzero subdiagonal
entries, the QR decomposition via Givens rotators will take
$6k(n+1)\sim 6kn$ flops.

When the lower triangle of $A$ is mostly filled with nonzero numbers,
then $k$ is close to its maximal value, $mn - n^2/2$. In that case the
total cost will be very high, it will greatly exceed the cost of QR via
Householder reflectors. Hence Givens rotators are very inefficient for
generic matrices.

However, Givens rotators work well if the matrix $A$ is {\em sparse},
i.e., contains just a few nonzero components below the main diagonal.
Then Givens rotators can give the quickest result. We will see such
instances later.

\newpage

\begin{exercise}
Let $x,y\in\IC^n$ be such that $x\neq y$ and $\|x\|_2=\|y\|_2\neq 0$.
Show that there is a reflector matrix $P$ such that $Px=y$ \textbf{if
and only if} $\langle x,y\rangle \in\IR$. For an extra credit: show
that if the above reflector exists, then it is unique.
\end{exercise}

\begin{exercise}
(simplified of JPE, May 2011) Prove that any Givens rotator matrix in
$\IR^2$ is a product of two Householder reflector matrices. Can a
Householder reflector matrix be a product of Givens rotator matrices?
\end{exercise}

\begin{exercise}
[Bonus] (JPE May, 2010) Let
$$
  A=\left [ \begin{array}{cr} 3 & -3 \\ 0 & 4 \\ 4 & 1 \end{array}\right
  ]
$$
\begin{itemize} \item[(a)] Find the QR factorization of $A$ by Householder reflectors.
\item[(b)] Use the results in (a) to find the least squares
    solution of $Ax = b$, where $$b = [16\ \ 11\ \ 17]^T$$
\end{itemize}
    (\emph{Note: there is a typo in the original JPE exam, it is
    corrected here.})
\end{exercise}

\newpage

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\refstepcounter{section} \label{Ch15}

\centerline{Chapter \ref{Ch15} }\vspace{5mm}

\centerline{\Large\bf Computation of Eigenvalues: Theory}\vspace{3mm}

\addcontentsline{toc}{section}{\ref{Ch15}. Computation of Eigenvalues: Theory}

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\setcounter{subsection}{0} \setcounter{exercise}{0}
\setcounter{equation}{0}


\subsection{Preface} \label{19.1}

Eigenvalues of a matrix $A\in\IC^{n\times n}$ are the roots of its
characteristic polynomial
$$
   C_A(x) = \det(xI-A),\qquad \text{where}\quad \deg C_A(x) = n.
$$
It is a consequence of the famous Galois group theory (Abel's theorem) that there is no closed formula
for the roots of a generic polynomial of degree $>4$. Hence, there are
no finite algorithms for computation of the roots of polynomials (or
eigenvalues, for that matter).

Thus, all the methods for computing eigenvalues of matrices of
size $n\geq 5$ are necessarily iterative, they provide successive
approximations to the eigenvalues, but never exact results.
Furthermore, even though for $n=3$ and $n=4$ exact formulas exist,
they are rather impractical and often numerically unstable, so
even in these cases iterative methods are used instead.

For this reason, matrix decompositions that reveal eigenvalues (Schur
and SVD) cannot be implemented by finite algorithms. On the other hand,
decompositions that do \emph{not} reveal eigenvalues (QR, LU, Cholesky) \emph{can} be implemented by finite algorithms (and we
have seen some of those):

\begin{center}
\begin{tabular}{cc}
 {\footnotesize Decompositions} & {\footnotesize Decompositions} \\
  $\phantom{m}${\footnotesize  revealing eigenvalues} $\phantom{m}$ &  $\phantom{m}${\footnotesize not revealing eigenvalues} $\phantom{m}$ \\
\hline  \parbox[c][2cm][c]{6em}{$\phantom{mm}$SVD\\ $\phantom{mmi}$LU} &
 \parbox[c][2cm][c]{6em}{$\phantom{mm}$Schur\\ $\phantom{m}$Cholesky \\
    $\phantom{mmi}$QR} \\
\hline {\footnotesize No finite} & {\footnotesize Finite} \\
   {\footnotesize algorithms exist} & {\footnotesize algorithms exist}
\end{tabular}%\vspace*{0.2cm}
\end{center}


In the last chapters of the course we will learn iterative algorithms for computing eigenvalues and
eigenvectors. We note that the most important matrix decomposition, SVD, requires the eigenvalues of a Hermitian positive semidefinite matrix
$A^\ast A$. Hence it is particularly important to develop algorithms
for computing eigenvalues of Hermitian matrices.

If an eigenvalue $\lambda$ of a matrix $A$ is known, an eigenvector $x$ corresponding to $\lambda$
can be found by solving the linear system $(A-\lambda I)x=0$ (say, by
LU decomposition). Conversely, if an eigenvector $x$ is known, the
corresponding eigenvalue $\lambda$ can be immediately found by
simple formula $\lambda = (Ax)_i/x_i$, whenever $x_i\neq 0$. Thus eigenvalues help to compute eigenvectors and vice versa. For this reason eigenvalues and eigenvectors
are often computed `in parallel'. In this chapter, we develop a
theoretical basis for computation of eigenvalues and eigenvectors,
while in the next two chapters we will turn to practical algorithms.


\subsection{Rayleigh quotient} \label{18.4}

Let $A\in\IC^{n\times n}$. We call
$$
      r(x)= \frac{x^{\ast}Ax}{x^{\ast}x}
      =\frac{\la Ax,x\ra}{\la x,x\ra},
      \qquad x\neq 0
$$
the {\em Rayleigh quotient} of $x$.

\subsection{Restriction to the unit sphere} \label{18.4a}

The Rayleigh quotient is defined for all vectors $x\neq 0$, so it is a function on $\IC^n\setminus \{0\}$, with values in $\IC$. However, for any nonzero scalar $c\neq 0$
$$
  r(cx)=\frac{\la cAx,cx\ra}{\la cx,cx\ra}
  =\frac{|c|^2\la Ax,x\ra}{|c|^2\la x,x\ra}
  =\frac{\la Ax,x\ra}{\la x,x\ra}=r(x),
$$
hence $r$ is constant on the line span$\{x\}$. Since any nonzero vector $x\neq 0$ is a scalar multiple of a unit vector $u = x/\|x\|$, the function $r(x)$ is completely defined by its values on the unit sphere \eqref{UnitSphere}
$$
   {\mathbb S}_1 = \{u\in \IC^n\colon \|u\|=1\}.
$$
On the unit sphere ${\mathbb S}_1$ the Rayleigh quotient can be computed by a simpler formula:
$$
          r(u)= u^{\ast}Au=\la Au,u\ra.
$$
Note that $r(u)$ is a quadratic function of the coordinates of $u \in {\mathbb S}_1$.

\subsection{Properties of Rayleigh quotient} \label{18.4b}

\begin{itemize}
\item[(a)] If $A$ is Hermitian, then $r(x)\in\IR$ for any nonzero vector $x\in\IC^n$.\\ {\footnotesize $\phantom{mmmm}$[This follows from Section~\ref{12.10bb}(a)]}
\item[(b)] If $x$ is an eigenvector with eigenvalue $\lambda$, then $Ax=\lambda x$ and so
$$
       r(x)=\lambda
$$
\item[(c)] If $x$ is an arbitrary unit vector, then $r(x)\,x=(x^{\ast}Ax)\,x$
is the orthogonal projection of the vector $Ax$ onto the line
spanned by $x$. Hence
$$
   \|Ax-r(x)\,x\|_2=\min_{\mu\in\IC}\|Ax-\mu x\|_2
$$
\end{itemize}
Note: if one regards $x$ as an `approximate' eigenvector, then the Rayleigh
quotient $r(x)$ is the best choice that one can make for the
associated `approximate' eigenvalue in the sense that the value $\mu =
r(x)$ comes closest (in the 2-norm) to achieving the desired relation
$Ax-\mu x=0$.

\medskip\noindent
{\footnotesize Let $A\in\IC^{n\times n}$ and $x$ a unit eigenvector of $A$ corresponding to eigenvalue $\lambda$. Let $y\approx x$ be another unit vector close to $x$ and $r=y^{\ast}Ay$. It follows from the above that $r\approx \lambda$, but how close is $r$ to $\lambda$?}

\subsection{Theorem} \label{18.6}

Let $A\in\IC^{n\times n}$ and $x$ a unit eigenvector of $A$
corresponding to eigenvalue $\lambda$. Let $y$ be another unit
vector and $r=y^{\ast}Ay$. Then
\beq \label{lambdar1}
   |\lambda - r|\leq 2\, \|A\|_2\, \|x-y\|_2.
\eeq
Moreover, if $A$ is a Hermitian matrix, then there is a constant
$C=C(A)>0$ such that
\beq \label{lambdar2}
   |\lambda - r|\leq C\, \|x-y\|_2^2.
\eeq

{\footnotesize \proof To prove \eqref{lambdar1}, we write
$$
   \lambda-r=x^{\ast}A(x-y)+(x-y)^{\ast}Ay.
$$
Now the triangle inequality and Cauchy-Schwarz inequality readily give \eqref{lambdar1}.

Now, assume that $A$ is Hermitian. Then there is an ONB of
eigenvectors, and we can assume that $x$ is one of them. Denote that
ONB by $\{x,x_2,\ldots,x_n\}$ and the corresponding eigenvalues by
$\lambda,\lambda_2,\ldots, \lambda_n$. Let $y=cx+c_2x_2+\cdots
+c_nx_n$. Then
$$
     \|y-x\|^2=|c-1|^2+\sum_{i=2}^n |c_i|^2\geq \sum_{i=2}^n |c_i|^2
$$
On the other hand, $\|y\|=1$, i.e., $c^2+c_2^2+\cdots+c_n^2=1$, so
$$
      \lambda=\lambda |c|^2+\sum_{i=2}^n\lambda |c_i|^2
$$
Now, $Ay=c\lambda x+\sum_{i=2}^nc_i\lambda_ix_i$, therefore
$$
     r=\la Ay,y\ra =\lambda |c|^2+\sum_{i=2}^n\lambda_i |c_i|^2.
$$
Subtracting the last two formulas gives
$$
     \lambda-r=\sum_{i=2}^n(\lambda-\lambda_i)|c_i|^2.
$$
The result now follows with
$$
      C=\max_{2\leq i\leq n}|\lambda-\lambda_i|.
$$
The theorem is proved. \qed}

\medskip\noindent
Note: one may feel uncomfortable to deal with an unspecified constant $C$ in \eqref{lambdar2}. In fact, it easily follows from Sect.~\ref{12.10aa} that $C\leq 2\|A\|_2$.

\subsection{Lemma} \label{18.8}

Let $L$ and $G$ be subspaces of $\IC^n$ and dim$\, G >\,$dim$\, L$.
Then there is a nonzero vector $y\in G$ orthogonal to $L$, i.e., $y\in L^\perp$.

{\footnotesize \proof By way of contradiction, suppose $G\cap L^{\perp}=\{0\}$. Then
$G\oplus L^{\perp}$ would be a subspace of $\IC^n$, and its dimension would be
$$
  {\rm dim}\,G+\,{\rm dim}\,L^\perp =\,{\rm dim}\,G+ n-\,{\rm dim}\, L>n,
$$
which is impossible. \qed}

\medskip\noindent Next let $A\in\IC^{n\times n}$ be a Hermitian matrix with eigenvalues
$\lambda_1,\ldots,\lambda_n$. Since they are all real, we can
number them in an increasing order, so that $$
   \lambda_1\leq\cdots\leq\lambda_n.
$$
Also recall that $x^\ast Ax \in \IR$ for any vector $x\in\IC$, by Sect.~\ref{18.4b} (a).

\subsection{Courant-Fisher Minimax Theorem}
\label{18.9}

Let $A\in\IC^{n\times n}$ be a Hermitian matrix with eigenvalues
$\lambda_1\leq\cdots\leq\lambda_n$. Then for every $i=1,\ldots,n$
$$
     \lambda_i=\min_{L\colon {\rm dim}\, L=i}\ \max_{x\in L\setminus \{0\} }
       \frac{x^{\ast}Ax}{x^{\ast}x}
$$
where $L$ stands for a vector subspace of $\IC^n$.

\proof Let $\{u_1,\ldots,u_n\}$ be an ONB of eigenvectors of $A$
corresponding to the eigenvalues $\lambda_1,\ldots,\lambda_n$. If
dim$\, L=i$, then by Lemma~\ref{18.8} there is a nonzero vector $y\in
L\setminus \{0\}$ orthogonal to the space span$\{u_1,\ldots,u_{i-1}\}$. Hence, the
first $i-1$ coordinates of $y$ are zero, i.e.\ $y=\sum_{j=i}^nc_ju_j$.
Thus
$$
       \frac{y^{\ast}Ay}{y^{\ast}y}=
       \frac{\sum_{j=i}^n|c_j|^2\lambda_j}{\sum_{j=i}^n|c_j|^2}\geq\lambda_i
$$
because $\lambda_j \geq \lambda_i$ for all $j=i,\ldots,n$. Therefore,
$$
     \max_{x\in L\setminus \{0\} } \frac{x^{\ast}Ax}{x^{\ast}x}\geq
     \frac{y^{\ast}Ay}{y^{\ast}y} \geq \lambda_i.
$$
On the other hand, let $L_i=\,$span$\{u_1,\ldots,u_i\}$. Obviously,
dim$\, L_i=i$ and for every nonzero vector $x\in L_i$ we have
$x=\sum_{j=1}^ic_ju_j$, so
$$
       \frac{x^{\ast}Ax}{x^{\ast}x}=
       \frac{\sum_{j=1}^i|c_j|^2\lambda_j}{\sum_{j=1}^i|c_j|^2}\leq\lambda_i
$$
because $\lambda_j \leq \lambda_i$ for all $j=1,\ldots,i$. The theorem is proved. \qed


\subsection{Corollary} \label{18.9a}

A close examination of the proof of the above theorem gives us
\beq \label{CourantFisherAdd}
     \lambda_i=\max_{x\in L_i\setminus \{0\} }
       \frac{x^{\ast}Ax}{x^{\ast}x},
\eeq
where $L_i=\,$span$\{u_1,\ldots,u_i\}$.

\subsection{Theorem} \label{18.10}

Let $A$ and $\Delta A$ be two Hermitian matrices. Let $\alpha_1\leq
\cdots\leq\alpha_n$ be the eigenvalues of $A$. Let $\delta_{\min}$ and
$\delta_{\max}$ the smallest and the largest eigenvalues of $\Delta A$.
Denote the eigenvalues of the matrix $B=A+\Delta A$ by
$\beta_1\leq\cdots\leq\beta_n$. Then for each $i=1,\ldots,n$
\beq \label{AdeltaAB}
      \alpha_i+\delta_{\min}\leq\beta_i\leq\alpha_i+\delta_{\max}.
\eeq

\proof Let $\{u_1,\ldots,u_n\}$ be an ONB of eigenvectors of $A$
corresponding to its eigenvalues $\alpha_1, \ldots,\alpha_n$. For each $i=1,\ldots,n$, let
$L_i=\,$span$\{u_1,\ldots,u_i\}$. Then dim$\,L_i=i$, hence by Theorem~\ref{18.9} and Corollary~\ref{18.9a}
\begin{align*}
  \beta_i &\leq \max_{x\in L_i\setminus\{0\}}\frac{x^{\ast}Bx}{x^{\ast}x}\\
          &= \max_{x\in L_i\setminus\{0\}}\biggl[\frac{x^{\ast}Ax}{x^{\ast}x}
           + \frac{x^{\ast}\Delta
           Ax}{x^{\ast}x}\biggr]\\
          &\leq \max_{x\in L_i\setminus\{0\}}\frac{x^{\ast}Ax}{x^{\ast}x}
           + \max_{x\in L_i\setminus\{0\}}\frac{x^{\ast}\Delta
           Ax}{x^{\ast}x}\\
     &= \alpha_i+\max_{x\in L_i\setminus\{0\}}\frac{x^{\ast}\Delta
     Ax}{x^{\ast}x}\\
     &\leq \alpha_i+\max_{x\in \IC^n\setminus\{0\}}\frac{x^{\ast}\Delta
     Ax}{x^{\ast}x}\\
         &= \alpha_i+\delta_{\max}
\end{align*}
where at the last step we used Section~\ref{12.10bb} (b). Thus we proved the right inequality in \eqref{AdeltaAB}. The left one can be proved by a similar calculation, but we can achieve the result quicker by a trick. We can rewrite that inequality as
\beq \label{left2}
      \alpha_i \leq \beta_i - \delta_{\min}
\eeq
and note that $- \delta_{\min}$ is
the \emph{largest} eigenvalue of the matrix $-\Delta A$. Then we apply the (already proven) right inequality in \eqref{AdeltaAB} to the matrices $B$, $-\Delta A$ and $A=B+(-\Delta A)$ and obtain \eqref{left2}. The theorem is completely proved. \qed

\subsection{Corollary} \label{18.10a}

Since $\|\Delta A\|_2 = \max\{|\delta_{\min}|,|\delta_{\max}|\}$ ({\footnotesize due to Section~\ref{12.10aa}}),
we have
$$
      |\alpha_i-\beta_i| \leq \|\Delta A\|_2 \qquad \forall i=1,\ldots,n
$$

\subsection{Approximation analysis using the residual} \label{18.11}

{\footnotesize Suppose one finds an approximate eigenvalue $\mu$ and an
approximate unit eigenvector $x$ of a matrix $A$. How can one estimate the closeness of $\mu$ to the actual (but unknown) eigenvalue $\lambda$ of $A$?

One way to do this is to compute the residual $r=Ax-\mu x$. If $r=0$, then $\mu$ is an eigenvalue of $A$. If that $r$ is small, then one may hope that $\mu$ is close to an eigenvalue of $A$. But how close is it?

To place this question in the context of the previous theorems, we define the matrix $\Delta A=-rx^{\ast}$. Then
$$
     (A+\Delta A)x=Ax-rx^{\ast}x=\mu x,
$$
so $(\mu,x)$ are an exact (!) eigenpair of the perturbed matrix $A+\Delta A$. Also note that $\|\Delta A\|_2=\|r\|_2$, so the norm $\|\Delta A\|_2$ is readily computable.

If we could apply Corollary~\ref{18.10a}, we would be able to conclude that
\beq \label{mu-lambda}
   |\mu-\lambda| \leq \|\Delta A\|_2 = \|r\|_2.
\eeq
But! In Corollary~\ref{18.10a} the matrices $A$ and $\Delta A$ must be
Hermitian. And our matrices (especially $\Delta A=-rx^{\ast}$) may not be Hermitian. Thus we need to extend the estimate in Corollary~\ref{18.10a}
to more general matrices.}

\subsection{Bauer-Fike theorem} \label{18.12}

Let $A\in\IC^{n\times n}$ be a diagonalizable matrix, so that
$$
      X^{-1}AX=D=\, {\rm diag}\{\lambda_1,\ldots,\lambda_n\}
$$
If $\mu$ is an eigenvalue of a perturbed matrix $A+\Delta A$,
then
$$
    \min_{1\leq i\leq n}|\mu-\lambda_i|
       \leq \kappa_p(X)\,\|\Delta A\|_p
$$
where $\|\cdot \|_p$ stands for any $p$-norm ($1 \leq p\leq \infty$).

{\footnotesize \proof
The matrix $A+\Delta A-\mu I$ is singular. Hence so is the matrix
$$
  X^{-1}(A+\Delta A-\mu I)X = D+X^{-1}\Delta A X-\mu I.
$$
If $D-\mu I$ is singular, then $\mu$ is an eigenvalue of $A$ and the claim is trivial. Otherwise
$$
             (D-\mu I)^{-1}\bigl [ D+X^{-1}\Delta A X-\mu I \bigr ]
     =I+(D-\mu I)^{-1}(X^{-1}\Delta A X)
$$
is a singular matrix. The Neumann lemma
(Exercise~\ref{ExNeumann}) implies
$$
 1\leq \|(D-\mu I)^{-1}(X^{-1}\Delta A X)\|_p
  \leq \|(D-\mu I)^{-1}\|_p\|X^{-1}\|_p\|\Delta A\|_p\|X\|_p
$$
Lastly, observe that $(D-\mu I)^{-1}$ is diagonal, so
$$
  \|(D-\mu I)^{-1}\|_p=\max_{1\leq i\leq n}\frac{1}{|\lambda_i-\mu|}
  =\frac{1}{\min_{1\leq i\leq n}|\lambda_i-\mu|}
$$
The theorem now follows. \qed}

\subsection{Corollary}

If $A$ is a normal matrix, then in the above theorem
$$
    \min_{1\leq i\leq n}|\mu-\lambda_i|
       \leq \|\Delta A\|_2
$$
because $X$ is a unitary matrix and so $\kappa_2(X)=1$. {\footnotesize Thus, \eqref{mu-lambda} is actually true for any normal matrix, in particular for any Hermitian matrix $A$.}

\medskip{\small Theorem~\ref{18.12} answers the question raised in Section~\ref{18.11}, it
gives an estimate on $|\mu-\lambda|$ in terms of $\|\Delta
A\|_p$ and $\kappa_p(X)$. However, this answer may not be good enough -- it
gives \emph{one} estimate for \emph{all} eigenvalues. In practical applications, some eigenvalues
can be estimated better than others. It is important then to develop
finer estimates for individual eigenvalues.}

\subsection{Left eigenvectors (definition)} \label{18.1}

Let $A\in\IC^{n\times n}$. A nonzero vector $x\in\IC^n$ is called
a {\em left eigenvector} of $A$ corresponding to an eigenvalue
$\lambda$ if
$$
      x^{\ast}A=\lambda x^{\ast}
$$
{\footnotesize Note: this is equivalent to
$$
   A^{\ast}x=\bar{\lambda}x,
$$
i.e., $x$ is an ordinary (``right'') eigenvector of $A^{\ast}$
corresponding to the eigenvalue $\bar{\lambda}$:
\beq \label{RLevectors}
  (\lambda,x)\text{ is a left eigenpair for } A
  \qquad\Longleftrightarrow\qquad
  (\bar{\lambda},x)\text{ is a right eigenpair for } A^\ast.
\eeq
But why don't we introduce a notion of \emph{left eigenvalue}?}

\subsection{Lemma} \label{18.2}

A matrix $A$ has a left eigenvector $x\neq 0$ corresponding to eigenvalue $\lambda$ if
and only if $\lambda$ is a root of the
characteristic polynomial of $A$:{\footnotesize
$$
  (\lambda,x)\text{ is a left eigenpair for } A
  \quad\Longleftrightarrow\quad
  (\lambda,y)\text{ is a right eigenpair for } A\text{ for some }y\neq 0
$$

\proof $A^{\ast}x=\bar{\lambda}x$ can be written as
$(A^{\ast}-\bar{\lambda}I)x=0$, and since $x\neq 0$, we conclude that
$\det(A^{\ast}-\bar{\lambda}I)=0$. Due to \eqref{detAast}, we have $\det(A-\lambda
I)=0$, i.e., $C_A(\lambda)=0$. \qed}

\medskip
{\small Lemma~\ref{18.2} explains why we do not introduce a notion of
{\em left eigenvalue}: the set of eigenvalues for left eigenvectors is
just the same as the set of eigenvalues for ordinary (right)
eigenvectors.

Our next question is: for a given eigenvalue $\lambda$ of $A$, how different the corresponding left and right eigenvectors are? More precisely, how different the corresponding left and right \emph{eigenspaces} are? }

\subsection{Lemma} \label{18.3}

For any eigenvalue $\lambda$ of $A$, the dimension of the right eigenspace equals the dimension of the left eigenspace (i.e.,
the \emph{geometric multiplicity} of $\lambda$ is the same, in the left
and right senses).

{\footnotesize \proof Recall that the ordinary (right) eigenspace is Ker$(A-\lambda I)$. By \eqref{RLevectors}, the left eigenspace is Ker$(A^{\ast} -\bar{\lambda}I)$. Now we have, due to \eqref{RKA}:
$$
  {\rm dim}\,{\rm Ker}(A-\lambda I)=n-\,{\rm rank}(A-\lambda I)=
  n-\, {\rm rank}(A^{\ast}-\bar{\lambda}I)=\, {\rm dim}\,{\rm
  Ker}(A^{\ast} -\bar{\lambda}I).
$$
We used the fact that rank$\,B=\,$rank$\,B^\ast$ for any matrix $B$. \qed}

\medskip
{\small Lemma~\ref{18.3} shows that for each eigenvalue $\lambda$ of $A$ the corresponding left and right eigenspaces have the same dimension. But do those spaces have to coincide? If not, do they have to be close to each other?

When both eigenspaces are one-dimensional and real, their closeness can be measured by the angle between them. If $x$ is a right unit eigenvector and $y$ is a left unit eigenvector, then the angle $\theta$ between the corresponding eigenspaces span$(x)$ and span$(y)$ satisfies $\cos\theta=\la x,y\ra$; cf.\ \eqref{Angletheta}. Thus the ``indicator of closeness'' of those eigenspaces will be the inner product $\la x,y\ra = y^\ast x$.

Recall that by Cauchy-Schwarz inequality (Section~\ref{8.16}) for any unit vectors $x$ and $y$ we have $|\la x,y\ra| \leq 1$, so the range of values of $|\la x,y\ra|$ is the interval $[0,1]$. The angle between the left and right eigenspaces span$(x)$ and span$(y)$ is small if $|\la x,y\ra|$ is close to $1$, and large if $|\la x,y\ra|$ is close to $0$. These spaces coincide if $|\la x,y\ra| = 1$ and are orthogonal to each other if $|\la x,y\ra| = 0$. }

\begin{center}
\begin{tikzpicture}
   \draw[blue,->,-latex] (0,0)--(1,0)
     node[right]{\footnotesize $x$};
   \draw[green,->,-latex] (0,0)--(0.96,0.2)
     node[right]{\footnotesize $y$};
   \node[anchor=west] at (-0.1,-0.7) {\footnotesize $\theta$ small};
   \node[anchor=west] at (-0.4,-1.4) {\footnotesize $|\la x,y\ra| \approx 1$};
   \begin{scope}[xshift=3cm]
   \draw[blue,->,-latex] (0,0)--(1,0)
     node[right]{\footnotesize $x$};
   \draw[green,->,-latex] (0,0)--(0.3,0.95)
     node[right]{\footnotesize $y$};
   \node[anchor=west] at (-0.1,-0.7) {\footnotesize $\theta$ large};
   \node[anchor=west] at (-0.4,-1.4) {\footnotesize $|\la x,y\ra| \approx 0$};
   \end{scope}
   \begin{scope}[xshift=6cm]
   \draw[blue,->,-latex] (0,0)--(1,0)
     node[right]{\footnotesize $x$};
   \draw[green,->,-latex] (0,0)--(0,1)
     node[right]{\footnotesize $y$};
   \node[anchor=west] at (-0.1,-0.7) {\footnotesize $\theta=\pi/2$};
   \node[anchor=west] at (-0.4,-1.4) {\footnotesize $|\la x,y\ra| = 0$};
   \end{scope}
   \begin{scope}[xshift=-3cm]
   \draw[green,->,-latex] (0,0.02)--(1,0.02)
     node[above]{\footnotesize $y$};
   \draw[blue,->,-latex] (0,0)--(1,0)
     node[below]{\footnotesize $x$};
   \node[anchor=west] at (-0.1,-0.7) {\footnotesize $\theta=0$};
   \node[anchor=west] at (-0.4,-1.4) {\footnotesize $|\la x,y\ra| = 1$};
   \end{scope}
\end{tikzpicture}
\end{center}

\subsection{Lemma} \label{18.13a}

Let $\lambda$ be an eigenvalue with a right eigenvector $x$, and $\mu\neq\lambda$ be \emph{another} eigenvalue with a left eigenvector $y$. Then $\la x,y\ra=0$, i.e.\ $x\perp y$.

\medskip{\small (In other words, left and right eigenspaces corresponding to \emph{different} eigenvalues are orthogonal to one another.)}

{\footnotesize \proof First, we have $\la Ax,y\ra=\lambda\la x,y\ra$. On the other hand, according to \eqref{RLevectors}, we have
$$
  \la Ax,y\ra=\la x,A^{\ast}y\ra= \la
  x,\bar{\mu}y\ra=\mu\la x,y\ra.
$$
Hence, $\lambda\la x,y\ra=\mu\la x,y\ra$. Since $\lambda\neq\mu$, we must have $\la x,y\ra=0$. \qed}

\newpage

\subsection{Lemma} \label{18.13b}

Let $\lambda$ be a \emph{simple} eigenvalue (this means its algebraic multiplicity is one) with right and left eigenvectors $x$ and $y$, respectively. Then $\la x,y\ra \neq 0$.

\medskip{\small (In other words, left and right eigenspaces corresponding to the \emph{same} simple eigenvalue $\lambda$ cannot be orthogonal to each other.)}

{\footnotesize \proof We can always choose \emph{unit} eigenvectors, in particular let $\|x\|=1$. By the Schur decomposition (Section~\ref{17.1}), there is a unitary matrix $Q$ such that
$$
       Q^{\ast}AQ=\left[\begin{array}{ll}
       \lambda & h^{\ast} \\ 0 & B
       \end{array}\right]
$$
with some $h\in\IC^{n-1}$ and $B\in\IC^{(n-1)\times(n-1)}$. According to the proof in Section~\ref{17.1}, we can construct the matrix $Q$ so that its first column is the vector $x$, i.e.,
that $Qe_1=x$. Since $\lambda$ is a simple eigenvalue of $A$, it \emph{cannot} be
an eigenvalue of $B$. Thus the matrix $\lambda I-B$ is invertible. Hence $\bar{\lambda}I-B^{\ast}$ is invertible, too. Denote $z=
(\bar{\lambda}I-B^{\ast})^{-1}h$. Then
$$
     \bar{\lambda}z-B^{\ast}z=h
     \quad \Longrightarrow \quad
     h^{\ast}+z^{\ast}B=\lambda z^{\ast}.
$$
Now one can readily verify that
$$
     [\,1\,z^{\ast}]\,Q^{\ast}AQ=\lambda\, [\,1\,z^{\ast}]
     \quad \Longrightarrow \quad
     [\,1\,z^{\ast}]\,Q^{\ast}A=\lambda\, [\,1\,z^{\ast}]\,Q^{\ast}.
$$
Denote $w^{\ast}=[\,1\,z^{\ast}]\, Q^{\ast}$. The above equation now
takes a short form
$$
      w^{\ast}A=\lambda w^{\ast}.
$$
Hence $w$ is a left eigenvector of $A$. By the simplicity of
$\lambda$, the vector $w$ is a nonzero multiple of $y$. However,
observe that
$$
       w^{\ast}x=[\,1\,z^{\ast}]\,Q^{\ast}Qe_1=
       [\,1\,z^{\ast}]\,e_1=1\ (\neq 0),
$$
which proves the lemma. \qed}

\subsection{Theorem} \label{18.14}

Let $A\in\IC^{n\times n}$ have a simple eigenvalue $\lambda$ with right
and left unit eigenvectors $x$ and $y$, respectively. Let
$E\in\IC^{n\times n}$ be another matrix such that $\|E\|_2=1$. For small $\varepsilon$,
denote by $\lambda(\varepsilon)$, $x(\varepsilon)$, and $y(\varepsilon)$
the eigenvalue, the right and left unit eigenvectors of the matrix
$A+\varepsilon E$ that correspond to $\lambda$, $x$ and $y$ when $\varepsilon=0$. Then
$$
       |\lambda'(0)|\leq\frac{1}{|y^{\ast}x|}
$$

{\footnotesize \proof We note that the existence and differentiability of $\lambda(\varepsilon)$, $x(\varepsilon)$, and $y(\varepsilon)$ follow from the inverse function theorem; we leave out the details. Now we write the equation
$$
     (A+\varepsilon E)\, x(\varepsilon)=
     \lambda(\varepsilon)\, x(\varepsilon)
$$
and differentiate it with respect to $\varepsilon$:
 $$
    Ax'(\varepsilon)+Ex(\varepsilon)+\varepsilon Ex'(\varepsilon)=\lambda'(\varepsilon)x+\lambda\, x'(\varepsilon)
$$
Setting $\varepsilon=0$ gives
$$
    Ax'(0)+Ex=\lambda'(0)x+\lambda\, x'(0).
$$
Now we premultiply this equation by the vector
$y^{\ast}$:
$$
    y^{\ast}Ax'(0)+y^{\ast}Ex=\lambda'(0)y^{\ast}x+\lambda\, y^{\ast}x'(0).
$$
Since $y^{\ast}A=\lambda y^{\ast}$, the two terms with $x'(0)$ cancel each other out, and we arrive at
$$
       y^{\ast}Ex=\lambda'(0)\, y^{\ast}x.
$$
Now we use the Cauchy-Schwarz inequality and other standard inequalities:
$$
   |\lambda'(0)|\, |y^{\ast}x|=|y^{\ast}Ex|=|\la Ex,y \ra|\leq
   \|Ex\|_2\, \|y\|_2 \leq \|E\|_2\, \|x\|_2\, \|y\|_2 = 1.
$$
This proves the theorem. Note that $y^{\ast}x\neq 0$ by
Lemma~\ref{18.13b}. \qed}

\medskip{\small We regard the matrix $A+\varepsilon E$ as the perturbation of $A$ ``in the direction'' of $E$. If the matrix $E$ is
known, one can find $\lambda'(0)$ exactly:
$$
       \lambda'(0) =\frac{y^{\ast}Ex}{y^{\ast}x}.
$$
Knowing $\lambda'(0)$ would be useful, because by Taylor expansion
$$
        \lambda(\varepsilon)=\lambda+\frac{y^{\ast}Ex}{y^{\ast}x}\varepsilon
           +O(\varepsilon^2)
$$
we could have a fairly good approximation to $\lambda(\varepsilon)$ for small $\varepsilon$. In practice, however, the perturbation matrix $E$ is totally unknown, we only know the order of magnitude of perturbation, i.e., the value of $\varepsilon$. Hence we cannot compute an approximation to $\lambda(\varepsilon)$ by Taylor expansion, we can only use the upper bound of Theorem~\ref{18.14} to estimate how far $\lambda(\varepsilon)$ can be from $\lambda$:
$$
  |\lambda(\varepsilon)-\lambda(0)| \leq \frac{\varepsilon}{|y^{\ast}x|} + O(\varepsilon^2).
$$
This shows that the \emph{sensitivity} of the eigenvalue $\lambda$ to small perturbations of the matrix $A$ is determined by the value of $\frac{1}{|y^{\ast}x|}$.}

\subsection{Condition number of a simple eigenvalue}
\label{18.15}

Let $\lambda$ be a simple eigenvalue (this means that its algebraic
multiplicity is equal to one) of a matrix $A\in\IC^{n\times n}$ and $x,y$ the
corresponding right and left unit eigenvectors. Then
$$
      K(\lambda)=\frac{1}{|y^{\ast}x|}
$$
is called the {\em condition
number} of the eigenvalue $\lambda$.

\subsection{Remarks} \label{Klambda2}

\begin{itemize}\item[(a)] The condition number $K(\lambda)$ is well defined for every simple eigenvalue. Indeed, by
Lemma~\ref{18.13b} we have $|y^{\ast}x| \neq 0$. Also, the value of $|y^{\ast}x|$ does not depend on the particular choice of $x$ and $y$,  because $|(ay)^{\ast}(bx)|=|ab|\, |y^{\ast}x|=|y^{\ast}x|$ for any scalars $a$ and $b$ such that $|a|=|b|=1$. \item[(b)] If $\lambda$ is a multiple eigenvalue, the condition number $K(\lambda)$ is \emph{not} defined, and for a good reason. If the geometric
multiplicity of $\lambda$ is $\geq 2$, then for \emph{any} right eigenvector $x$ there is a left eigenvector $y$ such that $y^{\ast}x=0$ (Exercise~\ref{ExEmul2}), which would result in $K(\lambda)=\infty$.
\item[(c)]
Even if the geometric
multiplicity of $\lambda$ equals one, we may have a pathological situation where $y^{\ast}x=0$. Example: $A=\left[\begin{smallmatrix} 0 & 1
\\ 0 & 0\end{smallmatrix}\right]$ with a multiple eigenvalue $\lambda=0$; here the right eigenvector is $x=\left[\begin{smallmatrix} 1
\\ 0\end{smallmatrix}\right]$ and the left eigenvector is $y=\left[\begin{smallmatrix} 0
\\ 1\end{smallmatrix}\right]$.
\end{itemize}

\subsection{Properties of condition numbers for simple eigenvalues} \label{18.16}

\begin{itemize}\item[(a)] $K(\lambda)\geq 1$. \item[(b)] If the matrix $A$ is normal, then
$K(\lambda)=1$ for all its simple eigenvalues.
\item[(c)] Conversely, if a matrix $A$ has all simple eigenvalues
    with $K(\lambda)=1$ for each of them, then $A$ is
    normal.\end{itemize}

\noindent{\footnotesize \emph{Proof}. By Cauchy-Schwarz inequality, $|y^{\ast}x|
\leq \|x\|_2\|y\|_2=1$. This implies (a). For the proofs of (b) and (c), see Exercises~\ref{ExEN1} and \ref{ExEN2}.}

\subsection{Relation to Schur decomposition} \label{18.16sss}

We see that matrices with the most well-conditioned eigenvalues (such that $K(\lambda)=1$ for all the eigenvalues) are normal matrices. On the other hand, normal matrices are characterized by the fact that their
Schur decomposition $Q^{\ast}AQ=T$ results in an upper-triangular  matrix $T$ that is actually diagonal (i.e., its off-diagonal components are zeros).
One can expect that if the Schur matrix $T$ is \emph{nearly diagonal} (i.e., its
off-diagonal components are small), then the eigenvalues of $A$ are
well-conditioned. On the contrary, if some off-diagonal components of $T$
are large, then at least some eigenvalues are ill-conditioned.

\subsection{Multiple and ill-conditioned eigenvalues} \label{18.16iii}

Remarks~\ref{Klambda2} (b) and (c) show that if we attempt to define the condition number for a multiple eigenvalue $\lambda$, we are bound to get $K(\lambda)=\infty$. Not surprisingly, if a simple eigenvalue $\lambda$ is ill-conditioned (i.e., $K(\lambda) \approx \infty$), then it is `nearly multiple'. The following theorem makes this statement precise.

\subsection{Theorem (without proof)} \label{18.17}

Let $\lambda$ be a simple eigenvalue. Then there is a matrix $E$ such that
$$
      \frac{\|E\|_2}{\|A\|_2}\leq\frac{1}{\sqrt{K(\lambda)^2-1}}
$$
and $\lambda$ is a \emph{multiple} eigenvalue of $A+E$.

({\footnotesize This implies that a simple but ill-conditioned eigenvalue $\lambda$ becomes multiple under a small perturbation of the matrix $A$.})

\medskip{\small We conclude this chapter by two important theorems (first and second Gershgorin theorems).

Recall that the eigenvalues of a diagonal matrix $D=\,$diag$\{d_1,\ldots,d_n\}$ are its diagonal components $d_1,\ldots,d_n$. Naturally, under small perturbations of $D$ its eigenvalues should remain close to $d_1,\ldots,d_n$. More precisely, if $E$ is a small matrix, then the eigenvalues of $A=D+E$ should be close to  $d_1,\ldots,d_n$.

But how close? This is the subject of the next two theorems.}

\newpage

\subsection{First Gershgorin theorem} \label{18.18}

Let $D=\,$diag$\{d_1,\ldots,d_n\}$ be a diagonal matrix and $E=(e_{ij})$ any matrix. Then
every eigenvalue of $A=D+E$ lies in at least one of the closed disks
$$
      {\mathcal D}_i=\Bigl\{z\in\IC\colon\, |z-d_i|\leq\sum_{j=1}^n|e_{ij}|\Bigr\}
$$

{\small The closed disks ${\mathcal D}_i$ are called {\em Gershgorin disks}.}

{\footnotesize \proof Let $\lambda$ be an eigenvalue of $A$ and $x=[x_1,\ldots,x_n]^T$ the corresponding eigenvector. Let $x_r$ be the largest (in absolute value) component of $x$, i.e.,
$$
   |x_r|=\max\{|x_1|,\ldots,|x_n|\}.
$$
Since $x\neq 0$, we have $x_r\neq 0$.

Next we use the equation $Ax=\lambda x$. The $r$-th components of these two vectors are
$$
  (Ax)_r=d_rx_r+\sum_{j=1}^n e_{rj}x_j
        \qquad\text{and}\qquad (\lambda x)_r =\lambda x_r
$$
thus
$$
  d_rx_r+\sum_{j=1}^n e_{rj}x_j
        =\lambda x_r.
$$
Dividing by $x_r$ gives
$$
  d_r+\sum_{j=1}^n e_{rj}u_j
        =\lambda
$$
where $u_j = x_j/x_r$. Note that $|u_j| \leq 1$ for every $j=1,\ldots,n$. Therefore
$$
      |\lambda-d_r|=\biggl|\sum_{j=1}^n e_{rj}u_j\biggr| \leq\sum_{j=1}^n|e_{rj}|\, |u_j|
                   \leq\sum_{j=1}^n|e_{rj}|
$$
The theorem is proved. \qed}

\medskip {\small Theorem~\ref{18.18} says that every eigenvalue $\lambda$ of $A$ belongs to \emph{one of} the Gershgorin disks. A common misconception is that every Gershgorin disk must contain an eigenvalue of $A$. This is \emph{not} always true. (Example: $A= \left[ \begin{smallmatrix} 0 & 4 \\ 1 & 0 \end{smallmatrix} \right]$.)

If the Gershgorin disks are \emph{disjoint}, i.e., ${\mathcal D}_i \cap {\mathcal D}_j = \emptyset$ for $i\neq j$, then indeed each ${\mathcal D}_i$ contains exactly one eigenvalue of $A$. However if the disks overlap, the eigenvalues may ``travel'' from one disk to another, so that some disks contain two or more eigenvalues while others contain none.

\begin{center}
\begin{tikzpicture}
         \fill[gray!50] (1,0) circle(0.3);
         \fill[gray!50] (-1,0) circle(0.4);
         \fill[gray!50] (0.2,-0.5) circle(0.3);
         \fill (1.1,0) circle(0.05);
         \fill (-1.1,0.2) circle(0.05);
         \fill (0.1,-0.4) circle(0.05);
         \draw[->,-latex] (-2,0) -- (2,0) node[above]{\footnotesize Re};
         \draw[->,-latex] (0,-1) -- (0,1) node[right]{\footnotesize Im};
         \begin{scope}[xshift=7cm]
         \fill[gray!50] (0.7,0) circle(0.5);
         \fill[gray!50] (-1,0) circle(0.5);
         \fill[gray!50] (-0.5,-0.2) circle(0.4);
         \fill[gray] (-0.55,0.18) arc(101:220:0.4);
         \fill[gray] (-0.55,0.18) arc(22:-66:0.5);
         \fill (1,0.2) circle(0.05);
         \fill (-1.1,0.2) circle(0.05);
         \fill (-0.95,-0.3) circle(0.05);
         \draw[->,-latex] (-2,0) -- (2,0) node[above]{\footnotesize Re};
         \draw[->,-latex] (0,-1) -- (0,1) node[right]{\footnotesize Im};
         \end{scope}
\end{tikzpicture}
\end{center}

The second Gershgorin theorem describes the distribution of the eigenvalues of $A$ between the disks more precisely.}


\subsection{Second Gershgorin theorem} \label{18.19}

Suppose the union of some $k$ Gershgorin disks ${\mathcal D}_{i_1}, \ldots, {\mathcal D}_{i_k}$ is disjoint from the union of the other $n-k$ Gershgorin disks, i.e.,
$$
   \bigl(\cup_{i\in\{i_1,\ldots,i_k\}}{\mathcal D}_{i}\bigr)\cap
   \bigl(\cup_{i\notin\{i_1,\ldots,i_k\}}{\mathcal D}_{i}\bigr)
   =\emptyset.
$$
Then there are exactly $k$ eigenvalues of $A$
(counting multiplicity) in the former union and $n-k$ in the latter.

\begin{center}
\begin{tikzpicture}
         \fill[blue!30] (-1,0) circle(0.8);
         \fill[blue!30] (-0.6,0.2) circle(0.6);
         \fill[green!20] (0.8,0.5) circle(0.4);
         \fill[green!20] (1,-0.2) circle(0.5);
         \fill[green!20] (2,0.2) circle(0.4);
         \fill[blue!60] (-1,0) circle(0.4);
         \fill[blue!60] (-0.6,0.2) circle(0.3);
         \fill[green!60] (0.8,0.5) circle(0.2);
         \fill[green!60] (1,-0.2) circle(0.25);
         \fill[green!60] (2,0.2) circle(0.2);
         \fill (-1,0) circle(0.05);
         \fill (-0.6,0.2) circle(0.05);
         \fill (0.8,0.5) circle(0.05);
         \fill (1,-0.2) circle(0.05);
         \fill (2,0.2) circle(0.05);
         \draw[->,-latex] (-2,0) -- (3,0) node[above]{\footnotesize Re};
         \draw[->,-latex] (0,-1) -- (0,1) node[right]{\footnotesize Im};
  \node[anchor=north] at (-1,-0.8) {\footnotesize $k=2$};
  \node[anchor=north] at (1.4,-0.8) {\footnotesize $n-k=3$};
\end{tikzpicture}
\end{center}

\vspace{-5mm}{\footnotesize \proof For brevity, denote for every $i=1,\ldots,n$
$$
         h_i=\sum_{j=1}^n|e_{ij}|
$$
and consider the matrix $A(s)=D+sE$ for $0\leq s\leq 1$. We will use (without proof) the following standard fact in complex analysis: the roots of a complex
polynomial change continuously with its coefficients. This fact implies that the
eigenvalues of the matrix $A(s)$ depend continuously on $s$.

The Gershgorin disks ${\mathcal D}_i(s)$ for the matrix $A(s)$ have  centers $d_1,\ldots,d_n$ (the same for all $0\leq s\leq 1$) and radii
$sh_1,\ldots,sh_n$ that grow as $s$ increases from 0 to 1. When $s=0$, each Gershgorin disk ${\mathcal D}_i(0)$ is just a point, $d_i$, which is an eigenvalue of the matrix $A(0)=D$. Thus the statement of the theorem is true for the matrix $A(0)$. For $0<s<1$, each Gersgorin disk ${\mathcal D}_i(s)$ of the matrix $A(s)$ lies inside the corresponding disk ${\mathcal D}_i={\mathcal D}_i(1)$ of the matrix $A=A(1)$. Therefore
$$
   \bigl(\cup_{i\in\{i_1,\ldots,i_k\}}{\mathcal D}_{i}(s)\bigr)\cap
   \bigl(\cup_{i\notin\{i_1,\ldots,i_k\}}{\mathcal D}_{i}(s)\bigr)
   =\emptyset.
$$
for all $0\leq s\leq 1$.
Due to the continuity, the eigenvalues of $A(s)$ cannot ``jump'' from one union to the other and vice versa. Therefore the statement of the theorem remains valid for the matrix $A(s)$ for all $0\leq s\leq 1$. Lastly recall that $A(1)=A$. \qed}

\newpage

\begin{exercise}
(JPE May, 1994). Let $X^{-1}AX=D$, where $D$ is a diagonal
matrix.\begin{itemize} \item[(i)] Show that the columns of $X$ are
right eigenvectors and the conjugate rows of $X^{-1}$ are left
eigenvectors of $A$.\item[(ii)] Let $\lambda_1\ldots,\lambda_n$ be the
eigenvalues of $A$. Show that there are right eigenvectors
$x_1,\ldots,x_n$ and left eigenvectors $y_1,\ldots,y_n$ such that
$$ A=\sum_{i=1}^n\lambda_ix_iy^{\ast}_i$$
\end{itemize}
\end{exercise}


\begin{exercise} \label{ExEN1}
Let $A\in\IC^{n\times n}$. Show that\begin{itemize} \item[(i)]
$\lambda$ is an eigenvalue of $A$ iff $\bar{\lambda}$ is an eigenvalue
of $A^{\ast}$.\item[(ii)] if $A$ is normal, then for each eigenvalue
the left and right eigenspaces coincide;\item[(iii)] if $A$ is normal,
then for any simple eigenvalue $\lambda$ of $A$ we have $K(\lambda)=1$.
\end{itemize}
\end{exercise}

\begin{exercise} \label{ExEN2}
Let $A\in\IC^{n\times n}$ and $B=Q^{\ast}AQ$, where $Q$ is a unitary
matrix. Show that if the left and right eigenspaces of $A$ are equal,
then $B$ enjoys the same property. After that show that $A$ is normal.
Finally, prove that if $A$ has all simple eigenvalues with
$K(\lambda)=1$, then $A$ is normal.
\end{exercise}

\begin{exercise} \label{ExEmul2}
Suppose $\lambda$ is an eigenvalue of geometric multiplicity $\geq 2$
for a matrix $A$. Show that for each right eigenvector $x$ there is a
left eigenvector $y$ such that $y^{\ast}x=0$.
\end{exercise}

\begin{exercise}
Use the Gershgorin theorems to show that a symmetric, strictly row
diagonally dominant real matrix with positive diagonal elements is
positive definite.
\end{exercise}

\begin{exercise}
[Bonus] Let $A\in\IC^{n\times n}$ be Hermitian with eigenvalues
$\lambda_1\leq\cdots\leq\lambda_n$. Let $\mu_1\leq\cdots\leq\mu_{n-1}$
be all the eigenvalues of the $(n-1)$-st principal minor $A_{n-1}$ of
$A$. Use the Minimax theorem to prove the {\em interlacing property}
$$\lambda_1\leq\mu_1\leq\lambda_2\leq\cdots\leq
\lambda_{n-1}\leq\mu_{n-1}\leq\lambda_n$$
\end{exercise}
\newpage

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\refstepcounter{section} \label{Ch16}

\centerline{Chapter \ref{Ch16} }\vspace{5mm}

\centerline{\Large\bf Computation of Eigenvalues: Power Method}\vspace{3mm}

\addcontentsline{toc}{section}{\ref{Ch16}. Computation of Eigenvalues: Power Method}

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\setcounter{subsection}{0} \setcounter{exercise}{0}
\setcounter{equation}{0}

We now turn to practical methods for computing eigenvalues of matrices. This chapter presents simple and straightforward \emph{power method}, and the next (and last) chapter -- a more complicated one, called \emph{QR algorithm}.

\subsection{Preface} \label{PMpreface}

Here is our basic task: given a matrix $A \in \IC^{n\times n}$, compute its eigenvalues (and eigenvectors). According to Chapter~\ref{Ch15}, there is no finite algorithm (no closed formula) for solving this problem. All solutions are necessarily iterative: we will just construct a sequence of numbers $\lambda^{(k)}$, where $k=1,2,\ldots$, that would converge to an eigenvalue $\lambda$ of $A$, i.e., $\lim_{k\to\infty} \lambda^{(k)} = \lambda$.

Also, our methods will not apply to \emph{all} matrices, they will only work under certain conditions. Fortunately, our methods apply to \emph{most} matrices. More precisely, our methods apply to \emph{typical}, \emph{generic} matrices; they only fail in exceptional cases (where some ``bad accident'' or some coincidence occurs). We will have to exclude such cases from our discussion.

First, we will always assume that the matrix $A$ is
\emph{diagonalizable}\footnote{Typical matrices are diagonalizable. In particular, matrices whose eigenvalues are \emph{distinct} are always diagonalizable. So we will only have to exclude \emph{some} matrices whose eigenvalues coincide (and coincidence can be regarded as an exceptional case).}. This assumption implies that there is a basis of eigenvectors. We will denote by $\lambda_1,\ldots,\lambda_n$ the eigenvalues of $A$ and the corresponding eigenvectors by $x_1,\ldots,x_n$, which are chosen so that $\{x_1,\ldots,x_n\}$ makes a basis in $\IC^n$.

We always number the eigenvalues so that their absolute values decrease:
$$
      |\lambda_1|\geq |\lambda_2|\geq\cdots\geq |\lambda_n|.
$$
In other words, $\lambda_1$ is the \emph{largest} eigenvalue and $\lambda_n$ is the \emph{smallest} one.

Second, we assume that $|\lambda_1|>|\lambda_2|$. This excludes a possible coincidence of $|\lambda_1|$ and $|\lambda_2|$ ({\footnotesize this coincidence is another exceptional case that we rule out of the discussion}). We call $\lambda_1$ the {\em dominant eigenvalue}. Our assumption implies that $\lambda_1$ is a simple eigenvalue and its eigenspace span$\{x_1\}$ is one-dimensional.

\subsection{Power method: a general scheme} \label{19.3}

Let $q\in\IC^n$ be any vector. Since there is basis $\{x_1,\ldots,x_n\}$ of eigenvectors of $A$, we can write
$$
       q=c_1x_1+\cdots +c_nx_n.
$$
Note that
\begin{align*}
    Aq&=c_1Ax_1+\cdots +c_nAx_n\\
    &=c_1\lambda_1x_1+\cdots +c_n\lambda_nx_n.
\end{align*}
By induction, for any $k\geq 1$ we have
$$
    A^kq=c_1\lambda_1^kx_1+\cdots +c_n\lambda_n^kx_n.
$$
Let us consider vector
\beq \label{q(k)}
   q^{(k)}=A^kq/\lambda_1^k
\eeq
i.e.,
\beq \label{q(k)Deltak}
       q^{(k)}=c_1x_1+
     \underbrace{c_2(\lambda_2/\lambda_1)^kx_2+
                \cdots +c_n(\lambda_n/\lambda_1)^kx_n}_{\Delta_k}.
\eeq
The first term $c_1x_1$ is fixed (it does not change with $k$), and the other terms converge to zero, because $|\lambda_i/\lambda_1| <1$ for all $i=2,\ldots,n$. Therefore
\beq \label{19.4}
   \lim_{k\to\infty} q^{(k)} = c_1x_1.
\eeq
Moreover, the ``remainder'' $\Delta_k = q^{(k)} - c_1x_1$ satisfies
\beq \label{19.4a}
       \|\Delta_k\|=\|q^{(k)}-c_1x_1\|\leq\,{\rm const}\cdot r^k
\eeq
where
$$
   r=|\lambda_2/\lambda_1|<1.
$$
The number $r$ characterizes the speed of convergence $\|\Delta_k\|\to 0$. More precisely,
$$
      \|\Delta^{(k)}\|/\|\Delta^{(k-1)}\|\to r
      \qquad\text{as}\ \ k\to\infty
$$
(assuming $c_2 \neq 0$); thus each ``error'' $\|\Delta^{(k)}\|$ is approximately $r$ times the previous ``error'' $\|\Delta^{(k-1)}\|$, i.e., the ``errors'' decrease as a geometric progression with ratio $r$. For this reason $r$ is called the {\em convergence ratio} or the {\em contraction number}. The smaller $r$, the faster the convergence in \eqref{19.4}.

\subsection{Linear, quadratic and cubic convergence}
\label{19.5}

Let $a_k\in\IC$ be a sequence of complex numbers such that $a_k\to a$ as $k\to\infty$. We say that the convergence $a_k\to a$ is {\em linear} if
$$
     |a_{k+1}-a| \approx r|a_k-a|
$$
for some $0<r<1$ and all sufficiently large $k$. This means that the distance from the current value $a_k$ to the limit (``target'') value $a$ decreases (shrinks) by a factor of $r$, i.e., as a geometric progression. The convergence in \eqref{19.4} is linear, with $r=|\lambda_2/\lambda_1|$.
If
$$
     |a_{k+1}-a| \approx C|a_k-a|^p
$$
with some constant $C>0$ and power $p>1$, then the convergence is said to be {\em
superlinear}; it is faster than linear. For $p=2$ the convergence is said to be {\em quadratic}, and for $p=3$ it is {\em cubic}.

\subsection{Remarks on convergence} \label{19.6}

{\footnotesize If the convergence is linear, then by induction we have
$$
   |a_k-a|\approx |a_0-a|\, r^k.
$$
We see that the ``remainder'' $|a_k-a|$ decreases as $r^k$, i.e.,
exponentially fast, which is very fast by calculus standards. But in numerical calculations standards are different, and such a convergence is \emph{slow}; see below.

Suppose, for example, $r=0.5$. Then
$$
  \frac{|a_k-a|}{|a_0-a|} \approx \frac{1}{2^k}.
$$
If $a_k$ denotes the computed approximation to an unknown number $a$, then its relative error is $\sim 1/2^k$. This means that the first $k$ binary digits in $a$ and $a_k$ coincide, i.e., we can trust $k$ binary digits in $a_k$.
Therefore each iteration adds one accurate binary digits (one bit) to the result. To compute $a$ accurately in single precision, one would need about 22--23 iterations, and in double precision -- about 52--53 iterations. This is too much for practical applications. And we picked a fairly small ratio $r=0.5$. Now imagine how long the process might take if $r=0.9$ or $r=0.99$.

Now suppose the convergence is quadratic. Then, if the error at the current iteration is $\frac{1}{2^m}$, it will be $\sim \frac{1}{2^{2m}}$ at the next iteration. Hence if the current approximation carries $m$ accurate binary digits, the next one will carry $2m$. Thus each
iteration doubles (!) the number of accurate digits. Starting
with just one accurate binary digit, one gets 2, 4, 8, 16, 32 digits at the following iterations. So one needs 4--5 iterations in
single precision and only one more iteration in double
precision. This is fast enough for practical applications.

Similarly, the cubic convergence means that each iteration triples
(!!!) the number of accurate digits. See Example~\ref{19.16}
for an illustration.}

\subsection{Scaling problem in the power method} \label{19.7}

{\small Back to the power method: note that the limit vector $c_1x_1$ in \eqref{19.4} is an eigenvector of $A$ corresponding to $\lambda_1$ (unless $c_1=0$). Thus we found a sequence of vectors converging to an eigenvector of $A$.

However, the vector $q^{(k)}=A^kq/\lambda_1^k$ cannot be computed in practice, because we do not know $\lambda_1$ in advance (in fact, computing $\lambda_1$ is our ultimate goal). On the other hand, we cannot just drop the factor $1/\lambda_1^k$ and use the sequence of vectors $A^kq$. Indeed, $\|A^kq\|\to\infty$ if
$|\lambda_1|>1$ and $\|A^kq\|\to 0$ if $|\lambda_1|<1$, thus
our vectors would either grow too large or get too small. This would cause a breakdown (overflow or underflow) in computations. We
must somehow normalize, or scale, the vector $A^kq$. To this end we will define a new sequence of vectors recursively: $q_0=q$ and
\beq \label{qkqk-1}
        q_k=Aq_{k-1}/\sigma_k
\eeq
for $k\geq 1$, where $\sigma_k$ is a properly chosen scaling factor; see below. We will also define an approximation $\lambda_1^{(k)}$ to the eigenvalue $\lambda_1$ by using the vector $q_k$. }

\subsection{First choice for the scaling factor} \label{19.8a}

There are two standard choices for $\sigma_k$. One is
\beq \label{sigmak1}
   \sigma_k=\|Aq_{k-1}\|_2.
\eeq
This ensures $\|q_{k}\|_2=1$, i.e., all our $q_k$'s will be unit vectors. Thus they will not grow too large or too small.

Now one can approximate the eigenvalue $\lambda_1$ by the
Rayleigh quotient
\beq \label{lambda1kRQ}
      \lambda_1^{(k)}=q_k^{\ast}Aq_{k}.
\eeq
{\small We will show that $\lambda_1^{(k)}$ indeed converges to $\lambda_1$, as $k\to\infty$, and the vectors $q_k$ get closer and closer to the eigenspace span$\{x_1\}$ corresponding to $\lambda_1$. However, the sequence of vectors $q_k$ need not converge to any specific limit vector, as the following example shows.}

\subsection{Example} \label{19.11a}

{\footnotesize Let $A=\left[\begin{array}{rr} -1 & 0 \\ 0 &
0\end{array}\right]$. Let us choose $q_0=[1,1]^T$. It is easy to see that $q_k=[(-1)^k,0]^T$ and $\lambda_1^{(k)} = -1$ for all $k\geq 1$. Hence $\lambda_1^{(k)}$ coincides with the dominant eigenvalue $\lambda_1=-1$ of the matrix $A$, and $q_k$ is a corresponding eigenvector -- a perfect result. However, $q_k$ does not converge to a limit, it keeps oscillating (flipping back and forth between two opposite vectors $[1,0]^T$ and $[-1,0]^T$).}\medskip

\subsection{Example} \label{19.11aa}

{\small Here is a more interesting example. Let $$
  A=\left[\begin{array}{rr} -3 & 1 \\ 1 &
   1\end{array}\right].
$$
This matrix has eigenvalues\footnote{We will provide approximate numerical values for all our constants with four digits after the decimal point. The calculations were performed by MATLAB\textregistered.}
\begin{align*}
  \lambda_1&=-1-\sqrt{5}\approx -3.2361\\
  \lambda_2&=-1+\sqrt{5}\approx +1.2361
\end{align*}
Note that $\lambda_1$ is the dominant eigenvalue because $|\lambda_1| > |\lambda_2|$. The corresponding eigenvector is $x_1=[0.9732,-0.2298]^T$.

Let us pick $q_0=[1,0]^T$ and use \eqref{sigmak1} for the scaling factor and \eqref{lambda1kRQ} for the approximation to $\lambda_1$. Then $Aq_0=[-3,1]^T$, so $q_1=[-0.9487,0.3162]^T$ and $\lambda_1^{(1)}=-3.2$.

At the next iteration, $Aq_1=[3.1623, -0.6325]^T$, so $q_2= [0.9806,-0.1961]^T$ and $\lambda_1^{(1)}=-3.2308$. The table below summarizes the first five iterations.

Here
$\lambda_1^{(k)}$ converges to the dominant eigenvalue
$\lambda_1$
and $q_k$ gets closer and closer to the eigenspace span$\{x_1\}$. But
$q_k$ does not have a limit -- it keeps oscillating between two limit vectors: $x_1$ and $-x_1$.}

\begin{center}
{\footnotesize
\begin{tabular}{cccccc}
   & $q_{k-1}$ & $Aq_{k-1}$ & $\sigma_k$ & $q_k=Aq_{k-1}/\sigma_k$ & $\lambda_1^{(k)}$ \\
  \hline
 & & & & & \\
  $k=1$ & $[+1.0000,+0.0000]$ & $[-3.0000,+1.0000]$ & $3.1623$ & $[-0.9487,+0.3162]$ & $-3.2000$\\
  $k=2$ & $[-0.9487,+0.3162]$ & $[+3.1623,-0.6325]$ & $3.2249$ & $[+0.9806,-0.1961]$ & $-3.2308$\\
  $k=3$ & $[+0.9806,-0.1961]$ & $[-3.1379,+0.7845]$ & $3.2344$ & $[-0.9701,-0.2425]$ & $-3.2353$\\
  $k=4$ & $[-0.9701,-0.2425]$ & $[+3.1530,-0.7276]$ & $3.2358$ & $[+0.9744,-0.2249]$ & $-3.2360$\\
  $k=5$ & $[+0.9744,-0.2249]$ & $[-3.1480,+0.7495]$ & $3.2360$ & $[-0.9728,+0.2316]$ & $-3.2361$\\
 & $\downarrow$ & & & $\downarrow$ & $\downarrow$ \\
 Limit: & no limit & & & no limit & $-3.2361$ \\
   & (oscillates) & & & (oscillates) & \\ \hline
\end{tabular}
}
\end{center}

\newpage To estimate how close the unit vector $q_k$ is to the
one-dimensional eigenspace span$\{x_1\}$, denote by $p_{k}$ the
orthogonal projection of $q_{k}$ on span$\{x_1\}$ and by
$d_{k}=q_{k}-p_{k}$ the orthogonal component. Then $\|d_{k}\|$
measures the distance from $q_{k}$ to span$\{x_1\}$.

\subsection{Theorem (convergence of the power method)} \label{19.9}

Assume the following:
\begin{itemize}
\item[(i)] The matrix $A$ is diagonalizable, as described in Section~\ref{PMpreface}.
    \item[(ii)] $\lambda_1$ is the dominant eigenvalue, as described in Section~\ref{PMpreface}. \item[(iii)] $q_{0}=
c_1x_1+\cdots+c_nx_n$ is chosen so that $c_1\neq 0$. \end{itemize}
Then the distance from
$q_{k}$ to the eigenspace ${\rm span}\{x_1\}$ converges to zero and
$\lambda_1^{(k)}$ converges to $\lambda_1$. Furthermore,
\beq \label{dkrk}
  \|d_{k}\|\leq {\rm const}\cdot r^k
\eeq
and
\beq \label{lambda1klambda1}
  |\lambda_1^{(k)}-\lambda_1|\leq {\rm const}\cdot r^k
\eeq
where $r=|\lambda_2/\lambda_1|<1$.

{\footnotesize \proof By induction, we have
$$
  q_k = \frac{Aq_{k-1}}{\sigma_k}  = \frac{A^2q_{k-2}}{\sigma_{k-1}\sigma_k} = \cdots
  = \frac{A^kq_0}{\sigma_1\cdots\sigma_k}
$$
According to \eqref{q(k)}--\eqref{q(k)Deltak} we have
$$
   q_k = \frac{\lambda_1^k}{\sigma_1\cdots\sigma_k}\,q^{(k)}
    = \frac{\lambda_1^k}{\sigma_1\cdots\sigma_k}\,(c_1x_1+\Delta_k)
$$
i.e., $q_k$ is proportional to $c_1x_1+\Delta_k$.
Note that $c_1x_1$ is a fixed non-zero vector, $\|\Delta_k\|\to 0$ by \eqref{19.4a}. Also, $\|q_k\|_2=1$ by construction, and $\|c_1x_1+\Delta_k\|\approx \|c_1x_1\|>0$, so for all large enough $k$ we have $\frac{1}{2}\,\|c_1x_1\| < \|c_1x_1+\Delta_k\| < 2\|c_1x_1\|$. Therefore the scalar $\frac{\lambda_1^k}{\sigma_1\cdots\sigma_k}$ cannot get too large or too small, i.e.,
$$
   c_1 < \biggl|\frac{\lambda_1^k}{\sigma_1\cdots\sigma_k}\biggr| <c_2
$$
for some positive constants $0<c_1<c_2<\infty$. Now the claim \eqref{dkrk} can be easily derived from \eqref{19.4a} (we leave this as an exercise). The last claim \eqref{lambda1klambda1} follows from \eqref{dkrk} and \eqref{lambdar1}. \qed
}

\subsection{Second choice for the scaling factor} \label{19.8b}

The other popular choice for $\sigma_k$ is the largest (in absolute
value) component of the vector $Aq_{k-1}$, i.e.,
\beq \label{sigmak2}
  \sigma_k = (Aq_{k-1})_m \qquad
  \text{so that}\quad |(Aq_{k-1})_m| = \max_i |(Aq_{k-1})_i|.
\eeq
According to \eqref{qkqk-1}, the
largest (in absolute value) component of $q_k$ will be now equal to one, i.e.,
\beq \label{qkm=1}
  (q_k)_m = 1\qquad
  \text{and }\ \ \max_i|(q_{k})_i| \leq 1
\eeq
This implies $1\leq \|q_k\|_2 \leq \sqrt{n}$, so again our vectors $q_k$ will not grow too large or get too small. As in Section~\ref{19.8a}, one can approximate the eigenvalue $\lambda_1$ by the Rayleigh quotient
\beq \label{lambda1kRQ}
      \lambda_1^{(k)}=\frac{q_k^{\ast}Aq_{k}}{q_k^{\ast}q_{k}}.
\eeq
Theorem~\ref{19.9} applies to this new procedure word for word. Its proof only needs one minor change: instead of $\|q_k\|_2=1$ we now have $1\leq \|q_k\|_2 \leq \sqrt{n}$, but this change does not affect the validity of the argument.

Since for large $k$ the vector $q_k$ will be almost an eigenvector of $A$,  we will get $Aq_k \approx \lambda_1 q_k$. Therefore the largest component of $Aq_k$ will be $\approx \lambda_1$. On the other hand, the largest component of $Aq_k$ will be our $\sigma_{k+1}$, according to \eqref{sigmak2}. Thus
the scaling factor itself can be used as an approximation to $\lambda_1$, i.e., we can set
\beq \label{lambda1ksigma}
       \tilde{\lambda}_1^{(k)} = \sigma_k.
\eeq
This approximation to $\lambda_1$ works well in typical cases. More precisely, it works if the eigenvector $x_1$ has \emph{one} component with the largest absolute value, i.e.,
\beq \label{x1m}
    \exists m\in\{1,\ldots,n\} \colon \max_{i\neq m} |(x_1)_i| < |(x_1)_m|.
\eeq
Then if $k$ is large enough, $q_k$ will be almost proportional to $x_1$, hence the largest component of $q_k$ will be $(q_k)_m$, where $m$ is the same as in \eqref{x1m}, i.e.,
$$
    \max_{i\neq m} |(q_k)_i| < |(q_k)_m|.
$$
Thus the index $m$ in \eqref{sigmak2}--\eqref{qkm=1} will be the same for all large $k$, and therefore \eqref{lambda1ksigma} will give a good approximation to $\lambda_1^{(k)}$; see the next example.

\subsection{Example} \label{19.11}

{\footnotesize We continue Example~\ref{19.11aa}. We work with the same matrix $A=\left[\begin{array}{rr} -3 & 1 \\ 1 &
1\end{array}\right]$ and pick the same initial vector $q_0=[1,0]^T$. But now we use \eqref{sigmak2} for the scaling factor and \eqref{lambda1ksigma} for the approximation to $\lambda_1$.

The table below summarizes the first nine iterations.
Here
$\tilde{\lambda}_1^{(k)}$ converges to the dominant eigenvalue
$$
   \lambda_1=-1-\sqrt{5}\approx -3.2361
$$
and $q_k$ converges to the dominant eigenvector
$$
  [1,2-\sqrt{5}]^T \approx [1.0000,-0.2361]^T.
$$}

\begin{center}
{\footnotesize
\begin{tabular}{ccccc}
   & $q_{k-1}$ & $Aq_{k-1}$ & $\sigma_k=\tilde{\lambda}_1^{(k)}$ & $q_k=Aq_{k-1}/\sigma_k$  \\
  \hline
 & & & & \\
  $k=1$ & $[+1.0000,+0.0000]$ & $[-3.0000,+1.0000]$ & $-3.0000$ & $[+1.0000,-0.3333]$ \\
  $k=2$ & $[+1.0000,-0.3333]$ & $[-3.3333,+0.6667]$ & $-3.3333$ & $[+1.0000,-0.2000]$ \\
  $k=3$ & $[+1.0000,-0.2000]$ & $[-3.2000,+0.8000]$ & $-3.2000$ & $[+1.0000,-0.2500]$ \\
  $k=4$ & $[+1.0000,-0.2500]$ & $[-3.2500,+0.7500]$ & $-3.2500$ & $[+1.0000,-0.2308]$ \\
  $k=5$ & $[+1.0000,-0.2308]$ & $[-3.2308,+0.7692]$ & $-3.2308$ & $[+1.0000,-0.2381]$ \\
  $k=6$ & $[+1.0000,-0.2381]$ & $[-3.2381,+0.7619]$ & $-3.2381$ & $[+1.0000,-0.2353]$ \\
  $k=7$ & $[+1.0000,-0.2353]$ & $[-3.2353,+0.7647]$ & $-3.2353$ & $[+1.0000,-0.2364]$ \\
  $k=8$ & $[+1.0000,-0.2364]$ & $[-3.2364,+0.7636]$ & $-3.2364$ & $[+1.0000,-0.2360]$ \\
  $k=9$ & $[+1.0000,-0.2360]$ & $[-3.2360,+0.7640]$ & $-3.2360$ & $[+1.0000,-0.2361]$ \\
 & $\downarrow$ & & $\downarrow$ & $\downarrow$ \\
 Limit: & $[+1.0000,-0.2361]$ & & $-3.2361$ & $[+1.0000,-0.2361]$  \\
\end{tabular}
}
\end{center}

\medskip The choice of the scaling factor \eqref{sigmak2} and the approximation \eqref{lambda1ksigma} have two attractive advantages:
\begin{itemize} \item[(a)] they are easier to compute than \eqref{sigmak1} and \eqref{lambda1kRQ} \item[(b)] the vector $q_k$ \emph{does} converge to a limit, as $k\to\infty$ {\footnotesize (unlike our Example~\ref{19.11a})}. \end{itemize}

Back to Example~\ref{19.11a}: if we use the scaling factor \eqref{sigmak2} and the approximation \eqref{lambda1ksigma}, instead of \eqref{sigmak1} and \eqref{lambda1kRQ}, then we will get the vector $q_k = [1,0]$ for all $k\geq 1$, so there will be no oscillations.\bigskip

The scaling factor \eqref{sigmak2} and the approximation \eqref{lambda1ksigma} work well in typical cases, where the dominant eigenvector $x_1$ has \emph{one} component with the largest absolute value. However, if the eigenvector $x_1$ has \emph{more than one} largest component, things can go wrong; see the next example. ({\footnotesize Incidentally, our example shows that the JPE problem \#8 in Fall 2012 is stated incorrectly.})

\newpage

\subsection{Example}

{\footnotesize Let $A=\left[\begin{array}{rr} 1 & -2\\ -2 & 1\end{array}\right]$. This is a symmetric matrix with eigenvalues $\lambda_1=3$ and $\lambda_2=-1$; the corresponding eigenvectors are $x_1=[1,-1]^T$ and $x_2=[1,1]^T$. Note that the two components of $x_1$ have equal absolute values. Let us choose $q_{0}=[1,0]^T$. Then $Aq_{0} = [1,-2]^T$, and our rules \eqref{sigmak2} and \eqref{lambda1ksigma} give
$$
       \tilde{\lambda}_1^{(1)} = \sigma_1 = -2
$$
At the next iteration we get
$$
    q_1=Aq_{0}/\sigma_1=[-0.5,1]^T
$$
and $Aq_1=[-2.5,2]$, hence
$$
       \tilde{\lambda}_1^{(2)} = \sigma_{2} = -2.5
$$
which even worse than $\tilde{\lambda}_1^{(1)}$ (it is farther away from $\lambda_1=3$). At the following iteration $q_{2}=[1,-0.8]^T$ and $Aq_{2}= [2.6,-2.8]^T$, so that
$$
       \tilde{\lambda}_1^{(3)} = \sigma_{3} = -2.8
$$
which is still worse than $\tilde{\lambda}_1^{(2)}$. In this example $\tilde{\lambda}_1^{(k)}$ converges to the wrong limit:
$$
  \tilde{\lambda}_1^{(k)} \to -3\ \ \text{as}\ \ k\to\infty,
  \qquad\text{while}\quad \lambda_1=3.
$$
The vectors $q_k$ get closer and closer to the eigenspace span$\{x_1\}$ corresponding to $\lambda_1$, but they do \emph{not} converge to any specific limit, they oscillate between two limit points: $[1,-1]^T$ and $[-1,1]^T$. }

\begin{center}
{\footnotesize
\begin{tabular}{ccccc}
 & & & & \\
   & $q_{k-1}$ & $Aq_{k-1}$ & $\sigma_k=\tilde{\lambda}_1^{(k)}$ & $q_k=Aq_{k-1}/\sigma_k$ \\
  \hline
  $k=1$ & $[+1.0000,+0.0000]$ & $[+1.0000,-2.0000]$ & $-2.0000$ & $[-0.5000,+1.0000]$ \\
  $k=2$ & $[-0.5000,+1.0000]$ & $[-2.5000,+2.0000]$ & $-2.5000$ & $[+1.0000,-0.8000]$ \\
  $k=3$ & $[+1.0000,-0.8000]$ & $[+2.6000,-2.8000]$ & $-2.8000$ & $[-0.9286,+1.0000]$ \\ $k=4$ & $[-0.9286,+1.0000]$ & $[-2.9286,+2.8571]$ & $-2.9286$ & $[+1.0000,-0.9756]$ \\
  $k=5$ & $[+1.0000,-0.9756]$ & $[+2.9512,-2.9756]$ & $-2.9756$ & $[-0.9918,+1.0000]$ \\
 & $\downarrow$ & & $\downarrow$ & $\downarrow$ \\
 Limit: & no limit & & $-3.0000$ & no limit \\
   & (oscillates) & & (wrong limit) & (oscillates) \\ \hline
\end{tabular}
}
\end{center}

\medskip
We should admit that the matrix $A$ in the above example is somewhat exceptional: in practice it rarely happens that the dominant eigenvector $x_1$ has two or more equally large components.

\subsection{Initial vector} \label{19.12}

{\small The initial vector $q_0$ only has to fulfill the
requirement $c_1\neq 0$. Unacceptable vectors (those with $c_1=0$) form a hyperplane in $\IC^n$ defined by
$$
   H_{\rm bad} = \,{\rm span}\,\{x_2,\dots,x_n\}
$$
A hyperplane is a ``tiny'' part of the space $\IC^n$, so if we pick a vector $q_0$ ``randomly'', it will most likely be \emph{not} in that bad hyperplane.

Even if we have a bad luck of choosing a vector $q_0 \in H_{\rm bad}$ accidentally, something else may come to the rescue. All the vectors $q_k$, $k\geq 1$, must lie in $H_{\rm bad}$, theoretically, but due to round-off errors the numerically computed vectors $q_k$ will very likely drift away from that hyperplane, and then they would approach the dominant eigenspace span$\{x_1\}$, as desired.

If that does not seem to be enough, we can apply the following strategy that guarantees an absolute safety. We should choose not just one, but $n$ different initial vectors that make a basis in $\IC^n$ (for instance, we can choose $e_1,\ldots,e_n$). Then we carry out the power method starting with each of these initial vectors, i.e., carry out the power method $n$ times, each time with a different initial vector, and run it until convergence. Since our initial vectors make a basis, one of them has to be away from the hyperplane $H_{\rm bad}$, so we are bound to converge to the dominant eigenvalue $\lambda_1$ at least once.}

\subsection{Aiming at the smallest eigenvalue}

\medskip{\small We spent so much effort on developing methods for computing the \emph{dominant} eigenvalue $\lambda_1$. But what about the other eigenvalues, $\lambda_2,\ldots,\lambda_n$? Our next goal is to compute the \emph{smallest} eigenvalue $\lambda_n$. Again, we need certain assumptions on the matrix $A$.

First, we keep our main assumption made in Section~\ref{PMpreface}: the matrix $A$ is diagonalizable. We denote its eigenvalues by  $\lambda_1,\ldots,\lambda_n$ and order them so that
$$
      |\lambda_1|\geq |\lambda_2|\geq\cdots\geq |\lambda_n|.
$$
We still denote by $x_1,\ldots,x_n$ corresponding eigenvectors that make a basis in $\IC^n$.

In addition, we now make two new assumptions:
\begin{itemize}
\item[(i)] The matrix $A$ is invertible (this implies $\lambda_n \neq 0$). \item[(ii)] The matrix $A$ has \emph{one} smallest eigenvalue, i.e., $|\lambda_n| < |\lambda_{n-1}|$.
\end{itemize}
Both new assumptions rule out certain exceptional matrices, so all typical matrices $A$ satisfy our assumptions. We also note that if assumption (i) fails, then $\lambda_n=0$, and we do not need an elaborate algorithm to compute it. }

\subsection{Inverse power method} \label{19.13}

The inverse matrix $A^{-1}$ has eigenvalues $\lambda_1^{-1},\ldots,\lambda_n^{-1}$
and the same corresponding eigenvectors $x_1,\ldots,
x_n$. Note that $|\lambda_1^{-1}|\leq\cdots\leq |\lambda_n^{-1}|$. Our assumption (ii) implies $|\lambda_n^{-1}| > |\lambda_{n-1}^{-1}|$, hence the matrix $A$ has \emph{one} dominant eigenvalues, $\lambda_n^{-1}$, with eigenvector $x_n$.

Thus the matrix $A^{-1}$ satisfies our assumptions made in Section~\ref{PMpreface}, so that one can apply the power method to $A^{-1}$. This gives, of course, its dominant eigenvalue, which is  $\lambda_n^{-1}$, and the corresponding eigenspace span$\{x_n\}$. The reciprocal of the former will be the smallest eigenvalue $\lambda_n$ of $A$.
This trick is called {\em inverse power method}, because it consists of the power method applied to the inverse matrix $A^{-1}$.

According to Theorem~\ref{19.9}, the inverse power method converges  linearly, with the ratio
\beq \label{rPMinv}
   r=|\lambda_{n-1}^{-1}/\lambda_{n}^{-1}| =|\lambda_n/\lambda_{n-1}|<1.
\eeq

\subsection{Practical implementation}

The inverse power method goes through the same steps as the power method does, except $A$ is replaced with $A^{-1}$. In particular, the recurrent formula \eqref{qkqk-1} now takes form
\beq \label{qkqk-1inv}
           q_k=A^{-1}q_{k-1}/\sigma_k.
\eeq
In manual calculations (on paper), one just computes $A^{-1}$ and uses it.

However, in numerical calculations done by computer, the construction of the inverse matrix $A^{-1}$ is too expensive and should be avoided. Fortunately, it can be avoided in the present situation. All we need is to compute vectors $A^{-1}q_0$, $A^{-1}q_1, \ldots$, according to \eqref{qkqk-1inv}. In order to compute the product $A^{-1}b$ for any given vector $b\in\IC^n$, one can just solve the system of linear equations $Ax=b$, and its solution will be the desired vector $x=A^{-1}b$. The system $Ax=b$ can be solved by the LU decomposition of the matrix $A$, as described in Chapter~\ref{Ch7}. This trick is routinely used in the computer implementation of the inverse power method.

\bigskip\noindent{\small Now we know how to compute the \underline{largest} and the
\underline{smallest} eigenvalues of a given matrix $A$. Now we turn to the task of
computing \underline{any} eigenvalue.}

\newpage

\subsection{Aiming at any eigenvalue}

Recall that for any eigenvalue $\lambda$ of $A$
with eigenvector $x$ we have $Ax=\lambda x$. Thus for any scalar $\rho\in\IC$ we have
$$
   (A-\rho I)x = (\lambda-\rho)x
$$
Therefore $\lambda-\rho$ is an eigenvalue of the matrix $A-\rho I$,
with the same eigenvector $x$.

Recall that, in our notation, the given matrix $A$ has eigenvalues $\lambda_1,\ldots,\lambda_n$ with eigenvectors $x_1,\ldots,x_n$. Now the matrix $A-\rho I$ has eigenvalues
$$
  \lambda_1-\rho,\lambda_2-\rho,\ldots,\lambda_n-\rho
$$
with the same eigenvectors $x_1,x_2,\ldots,x_n$. We see that the whole spectrum (the set of all eigenvalues) of $A$ is \emph{shifted} by subtracting a constant scalar $\rho$.

If we can compute an eigenvalue $\lambda'$ of the new matrix $A'=A-\rho I$, then $\lambda+\rho$ would be an eigenvalue of $A$. The corresponding eigenvector $x'$ of $A'=A-\rho I$ would be also an eigenvector of $A$.

Now suppose we want to compute an eigenvalue $\lambda_i$ of the given matrix $A$, which is neither the largest nor the smallest one. Then the power method or the inverse power method would not be able to find $\lambda_i$. But it is possible that for some scalar $\rho\in\IC$ and for the new matrix $A'=A-\rho I$, the corresponding eigenvalue $\lambda'=\lambda_i-\rho$ will be either the largest or the smallest one, i.e.,
\beq \label{rhomax}
  |\lambda_i-\rho|=\max_{1\leq j\leq n}|\lambda_j-\rho|
\eeq
or
\beq \label{rhomin}
  |\lambda_i-\rho|=\min_{1\leq j\leq n}|\lambda_j-\rho|.
\eeq
Then we can apply the power method, or the inverse power method, to $A'$, compute $\lambda'$, and lastly determine $\lambda_i = \lambda'+\rho$.

It remains to find a scalar $\rho$ so that either \eqref{rhomax} or \eqref{rhomin} holds. A moment's thought reveals that it may be possible to ensure \eqref{rhomax}, but there are examples where it is not possible. On the other hand, it is \emph{always possible} to ensure \eqref{rhomin} -- one just needs to choose a number $\rho$ which is \emph{closer} to the desired eigenvalue $\lambda_i$ than to any other eigenvalue $\lambda_j$.

In other words, we need just a rough approximation to the desired eigenvalue $\lambda_i$, just a number $\rho$ that is closer to  $\lambda_i$ than to any other eigenvalue $\lambda_j$. In most practical applications, this is not a difficult task.


\subsection{Power method with shift} \label{19.14}

Assume that $\rho$ is a rough approximation to an eigenvalue
$\lambda_i$ of $A$, i.e., $\rho$ is closer to  $\lambda_i$ than to any other eigenvalue $\lambda_j$ of $A$:
$$
  |\lambda_i-\rho| < |\lambda_j-\rho| \quad \forall j\neq i.
$$
Then the matrix $A'=A-\rho I$ will have the smallest
eigenvalue $\lambda'=\lambda_i-\rho$ with the eigenvector $x'=x_i$.

We can apply the inverse power method to $A'=A-\rho I$. It will give us the smallest eigenvalue $\lambda'$ and a corresponding eigenvector $x'$ of the matrix $A'$. Then we can determine $\lambda_i = \lambda'+\rho$ and $x_i=x'$, i.e., the desired eigenvalue and eigenvector of $A$. This trick is called the {\em power method with shift}, because it is based on shifting the eigenvalues of $A$ by a conveniently chosen scalar $\rho\in\IC$. That scalar is called the {\em shift}.

According to Theorem~\ref{19.9} and \eqref{rPMinv}, the power method with shift $\rho$ converges linearly, with the ratio
\beq \label{rPMshift}
     r=\frac{|\lambda_i-\rho|}{\min_{j\neq i}|\lambda_j-\rho|}<1.
\eeq

\subsection{Note on the speed of convergence}

The speed of convergence of the power method depends on how big or small the ratio $r$ is; see Section~\ref{19.6}. If $r$ is small (close to zero), the convergence is actually pretty fast\footnote{For example, if $r=0.1$, then each iteration adds one accurate \emph{decimal} digit; hence, in single precision, 6--7 iterations might give us maximum accuracy. If $r=0.01$, then each iteration adds \emph{two} accurate decimal digits, i.e., 6--7 iterations might suffice in double precision.}. So it is in our best interests to select the scalar $\rho$ so that $r$ will be small.  Fortunately, this is often possible.

The formula \eqref{rPMshift} suggests that if we keep changing $\rho$ so that $\rho\to\lambda_i$, then the numerator and denominator of \eqref{rPMshift} have the following limits:
$$
   \left.\begin{array}{l} |\lambda_i-\rho|\to 0\\
    \min_{j\neq i}|\lambda_j-\rho|\to\min_{j\neq i} |\lambda_j-\lambda_i|>0 \end{array} \right\}\ \ \Longrightarrow\ \  r\to 0.
$$
Thus, the better $\rho$ approximates $\lambda_i$, the smaller $r$ is, and the faster the convergence will be. We will explore this idea in the next section.

\newpage

\subsection{Power method with Rayleigh quotient shift}
\label{19.15}

As it turns out, we do not have to work hard to provide a good approximation $\rho$ to $\lambda_i$ that would accelerate the convergence -- the power method itself can do this for us.

Recall that at each iteration we find a vector $q_k$ that gets closer and closer to the eigenspace span$\{x_i\}$ corresponding to the eigenvalue $\lambda_i$ of the matrix $A$. Thus the corresponding Rayleigh quotient $\frac{q_k^{\ast}Aq_k}{q_k^{\ast}q_k}$ would get closer and closer to $\lambda_i$. Why not to use \emph{this} Rayleigh quotient as the best available approximation to $\lambda_i$? In other words, at every iteration we can reset (update) the shift $\rho$ by using the Rayleigh quotient of the vector $q_k$ obtained at the previous iteration.

This suggests the following scheme. First, one chooses an initial vector $q_0$ and an initial shift $\rho_0$ (which does not have to be a very good approximation to $\lambda_i$). Then for each $k\geq 1$ one computes
\beq \label{qqPMshift}
     q_{k}=\frac{(A-\rho_{k-1}I)^{-1}q_{k-1}}{\sigma_k}
\eeq
where $\sigma_k$ is a convenient scaling factor, for example,
$\sigma_k=\|(A-\rho_{k-1}I)^{-1}q_{k-1}\|$, and one updates the shift by
$$
     \rho_{k}=\frac{q_k^{\ast}Aq_k}{q_k^{\ast}q_k}
$$
This brilliant scheme is known as \emph{power method with Rayleigh quotient shift}.

Its convergence is, generally,
quadratic (better than linear). If the matrix $A$ is Hermitian, the
convergence is even faster -- it is cubic!!!


\subsection{Example} \label{19.16}

{\footnotesize Consider the symmetric matrix
$$
   A=\left[\begin{array}{rrr} 2 & 1 & 1 \\
   1 & 3 & 1 \\ 1 & 1 & 4
   \end{array}\right]
$$
and let $q_0=(1,1,1)^T$ be the initial vector and $q_0=5$ the
initial shift. When power method with Rayleigh quotient shift is applied to $A$, the following values $\rho_k$ are computed at the first two iterations:
$$
   \rho_1=5.2131,\ \ \ \
   \rho_2=5.214319743184
$$
The actual value is $\lambda=5.214319743377$. After only two
iterations, the method produces 10 accurate digits.
The next iteration would bring about 30 accurate digits -- more than
enough for double precision arithmetic.}

\subsection{Power method: pros and cons} \label{19.17}

\begin{itemize} \item[(a)]
The power method is classic. Its logic is simple, and the method generally works well. The convergence is quadratic, and for Hermitian matrices the convergence is cubic.
\item[(b)]
The power method is still very slow for two reasons. The main one is that each iteration is rather expensive. Indeed, according to \eqref{qqPMshift} we need to compute $(A-\rho_{k-1}I)^{-1}q_{k-1}$, i.e., solve a system of linear equations $(A-\rho_{k-1}I)x=q_{k-1}$. This can be done by LU decomposition of the matrix $A-\rho_{k-1}I$, which keeps changing at every iteration (because of $\rho_{k-1}$). Hence the LU decomposition must be performed anew at every iteration, and each LU decomposition
takes $\approx 2n^3/3$ flops (Section~\ref{costLU}). Thus the cost of computing one eigenvalue is $\approx 2pn^3/3$, where $p$ is the number of iterations (typically, $p$ is about 5 or 10).

The other reason is that each eigenvalue must be computed independently -- we need to choose an initial vector $q_0$ and an initial shift $\rho_0$, and then run the inverse power method until its iterations converge. Thus, if we want to compute all $n$ eigenvalues of $A$, the total computational cost will be $2pn^4/3$ flops. This is quite a big number.
\item[(c)] Lastly, if the matrix $A$ is real (and this is the most common case in practice), then in order to
    compute its non-real complex eigenvalues $\lambda_i\notin\IR$ one has to use complex shifts $\rho_k\notin\IR$. Thus one has to deal with complex
    matrices $(A-\rho_{k-1}I)$, which is inconvenient and
    expensive.

    It would be nice to operate with real matrices only, for as
    long as possible, and obtain pairs of complex conjugate
    eigenvalues only at the very last step. The QR algorithm, to be
    discussed in the next chapter, provides such a luxury.
\end{itemize}


\begin{exercise}
(JPE, May 2003) Let $A$ be a symmetric matrix with eigenvalues such
that $|\lambda_1| >|\lambda_2| \geq\cdots\geq
|\lambda_{n-1}|>|\lambda_n|$. Suppose $z\in\IR^n$ with $z^Tx_1\neq 0$,
where $Ax_1=\lambda_1x_1$. Prove that, for some constant $C$,
$$
     \lim_{k\to\infty}\frac{A^kz}{\lambda_1^k} = Cx_1
$$
and use this result to devise a reliable algorithm for computing
$\lambda_1$ and $x_1$. Explain how the calculation should be modified
to obtain (a) $\lambda_n$ and (b) the eigenvalue closest to
2.
\end{exercise}

\begin{exercise}
(JPE, September 1996) The matrix
$$
  A=\left (\begin{array}{rrr} 2 & 0 & 0 \\
    0 & 1 & 2 \\ 0 & 2 & 1 \end{array}\right )
$$
has eigenpairs
$$
    (\lambda,x)=
     \left ( 2, \left [\begin{array}{r} 1\\ 0\\ 0\end{array}\right ]\right ),
     \left (-1, \left [\begin{array}{r} 0\\ 1\\-1\end{array}\right ]\right ),
     \left ( 3, \left [\begin{array}{r} 0\\ 1\\ 1\end{array}\right ]\right ),
$$
Suppose the power method is applied with starting vector
$$
     z_0=[1,1,-1]^t/\sqrt{3}
$$
\begin{itemize}
\item[(a)] Determine whether or not the iteration will converge to
    an eigenpair of $A$, and if so, which one. Assume exact
arithmetic.
\item[(b)] Repeat (a), except now use the inverse iteration with
    the same starting vector $z_0$ and the Rayleigh quotient of
    $z_0$ as approximation for the eigenvalue.
\item[(c)] Now answer both
(a) and (b) again, except this time use standard fixed precision
floating point arithmetic, i.e. computer arithmetic.
\end{itemize}
\end{exercise}

\newpage

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\refstepcounter{section} \label{Ch17}

\centerline{Chapter \ref{Ch17} }\vspace{5mm}

\centerline{\Large\bf Computation of Eigenvalues: QR Algorithm}\vspace{3mm}

\addcontentsline{toc}{section}{\ref{Ch16}. Computation of Eigenvalues: QR Algorithm}

\noindent\rule{14cm}{0.8pt}\vspace{5mm}

\setcounter{subsection}{0} \setcounter{exercise}{0}
\setcounter{equation}{0}

In this last chapter of the course we present the QR algorithm for computing eigenvalues of any matrix $A\in\IC^{n\times n}$. The QR algorithm (not to be confused with QR decomposition!) dates back
to the early 1960s, and in the recent decades it became the most widely
used method for calculating eigenvalues.

\subsection{``Pure'' QR algorithm} \label{19.18}

Let $A\in\IC^{n\times n}$. The algorithm starts with $A_0=A$ and
generates a sequence of matrices $A_k$ defined as follows:
\begin{align*}
         A_{k-1}&=Q_kR_k& &\text{(QR decomposition)}\\
         A_k&=R_kQ_k& &\text{(reverse multiplication)}
\end{align*}
{\small That is, a QR decomposition of the current matrix $A_{k-1}$ is computed first, and then its factors are multiplied together in the reverse order to produce the next matrix $A_k$. One iteration
of the QR algorithm is called {\em QR step}. It consists of two substeps: (i) QR decomposition and (ii) reverse multiplication.}

\subsection{Two basic facts} \label{19.19}

\medskip\noindent (i) All matrices $A_k$ in the QR algorithm are unitary equivalent.

\medskip{\footnotesize \emph{Proof}. Indeed, $A_{k}=R_kQ_k=Q_k^{-1}A_{k-1}Q_k$. \qed}

\medskip\noindent{\small  As a result, all matrices $A_k$ have the same eigenvalues. They also have the same 2-condition number, ensuring that the QR algorithm is numerically stable.}

\bigskip\noindent (ii) If $A$ is a
Hermitian matrix, then all $A_k$'s are Hermitian matrices as well.

\medskip{\footnotesize \emph{Proof}. This follows from (i) and Section~\ref{SAequiv}. \qed}

\bigskip\noindent{\small  The fact (b) ensures that the important Hermitian property is not lost during the QR procedure.}

{\small Note that $\|R_k\|_F = \|A_k\|_F = \|A\|_F$ for all $k\geq 1$ (Exercise~\ref{ExFFF}). Thus all our matrices $A_k$ and $R_k$ have the same \emph{mass} (or \emph{energy}); see Section~\ref{Energy}. At the first part of each QR step (the QR decomposition of the matrix $A_{k-1}$), the entire mass is pulled up to the upper triangle. At the second step (the reverse multiplication of $R_k$ and $Q_k$), the mass tends to stay on top of $A_k$, even though some of it ``leaks back'' to the bottom. This is illustrated by the following example, where for simplicity both $R_k$ and $Q_k$ are filled with identical numbers:}
{\footnotesize
$$
   \begin{bmatrix} 2 & 2 & 2 & 2\\
        0 & 2 & 2 & 2\\
        0 & 0 & 2 & 2\\
        0 & 0 & 0 & 2\end{bmatrix}\times
   \begin{bmatrix} 0.5 & 0.5 & 0.5 & 0.5\\
        0.5 & 0.5 & 0.5 & 0.5\\
        0.5 & 0.5 & 0.5 & 0.5\\
        0.5 & 0.5 & 0.5 & 0.5\end{bmatrix}=
   \begin{bmatrix} 4 & 4 & 4 & 4\\
        3 & 3 & 3 & 3\\
        2 & 2 & 2 & 2\\
        1 & 1 & 1 & 1\end{bmatrix}
$$
$$
    R_k \qquad\quad\times\qquad\qquad Q_k \qquad\qquad=\qquad\quad A_k
$$}
{\small As $k$ grows, successive repetitions of the QR step strengthen the effect of pulling the mass upward. These considerations make the following theorem plausible (though its formal proof is beyond the scope of the course):}


\subsection{Theorem (convergence of the QR algorithm)}
\label{19.20}

Let $\lambda_1,\ldots,\lambda_n$ be the eigenvalues of $A$
satisfying
$$
       |\lambda_1|>|\lambda_2|>\cdots >|\lambda_n|>0
$$
Under one technical condition\footnote{The technical condition is this: the matrix $Y$ whose $i$-th row is a left eigenvector of
$A$ corresponding to $\lambda_i$, for all $1\leq i\leq n$, must have an LU
decomposition (i.e., all its principal minors must be nonsingular; see Chapter~\ref{Ch7}).}, the matrix
$A_k=(a_{ij}^{(k)})$ is guaranteed to converge to an upper triangular
form, so that
\begin{itemize}\item[(a)] $a_{ij}^{(k)}\to 0$ as $k\to\infty$ for all
$i>j$. \item[(b)] $a_{ii}^{(k)}\to\lambda_i$ as $k\to\infty$ for all
$i$. \end{itemize} The convergence is linear, with the ratio
$$
     r=\max_k|\lambda_{k+1}/ \lambda_k|<1.
$$

Note that $a_{ij}^{(k)}$ for $i<j$ need not converge to a limit, as $k\to\infty$ (see example below). This is why we say that $A_k$ converges to ``an upper triangular form'', but not necessarily to any particular upper triangular matrix.

\subsection{Example}

Here we apply the QR algorithm to the matrix $A=\begin{bmatrix} 3 & 2\\ 1 & 1 \end{bmatrix}$. The calculations were performed with MATLAB\textregistered, and the QR decomposition was done by the standard MATLAB function \textbf{qr}.\medskip

\begin{center}
{\tiny
\begin{tabular}{ccccc}
   & $A_{k-1}$ & $Q_{k}$ & $R_k$ & $A_{k}=R_k Q_k$ \\
  \hline
 & & & & \\
  $k=1$ & $\begin{bmatrix} +3.0000 & +2.0000\\ +1.0000 & +1.0000 \end{bmatrix}$ & $\begin{bmatrix} -0.9487 & -0.3162 \\
   -0.3162  & +0.9487 \end{bmatrix}$ & $\begin{bmatrix} -3.1623 &  -2.2136 \\  0  &  +0.3162 \end{bmatrix}$ & $\begin{bmatrix} +3.7000   & -1.1000 \\   -0.1000 &  +0.3000 \end{bmatrix}$ \\
 & & & & \\
  $k=2$ & $\begin{bmatrix} +3.7000   & -1.1000 \\   -0.1000 &  +0.3000 \end{bmatrix}$ & $\begin{bmatrix} -0.9996  &  +0.0270 \\  +0.0270    & +0.9996 \end{bmatrix}$ & $\begin{bmatrix} -3.7014 & +1.1077 \\
  0 & +0.2702 \end{bmatrix}$ & $\begin{bmatrix} +3.7299 & +1.0073 \\
  +0.0073  & +0.2701 \end{bmatrix}$ \\
 & & & & \\
  $k=3$ & $\begin{bmatrix} +3.7299 & +1.0073 \\  +0.0073  & +0.2701 \end{bmatrix}$ & $\begin{bmatrix}  -1.0000 & -0.0020 \\
   -0.0020 & +1.0000 \end{bmatrix}$ & $\begin{bmatrix} -3.7299 &  -1.0078 \\ 0 & +0.2681 \end{bmatrix}$ & $\begin{bmatrix} +3.7319   & -1.0005 \\ -0.0005 & +0.2681 \end{bmatrix}$ \\
 & & & & \\
  $k=4$ & $\begin{bmatrix} +3.7319   & -1.0005 \\ -0.0005 & +0.2681 \end{bmatrix}$ & $\begin{bmatrix}  -1.0000 & +0.0001 \\
   +0.0001 & +1.0000 \end{bmatrix}$ & $\begin{bmatrix} -3.7319    & +1.0006 \\  0 & +0.2680 \end{bmatrix}$ & $\begin{bmatrix} +3.7320   & +1.0000 \\ +0.0000 & +0.2680 \end{bmatrix}$ \\
 & & & & \\
  $k=5$ & $\begin{bmatrix} +3.7320   & +1.0000 \\ +0.0000 & +0.2680 \end{bmatrix}$ & $\begin{bmatrix}  -1.0000 & -0.0000 \\
   -0.0000 & +1.0000 \end{bmatrix}$ & $\begin{bmatrix} -3.7320    & -1.0000 \\  0 & +0.2679 \end{bmatrix}$ & $\begin{bmatrix} +3.7321   & -1.0000 \\ -0.0000 & +0.2679 \end{bmatrix}$
\end{tabular}
}
\end{center}

\medskip The matrix $A_k$ converges to an upper triangular form: its diagonal components converge to the eigenvalues of $A$:
$$
   \lambda_1=2+\sqrt{3}\approx 3.7321,\qquad
   \lambda_2=2-\sqrt{3}\approx 0.2679,
$$
its component below the diagonal converges to zero, and its component above the diagonal oscillates between two limit points: $+1$ and $-1$.

\subsection{Advantages of the QR algorithm} \label{19.21}

\begin{itemize}\item[(a)]
The QR algorithm produces \emph{all} eigenvalues of $A$ in ``one go'', by a single sequence of iterations. By contrast, the power method (Chapter~\ref{Ch16}) requires a separate sequence of iterations for each eigenvalue.
\item[(b)] The QR algorithm does \emph{not} require an initialization -- there is no need for an arbitrary choice of an initial vector $q_0$ or an initial shift $\rho_0$ that the power method relies upon.
\item[(c)] An important case is that of a Hermitian matrix $A$. We know that all the $A_k$'s are Hermitian matrices, too. Since the lower triangle of $A_k$ converges to zero, so does its mirror image -- the upper triangle. Therefore $A_k$ converges to a diagonal matrix, a very nice result.
\end{itemize}


{\small Despite all the obvious advantages, the ``pure'' QR algorithm is still quite expensive. Mainly, each iteration is very costly: the QR decomposition takes $2n^3$ flops (Sect.~\ref{QR7}), and the multiplication of two $n\times n$ matrices $R_k$ and $Q_k$ generally takes $2n^3$ flops, as one can verify directly. We can take advantage of the triangular structure of $R_k$ and avoid multiplications by zeros, then the cost of multiplication $R_k\times Q_k$ will be half of $2n^3$ flops, i.e., $n^3$ flops. This makes it $3n^3$ flops total, for each QR step. Thus if we make $p$ QR steps, the total
cost will be $3pn^3$.

This seems to be an improvement over $2pn^4/3$ flops needed by the power method (Section~\ref{19.17}), but in practice this cost is still too high. We also need to remember that the convergence is slow (linear), thus the pure QR algorithm may require many more
iterations than the power method does (see Section~\ref{19.5}).

Fortunately, the computational cost can be dramatically reduced with the help of Hessenberg matrices, see below. The use of Hessenberg matrices will also allow us to develop tricks that speed up the convergence, from linear to quadratic.}

\subsection{Hessenberg matrix} \label{19.22}

$A\in \IC^{n\times n}$ is called an {\em (upper) Hessenberg matrix} if
$a_{ij}=0$ for all $i>j+1$, i.e., whenever $A$ has the form
\vspace{-2mm}
$${\footnotesize
      A =
    \left[\begin{array}{ccccc}
       \times & \times & \cdots & \cdots & \times \\
       \times & \times & \ddots & \ddots & \vdots \\
       0 & \times & \times & \ddots & \vdots \\
       \vdots & \ddots & \ddots & \ddots & \vdots \\
       0 & \cdots & 0 & \times & \times
     \end{array}\right]
}$$
\vspace{-1mm}
Thus, $A$ may have non-zero components in its upper triangle and on its subdiagonal. It is just ``one step away'' from being an upper triangular.

{\small Recall that an upper triangular matrix has its eigenvalues on its diagonal, i.e., its eigenvalues are ``exposed''. The eigenvalues of a Hessenberg matrix are \emph{not} exposed, they are ``hidden'', but if any subdiagonal component is zero, i.e., $a_{i+1,i}=0$ for some $i$, then the eigenvalue problem can be simplified:}

\subsection{Lucky reduction of the eigenvalue problem} \label{LuckyHessenberg}

{\small Suppose $A$ is Hessenberg and $a_{i+1,i}=0$ for some $1\leq i\leq n-1$. Then $A$ has a block-diagonal structure
$$
       A = \begin{bmatrix} A_1 & B \\ 0 & A_2 \end{bmatrix}
$$
where $A_1\in\IC^{i\times i}$ and $A_2\in\IC^{(n-i)\times (n-i)}$.
Now the eigenvalues of $A$ consist of those of $A_1$ and $A_2$. Thus it is enough to find the eigenvalues of the two smaller matrices $A_1$ and $A_2$, while the matrix $B$ can be completely ignored.}

\subsection{Theorem (Hessenberg decomposition)} \label{19.23}

Every matrix $A\in\IC^{n\times n}$ is unitary equivalent to a
Hessenberg matrix, i.e.
$$
        A=Q^{\ast}HQ
$$
where $H$ is a Hessenberg matrix and $Q$ is a unitary matrix. There is
an exact finite algorithm (\emph{Arnoldi algorithm}) for computing $H$ and $Q$.

\medskip{\footnotesize \emph{Proof}. The matrix equation $A=Q^{\ast}HQ$ can be rewritten
as $AQ^{\ast}=Q^{\ast}H$. Denote by $q_i$, $1\leq i\leq n$, the columns
of the unitary matrix $Q^{\ast}$ and by $h_{ij}$ the entries of $H$.
Equating the columns of the matrices $AQ^{\ast}$ and $Q^{\ast}H$ (and
remembering that $h_{ij}=0$ for $i>j+1$) we obtain the following system of
equations
\begin{align*}
    Aq_1 &= h_{11}q_1+ h_{21}q_{2}\\
    Aq_2 &= h_{12}q_1+ h_{22}q_2+h_{32}q_{3}\\
       & \cdots \\
    Aq_i &= h_{1i}q_1+ \cdots +h_{ii}q_i+h_{i+1,i}q_{i+1}\\
       & \cdots \\
    Aq_n &= h_{1n}q_1+ \cdots +h_{n-1,n}q_{n-1}+h_{n,n}q_{n}\\
\end{align*}
Note that the last equation is slightly different from the others,
since it terminates on a diagonal entry of $H$.

Now the Arnoldi algorithm goes along the lines similar to the classical
Gram-Schmidt orthogonalization. We pick an arbitrary unit vector $q_1$,
compute $v_1=Aq_1$ and
$$
   w_2 = v_1 - \la v_1,q_1\ra\, q_1
$$
It is easy to check that $w_2$ will be orthogonal to $q_1$. Then we set $h_{11}=\la v_1,q_1\ra$
and $h_{21}=\|w_2\|$ and define $q_2=w_2/\|w_2\|$. This enforces the
first equation in the above system.

Then, for every $i=2,\ldots,n-1$ we make four steps:

\medskip\noindent{\bf Step 1}:  $\ \ $compute $v_{i}=Aq_{i}$.

\smallskip\noindent{\bf Step 2}: $\ \ $for all $j=1,\ldots,i$ compute $h_{ji}=\la v_{i}, q_j\ra$.

\smallskip\noindent{\bf Step 3}: $\ \ $compute $w_{i} = v_{i}-\sum_{j=1}^i h_{ji}q_j$.

\smallskip (note: the
vector $w_i$ is guaranteed to be orthogonal to $q_1,\ldots,q_i$).

\smallskip\noindent{\bf Step 4}: $\ \ $compute
$h_{i+1,i}=\|w_{i}\|$ and $q_{i+1}=w_{i}/h_{i+1,i}$.\\

Finally, for $i=n$ we execute steps 1
and 2 only.

\smallskip An exception may occur at step 4 if it happens that $h_{i+1,i}=0$. In that case we simply pick an arbitrary unit vector $q_{i+1}$
orthogonal to $q_1,\ldots,q_i$ and continue the procedure ``as if nothing happened''. \qed}

\newpage

{\small The exceptional case $h_{i+1,i}=0$ in Step 4 is sometimes referred to as the {\em breakdown} of the Arnoldi algorithm. This term is quite misleading, however, since the method does not really break down. In fact, the resulting Hessenberg matrix $H$ will then have a simpler structure leading to a lucky reduction of the eigenvalue problem (Secion~\ref{LuckyHessenberg}).

The matrices $H$ and $Q$ in Theorem~\ref{19.23} are far from being unique. The very first step of the Arnoldi algorithm is based on an arbitrary choice of a unit vector $q_1$ (the first column of $Q^\ast$). Actually, by choosing a simple vector $q_1$ (such as $q_1=e_1$), we can reduce the amount of computational work, see next.}

\subsection{Cost of Arnoldi algorithm}

{\small The cost of Step 1 is $2n^2$ flops, the cost of Step 2 is $2ni$ flops,
the same for Step 3, and lastly Step 4 takes $3n$ flops. The total cost
is then
$$
   \sum_{i=1}^n (2n^2+4ni+3n) \sim 4n^3
$$
(by choosing $q_1=e_1$ one can save some work and compute $H$
and $Q$ in $\tfrac{10}{3}n^3$ flops; see the textbook for more
details). This cost is comparable to the cost of one QR step for a
generic (non-Hessenberg) matrix.

The Hessenberg decomposition can be regarded
as a {\em pre-processing} of the matrix $A$; it only needs to be done
once, and then the QR algorithm can be applied to the resulting Hessenberg matrix. It is important, obviously, that the Hessenberg property is not lost during the QR iterations.}

\subsection{Preservation of Hessenberg structure} \label{19.24}

If an invertible matrix $A_0$ is Hessenberg, then all the matrices
$A_k$ generated by the QR algorithm will be Hessenberg matrices as well.

\medskip{\footnotesize \emph{Proof}. By induction, let $A_{k-1}$ be Hessenberg. Then $A_{k-1}=Q_kR_k$
and so $Q_k=A_{k-1}R_k^{-1}$. This is a product of a Hessenberg
matrix $A_{k-1}$ and an upper triangular matrix $R_k^{-1}$. One can verify by direct
inspection that such a product is always a Hessenberg matrix. Now $A_k=R_kQ_k$ is also a
product of an upper triangular matrix $R_k$ and a Hessenberg matrix $Q_k$, so it is
a Hessenberg matrix, too. \qed }

\medskip{\small For a noninvertible Hessenberg matrix $A_0$, the above property remains valid, but the proof is more complicated and we leave it out. Basically, one can show that $Q_k$
constructed by Gram-Schmidt orthogonalization (Chapter~\ref{Ch9}) is also Hessenberg, thus $A_k$ will again be a
Hessenberg matrix.}

\subsection{Cost of a QR step for Hessenberg matrices}

{\small For Hessenberg matrices, the QR algorithm can be implemented with a
substantial reduction of computational cost. First, the QR
decomposition can be done via Givens rotators, it will take $6n^2$ flops, according to
Section~\ref{16.25a}. Second, $Q_k$ is a product of $n-1$ rotators,
and the multiplication of $R_k$ by $Q_k$ can be organized so that it will take $\leq 6n^2$ flops (we omit details). The
total for one QR step is then $\leq 12n^2$ flops, a dramatic drop from
$3n^3$ flops required for a non-Hessenberg matrix $A$ (as we said after Section~\ref{19.21}).}


\begin{center}
\begin{tikzpicture}
   \node[anchor=east] at (-0.5,1) {\footnotesize ``Pure'' QR:};
   \node[anchor=east] at (-0.7,-0.8) {\footnotesize QR with}; \node[anchor=east] at (-0.5,-1.2) {\footnotesize Hessenberg:};
   \node[anchor=east] at (0,0) {\small $A$};
   \draw[->,-latex] (0,0) -- (1,0.5);
   \draw[->,-latex] (0,0) -- (1,-0.5);
   \node[anchor=west] at (1,0.5) {\small $A_0$};
   \node[anchor=west] at (1,-0.5) {\small $A_0$};
   \draw[->,-latex] (1.7,0.5) -- (2.7,0.5);
   \draw[->,-latex] (1.7,-0.5) -- (2.7,-0.5);
   \node[anchor=west] at (2.7,0.5) {\small $A_1$};
   \node[anchor=west] at (2.7,-0.5) {\small $A_1$};
   \fill[gray!50] (0.5,-1) circle(0.45);
   \node[anchor=east] at (1,-1) {\footnotesize $\tfrac{10}{3}n^3$};
   \fill[gray!50] (2.2,-1) circle(0.25);
   \node[anchor=east] at (2.6,-1) {\tiny $12n^2$};
   \fill[gray!50] (2.2,1) circle(0.4);
   \node[anchor=east] at (2.6,1) {\footnotesize $3n^3$};
   \begin{scope}[xshift=1.7cm]
   \draw[->,-latex] (1.7,0.5) -- (2.7,0.5);
   \draw[->,-latex] (1.7,-0.5) -- (2.7,-0.5);
   \node[anchor=west] at (2.7,0.5) {\small $A_2$};
   \node[anchor=west] at (2.7,-0.5) {\small $A_2$};
   \fill[gray!50] (2.2,-1) circle(0.25);
   \node[anchor=east] at (2.6,-1) {\tiny $12n^2$};
   \fill[gray!50] (2.2,1) circle(0.4);
   \node[anchor=east] at (2.6,1) {\footnotesize $3n^3$};
   \end{scope}
   \begin{scope}[xshift=3.4cm]
   \draw[->,-latex] (1.7,0.5) -- (2.7,0.5);
   \draw[->,-latex] (1.7,-0.5) -- (2.7,-0.5);
   \node[anchor=west] at (2.7,0.5) {\small $A_3$};
   \node[anchor=west] at (2.7,-0.5) {\small $A_3$};
   \fill[gray!50] (2.2,-1) circle(0.25);
   \node[anchor=east] at (2.6,-1) {\tiny $12n^2$};
   \fill[gray!50] (2.2,1) circle(0.4);
   \node[anchor=east] at (2.6,1) {\footnotesize $3n^3$};
   \end{scope}
   \begin{scope}[xshift=5.1cm]
   \draw[->,-latex] (1.7,0.5) -- (2.7,0.5);
   \draw[->,-latex] (1.7,-0.5) -- (2.7,-0.5);
   \node[anchor=west] at (2.7,0.5) {\small $A_4$};
   \node[anchor=west] at (2.7,-0.5) {\small $A_4$};
   \fill[gray!50] (2.2,-1) circle(0.25);
   \node[anchor=east] at (2.6,-1) {\tiny $12n^2$};
   \fill[gray!50] (2.2,1) circle(0.4);
   \node[anchor=east] at (2.6,1) {\footnotesize $3n^3$};
   \end{scope}
   \begin{scope}[xshift=6.8cm]
   \draw[->,-latex] (1.7,0.5) -- (2.7,0.5);
   \draw[->,-latex] (1.7,-0.5) -- (2.7,-0.5);
   \node[anchor=west] at (2.7,0.5) {\small $A_5$};
   \node[anchor=west] at (2.7,-0.5) {\small $A_5$};
   \fill[gray!50] (2.2,-1) circle(0.25);
   \node[anchor=east] at (2.6,-1) {\tiny $12n^2$};
   \fill[gray!50] (2.2,1) circle(0.4);
   \node[anchor=east] at (2.6,1) {\footnotesize $3n^3$};
   \end{scope}
   \draw[->,-latex] (10.2,0.5) -- (10.7,0.5);
   \draw[->,-latex] (10.2,-0.5) -- (10.7,-0.5);
\end{tikzpicture}
\end{center}

\subsection{The case of Hermitian matrices}

{\small An important case is that of Hermitian matrices. In that case $A_0$ will be both Hermitian (Section~{SAequiv}) and
Hessenberg. Hence $A_0$ will be a {\em tridiagonal matrix}
($h_{ij}=0$ for all $|i-j| > 1$). Its construction by the Arnoldi
algorithm takes only $\tfrac 43 n^3$ flops (see the textbook for details).

\medskip The use of Hessenberg matrices greatly reduces the cost of each QR step. Another standing issue is a slow (linear) convergence (Theorem~\ref{19.20}), with ratio
\beq \label{rQR}
     r=\max_k|\lambda_{k+1}/ \lambda_k|<1.
\eeq
For example, if $A$ has eigenvalues $1, 9$, and 10, then the ratio is $r=0.9$, and the process would be crawling. Fortunately, we have improvement in this respect, too:}


\subsection{Theorem (without proof)} \label{19.25}

Assume that $A_0$, and hence $A_k$ for all $k\geq 1$, are
Hessenberg matrices. Then the convergence $a_{i,i-1}^{(k)}\to 0$
as $k\to\infty$ in Theorem~\ref{19.20} is linear with ratio
$r_i=|\lambda_{i}/\lambda_{i-1}|$. In addition, the convergence
$a_{nn}^{(k)} \to \lambda_n$ is linear with ratio
$r_n=|\lambda_{n}/\lambda_{n-1}|$.

\medskip {\small Now each subdiagonal entry $a_{i,i-1}^{(k)}$ has \emph{its own} ratio of convergence $r_i$, so some may converge faster than others. The convergence is no longer governed by one single ratio \eqref{rQR}. For example, if $A$ has eigenvalues $1, 9$, and 10, then the last (bottom) subdiagonal entry converges with ratio $r=1/9$, which is very fast. This new flexibility allows us to develop smart tricks accelerating the convergence of some selected entries first, and then all the remaining entries.}

\subsection{QR algorithm with shift} \label{19.26a}

One can modify the matrix $A$ to decrease the ratio
$r_n=|\lambda_{n}/\lambda_{n-1}|$ and thus make the convergence
$$
  a_{n,n-1}^{(k)}\to 0\qquad\text{and}\qquad
  a_{nn}^{(k)}\to\lambda_n
$$
of the two bottom entries faster.

This can be done with the help of \emph{shifting}, as in
Section~\ref{19.14}. Recall that for any $\rho\in \IC$ the matrix $A-\rho I$ has eigenvalues $\lambda_1-\rho,\ldots,\lambda_n-\rho$. The new matrix $A-\rho I$ is still Hessenberg (or Hermitian) if $A$ itself was Hessenberg (resp., Hermitian), so we lose nothing by replacing $A$ with $A-\rho I$.

Now suppose $\rho$ is a good approximation to the smallest eigenvalue $\lambda_n$. If we apply the QR steps to the
matrix $A'=A-\rho I$, then the convergence of the two bottom entries of the new matrix will be linear with ratio
$$
     r_n'=|\lambda_n-\rho|/|\lambda_{n-1}-\rho|.
$$
Now we can control the speed of convergence by choosing/adjusting the shift $\rho$. Obviously, if $\rho\to \lambda_n$, then $r_n'\to 0$. Thus the better $\rho$
approximates $\lambda_n$ the faster the convergence of the two bottom entries will be.

\subsection{QR algorithm with Rayleigh quotient shift} \label{19.26b}

The approximation $\rho$ can be updated at every iteration, as in
Section~\ref{19.15}, by using the Rayleigh quotient
$$
        \rho=\rho_k=u_k^{\ast}A_ku_k
$$
where $u_k$ is an approximate unit eigenvector of the matrix $A_k$
corresponding to its smallest eigenvalue $\lambda_n$. In practice, a
simple and convenient choice for $u_k$ is $u_k=e_n$, which gives
$\rho_k = a_{nn}^{(k)}$. Then the {\em QR algorithm with shift} goes as
follows:
\begin{align*}
      A_{k-1}-\rho_{k-1}I&=Q_kR_k\qquad& &(\text{QR decomposition of }A_{k-1}-\rho_{k-1}I)\\
      R_kQ_k+\rho_{k-1}I&=A_k\qquad& &(\text{computation of the next matrix }A_k)\\
      \rho_{k}&=a_{nn}^{(k)}\qquad& &(\text{setting }\rho_k \text{ to the trailing entry of } A_k)
\end{align*}
This is called the {\em Rayleigh quotient shift}. The convergence of the two bottom entries of $A_k$
is now \underline{quadratic}, basically for the same reasons as it was in
Section~\ref{19.15}.

{\small The use of Rayleigh quotient shift allows us very quickly -- say, in about five iterations -- to ``kill'' the bottom subdiagonal entry $a_{n,n-1}^{(k)}$ and find the smallest eigenvalue $\lambda_n$ of the matrix $A$. But the other subdiagonal entries, $a_{i+1,i}^{(k)}$, $1\leq i\leq
n-2$, will move to zero slowly -- linearly, with variable ratios. For example if $A$ has eigenvalues $1, 9$, and 10, then the smallest one will be found very fast, but the other two may take 50--100 (or more) iterations, because the convergence will proceed with a pretty bad ratio $r=0.9$.

So we apply one more trick -- the ``lucky reduction'' described in Section~\ref{LuckyHessenberg} -- to ``split off'' the smallest eigenvalue $\lambda_n$ and reduce the size of the matrix $A$:}

\subsection{QR algorithm with deflation} \label{19.26c}

When $a_{n,n-1}^{(k)}$ becomes practically zero and $a_{nn}^{(k)}$ becomes practically equal
to $\lambda_n$, the matrix $A_k$ will have the following form:
$$
       A_k= \left[\begin{array}{cc}
          \hat{A}_k & b_k \\
       0 & \lambda_n \end{array}\right]
$$
where $\hat{A}_k$ is an $(n-1)\times (n-1)$ Hessenberg matrix. Its
eigenvalues are those of $A$, minus $\lambda_n$, i.e., they are  $\lambda_1,\ldots,\lambda_{n-1}$. Now we
can apply further steps of the QR algorithm (with shift) to the matrix
$\hat{A}_k$, instead of $A_k$. This quickly produces its smallest
eigenvalue, $\lambda_{n-1}$, which can be split off in the same manner. Then we reduce our matrix further, to the size $(n-2)\times (n-2)$, and so on.
This procedure is called the {\em deflation} of the matrix $A$:

\begin{center}
\begin{tikzpicture}
  \draw (0,0) rectangle (1,1);
  \fill[black!80] (0,1)--(1,1)--(1,0)--cycle;
  \foreach \x in {{0},...,{9}} \fill[black] (\x/10,0.9-\x/10) rectangle (0.1+\x/10,1-\x/10);
  \foreach \x in {{0},...,{7}} \fill[gray] (\x/10,0.8-\x/10) rectangle (0.1+\x/10,0.9-\x/10);
  \draw[red] (-0.1,0.1)--(0.9,0.1)--(0.9,1.1);
  \draw[thick,black!60,->,-latex] (1.3,0.5)--(1.7,0.5);

  \begin{scope}[xshift=2.0cm,yshift=0.1cm]
  \draw (0,0) rectangle (0.9,0.9);
  \fill[black!80] (0,0.9)--(0.9,0.9)--(0.9,0)--cycle;
  \foreach \x in {{0},...,{8}} \fill[black] (\x/10,0.8-\x/10) rectangle (0.1+\x/10,0.9-\x/10);
  \foreach \x in {{0},...,{6}} \fill[gray] (\x/10,0.7-\x/10) rectangle (0.1+\x/10,0.8-\x/10);
  \draw[red] (-0.1,0.1)--(0.8,0.1)--(0.8,1.0);
  \draw[thick,black!60,->,-latex] (1.2,0.45)--(1.6,0.45);
  \end{scope}

  \begin{scope}[xshift=3.9cm,yshift=0.2cm]
  \draw (0,0) rectangle (0.8,0.8);
  \fill[black!80] (0,0.8)--(0.8,0.8)--(0.8,0)--cycle;
  \foreach \x in {{0},...,{7}} \fill[black] (\x/10,0.7-\x/10) rectangle (0.1+\x/10,0.8-\x/10);
  \foreach \x in {{0},...,{5}} \fill[gray] (\x/10,0.6-\x/10) rectangle (0.1+\x/10,0.7-\x/10);
  \draw[red] (-0.1,0.1)--(0.7,0.1)--(0.7,0.9);
  \draw[thick,black!60,->,-latex] (1.1,0.4)--(1.5,0.4);
  \end{scope}

  \begin{scope}[xshift=5.7cm,yshift=0.3cm]
  \draw (0,0) rectangle (0.7,0.7);
  \fill[black!80] (0,0.7)--(0.7,0.7)--(0.7,0)--cycle;
  \foreach \x in {{0},...,{6}} \fill[black] (\x/10,0.6-\x/10) rectangle (0.1+\x/10,0.7-\x/10);
  \foreach \x in {{0},...,{4}} \fill[gray] (\x/10,0.5-\x/10) rectangle (0.1+\x/10,0.6-\x/10);
  \draw[red] (-0.1,0.1)--(0.6,0.1)--(0.6,0.8);
  \draw[thick,black!60,->,-latex] (1.0,0.35)--(1.4,0.35);
  \end{scope}

  \begin{scope}[xshift=7.4cm,yshift=0.4cm]
  \draw (0,0) rectangle (0.6,0.6);
  \fill[black!80] (0,0.6)--(0.6,0.6)--(0.6,0)--cycle;
  \foreach \x in {{0},...,{5}} \fill[black] (\x/10,0.5-\x/10) rectangle (0.1+\x/10,0.6-\x/10);
  \foreach \x in {{0},...,{3}} \fill[gray] (\x/10,0.4-\x/10) rectangle (0.1+\x/10,0.5-\x/10);
  \draw[red] (-0.1,0.1)--(0.5,0.1)--(0.5,0.7);
  \draw[thick,black!60,->,-latex] (0.9,0.3)--(1.3,0.3);
  \end{scope}

  \begin{scope}[xshift=9.0cm,yshift=0.5cm]
  \draw (0,0) rectangle (0.5,0.5);
  \fill[black!80] (0,0.5)--(0.5,0.5)--(0.5,0)--cycle;
  \foreach \x in {{0},...,{4}} \fill[black] (\x/10,0.4-\x/10) rectangle (0.1+\x/10,0.5-\x/10);
  \foreach \x in {{0},...,{2}} \fill[gray] (\x/10,0.3-\x/10) rectangle (0.1+\x/10,0.4-\x/10);
  \draw[red] (-0.1,0.1)--(0.4,0.1)--(0.4,0.6);
  \draw[thick,black!60,->,-latex] (0.8,0.25)--(1.2,0.25);
  \end{scope}

  \begin{scope}[xshift=10.5cm,yshift=0.6cm]
  \draw (0,0) rectangle (0.4,0.4);
  \fill[black!80] (0,0.4)--(0.4,0.4)--(0.4,0)--cycle;
  \foreach \x in {{0},...,{3}} \fill[black] (\x/10,0.3-\x/10) rectangle (0.1+\x/10,0.4-\x/10);
  \foreach \x in {{0},...,{1}} \fill[gray] (\x/10,0.2-\x/10) rectangle (0.1+\x/10,0.3-\x/10);
  \end{scope}

  \node at (9,0) {\footnotesize ``cutting lines'' $\quad$ are in red};
  \draw[red] (9.2,-0.1)--(9.4,-0.1)--(9.4,0.1);

\end{tikzpicture}
\end{center}

In practice, each eigenvalue of $A$ requires just 3-5 iterations (QR
steps), on the average. For Hermitian matrices, it is even faster --
just 2-3 iterations per eigenvalue. Remember also that each QR step for Hessenberg matrices is very cheap (just $12n^2$ flops). This is how the QR algorithm with Hessenberg matrices, Rayleigh quotient shift,
and deflation achieves a top speed.


\bigskip{\small The above procedure works well on generic matrices, but it fails occasionally in certain exceptional cases. One such case is a matrix $A$ with two equally large eigenvalues, i.e., $|\lambda_i| = |\lambda_j|$ for some $i\neq j$. Such a matrix does not meet the requirements of Theorem~\ref{19.20}, so the convergence is not guaranteed. In that case the QR algorithm may ``get stuck'', as illustrated by the following example. }

\newpage

\subsection{Example} \label{19.27a}

{\footnotesize Let $A=\begin{bmatrix} 0&1 \\ 1&0\end{bmatrix}$. Note that $A$ is already a Hessenberg matrix, thus the Hessenberg decomposition is not needed here. Now the pure QR
algorithm gives
$$
   A_0 = Q_1R_1 =
   \begin{bmatrix} 0&1 \\ 1&0\end{bmatrix}
   \begin{bmatrix} 1&0 \\ 0&1\end{bmatrix}
$$
hence
$$
   A_1 = R_1Q_1 =
   \begin{bmatrix} 1&0 \\ 0&1\end{bmatrix}
   \begin{bmatrix} 0&1 \\ 1&0\end{bmatrix} = A
$$
and we are ``back to square one''. The process goes nowhere. The Rayleigh quotient shift $\rho=a_{22}$
proposed in Section~\ref{19.26b} has no effect either, since $a_{22}=0$. The reason of this failure is
that the eigenvalues of $A$, which are $+1$ and $-1$, have equal
absolute values -- this is a symmetry which confuses the QR algorithm,
it ``cannot decide'' which eigenvalue to approach. To break this
symmetry, one needs to choose the shift $\rho$ differently.}

\medskip{\small An alternative shift $\rho$ can be used whenever the procedure ``gets stuck'':}

\subsection{Wilkinson shift} \label{19.27}

{\small At step $k$ of the QR algorithm with shift, let $B_k$ denote  the trailing $2\times 2$ block at the bottom right corner of $A_k$:
\beq \label{Bk}
     B_k  = \left[\begin{array}{ll}
     a_{n-1,n-1}^{(k)} & a_{n-1,n}^{(k)} \\
     a_{n,n-1}^{(k)} & a_{n,n}^{(k)}
     \end{array} \right]
\eeq
Now set $\rho_k$ to the eigenvalue of $B_k$ that is closer to
$a_{n,n}^{(k)}$ (in case of a tie, either one can be taken). This is
called {\em Wilkinson shift}.

The eigenvalues of a $2\times 2$ matrix can be easily (and precisely)
computed by the quadratic formula, whether they are real or complex. If
they are real, then the Wilkinson shift will help break the symmetry like in Example~\ref{19.27a}.}

\subsection{Example~\ref{19.27a} continued}

{\footnotesize The Wilkinson shift here is either $\rho=1$ or $\rho=-1$. Let us choose
$\rho=-1$. Then
$$
      A_0-\rho I=
   \begin{bmatrix} 1&1 \\ 1&1\end{bmatrix}
      =Q_1R_1=
   \left[\begin{array}{rr} 1/\sqrt{2}&1/\sqrt{2} \\
   1/\sqrt{2}&-1/\sqrt{2}\end{array}\right]
   \begin{bmatrix} \sqrt{2}&\sqrt{2} \\ 0&0\end{bmatrix}
$$
and then
$$
      A_1=R_1Q_1+\rho I =
   \begin{bmatrix} 2&0 \\ 0&0\end{bmatrix}+
   \left[\begin{array}{rr} -1&0 \\
   0&-1\end{array}\right] =
   \left[\begin{array}{rr} 1&0 \\
   0&-1\end{array}\right].
$$
The matrix $A_1$ is diagonal and its diagonal entries are the eigenvalues of $A$.

So the QR algorithm with the Wilkinson shift converges in just one step.}

\bigskip{\small Lastly, we discuss an important case of real matrices with complex eigenvalues. It was already mentioned in Section~\ref{19.17}(c).

If the matrix $A$ is real (which is the most common case in practice), then its complex eigenvalues come in
conjugate pairs $a\pm ib$. They have equal absolute values
$|a+bi|=|a-bi|$, immediately violating the main assumption of Theorem~\ref{19.20}.

Furthermore, the pure QR algorithm is bound to fail for the following simple reason. If $A$ is real, then all $Q_k$, $R_k$ and $A_k$ will be real
matrices as well, thus we cannot even expect the real diagonal
entries of $A_k$ to converge to the complex eigenvalues of $A$. In
this case $A_k$ may not converge to anything at all.

The QR algorithm with shift may be able to find complex eigenvalues of a real matrix $A$, but it needs to use complex shifts $\rho\notin\IR$ and complex matrices $A-\rho I$, which is inconvenient and
expensive. Fortunately, the QR algorithm can be organized so that it operates with real matrices only, for as
long as possible, and obtains pairs of complex conjugate
eigenvalues only at the very last step:}

\subsection{Wilkinson shifts for complex eigenvalues of real matrices} \label{19.27b}

Suppose at the step $k$ of the QR algorithm, the matrix $A_k$ is still real, but the eigenvalues of its $2\times 2$ trailing block $B_k$ in \eqref{Bk} are non-real complex numbers. Let us denote them by $\rho_k$ and $\bar{\rho}_k$ (remember that they are complex conjugate to each other). Then the Wilkinson shift $\rho_k$ will
be a complex number, and the matrix $A_k-\rho_k I$ will be complex, too. The QR
step will then produce a complex matrix $A_{k+1}$, which we do not really want.

In this case one can use the following trick to avoid
further working with complex matrices: at the very next QR step we set $\rho_{k+1}=\bar{\rho}_k$, i.e., we choose the shift $\rho_{k+1}$ to be the \emph{other} complex eigenvalue of $B_k$.
Then, remarkably, the resulting matrix $A_{k+2}$ will be real again, for the reasons explained bellow.

Furthermore, the values $\rho_k$ and $\bar{\rho}_k$ will approximate
two complex eigenvalues of the matrix $A$. Therefore, the QR algorithm
with Wilkinson shift is able to compute conjugate pairs of complex
eigenvalues of a real matrix $A$, and stay with real matrices for as long as possible.

Actually, there is no need to compute the complex matrix $A_{k+1}$
mentioned above. One can just combine the two QR steps together and
construct $A_{k+2}$ directly from $A_k$ (bypassing $A_{k+1}$). This can
be carried out entirely in real arithmetic. The resulting all-real
procedure is called the {\em double-step QR algorithm with Wilkinson
shift}.

\medskip{\footnotesize In the following, all complex scalars and matrices are marked red. This includes matrix $A_2$, which will remain hypothetically complex until we prove that it is real.}

\newpage

\subsection{Analysis of the double-step}

Let $A_0\in\IR^{n\times n}$ be a real matrix. Let $\color{red}\rho$ and $\color{red}\bar{\rho}$ be two complex non-real conjugate numbers that are \emph{not} eigenvalues of $A_0$. Let us apply two QR steps with the shifts $\color{red}\rho$ and $\color{red}\bar{\rho}$:
\begin{align}
      A_0-{\color{red}\rho} I &={\color{red}Q_1R_1}\qquad
      {\color{red}R_1Q_1}+{\color{red}\rho} I={\color{red}A_1}
      \label{QRstep1}\\
      {\color{red}A_1}-{\color{red}\bar{\rho}} I &={\color{red}Q_2R_2}\qquad
      {\color{red}R_2Q_2}+{\color{red}\bar{\rho}} I={\color{red}A_2}
      \label{QRstep2}
\end{align}
Since the matrices $A_0-{\color{red}\rho} I$ and ${\color{red}A_1}-{\color{red}\bar{\rho}}I$ are nonsingular,
the above QR decompositions may be constructed so that $\color{red}R_1$ and $\color{red}R_2$
have positive real entries, cf.\ Section~\ref{QR6}. Then, remarkably, $\color{red}A_2$
will be a real matrix.

\medskip{\footnotesize \emph{Proof}. It follows from \eqref{QRstep1} that ${\color{red}R_1}={\color{red}Q_1^{-1}}A_0-{\color{red}\rho}\, {\color{red}Q_1^{-1}}$, hence
$$
    {\color{red}A_1}=({\color{red}Q_1^{-1}}A_0-{\color{red}\rho}\, {\color{red}Q_1^{-1}}){\color{red}Q_1}+{\color{red}\rho} I
    ={\color{red}Q_1^{-1}} A_0 {\color{red}Q_1}
$$
Similarly, it follows from \eqref{QRstep2} that ${\color{red}R_2}={\color{red}Q_2^{-1}}{\color{red}A_1}-{\color{red}\bar{\rho}}\, {\color{red}Q_2^{-1}}$, hence
$$
    {\color{red}A_2}=({\color{red}Q_2^{-1}}{\color{red}A_1}-{\color{red}\bar{\rho}}\, {\color{red}Q_2^{-1}}){\color{red}Q_2}+{\color{red}\bar{\rho}} I
    ={\color{red}Q_2^{-1}} {\color{red}A_1} {\color{red}Q_2}
$$
Next we will compute the product $(A_0-{\color{red}\bar{\rho}} I)(A_0-{\color{red}\rho} I)$ in two ways. First,
\begin{align*}
  (A_0-{\color{red}\bar{\rho}} I)(A_0-{\color{red}\rho} I) &=
  A_0^2 - ({\color{red}\rho}+{\color{red}\bar{\rho}})A_0 + {\color{red}\rho\bar{\rho}}I\\
  &=A_0^2 - 2({\rm Re}\,{\color{red}\rho})A_0 + |{\color{red}\rho}|^2I.
\end{align*}
Since $({\rm Re}\,{\color{red}\rho})$ and $|{\color{red}\rho}|$ are real numbers, the matrix $A=(A_0-{\color{red}\bar{\rho}} I)(A_0-{\color{red}\rho} I)$
is entirely real.

On the other hand, we have
\begin{align*}
  A&=(A_0-{\color{red}\bar{\rho}} I)(A_0-{\color{red}\rho} I)\\
   &= (A_0-{\color{red}\bar{\rho}} I){\color{red}Q_1R_1} \\
   &= {\color{red}Q_1Q_1^{-1}}(A_0-{\color{red}\bar{\rho}} I){\color{red}Q_1R_1} \\
   &= {\color{red}Q_1}({\color{red}Q_1^{-1}} A_0 {\color{red}Q_1}-{\color{red}\bar{\rho}} I)
   {\color{red}R_1} \\
   &= {\color{red}Q_1}({\color{red}A_1}-{\color{red}\bar{\rho}} I)
   {\color{red}R_1} \\
   &= {\color{red}Q_1Q_2R_2R_1}.
\end{align*}
This is actually a QR decomposition of the matrix $A$, with orthogonal matrix ${\color{red}Q}={\color{red}Q_1Q_2}$ and upper triangular matrix ${\color{red}R}={\color{red}R_2R_1}$. Since the matrices $\color{red}R_1$ and $\color{red}R_2$ have positive diagonal entries, so does its product $\color{red}R$. Therefore, according to Section~\ref{QR6}, the above QR decomposition is unique.

But every real matrix $A$ has a real QR decomposition $A=QR$ with positive diagonal entries of $R$ (by Sections~\ref{QR5} and \ref{QR5aaa}), hence by its uniqueness both matrices $Q={\color{red}Q_1Q_2}$
and $R={\color{red}R_2R_1}$ must be real. Lastly,
$$
   {\color{red}A_2} = {\color{red}Q_2^{-1}} {\color{red}A_1} {\color{red}Q_2} = ({\color{red}Q_1Q_2})^{-1} A_0 ({\color{red}Q_1Q_2})
   = Q^{-1} A_0 Q.
$$
As a product of three real matrices, $A_2$ must be a real matrix as well. \qed}

\begin{exercise}
(JPE, September 2009) Let $A\in \IC^{n\times n}$ be nonsingular. Let $A
= Q_1R_1$ be a QR decomposition of $A$, and for $k\geq 1$ define
inductively $AQ_k = Q_{k+1}R_{k+1}$, a QR decomposition of $AQ_k$.
\begin{itemize} \item[(a)] Prove that there exists an upper triangular matrix $U_k$ such
that $Q_k = A^kU_k$ and a lower triangular matrix $L_k$ such that
$Q_k = (A^\ast)^{-k}L_k$. \item[(b)] Suppose $\lim_{k\to\infty} R_k
= R_\infty$ and $\lim_{k\to\infty} Q_k = Q_{\infty}$ exist.
Determine the eigenvalues of $A$ in terms of
$R_{\infty}$.\end{itemize}
\end{exercise}

\begin{exercise}
(JPE, May 2006) Let $A \in \IC^{n\times n}$ be tri-diagonal and
Hermitian, with all its super-diagonal entries nonzero. Prove that the
eigenvalues of $A$ are distinct.

\new (Hint: show that for any scalar $\lambda$, the matrix $A-\lambda I$
has rank at least $n-1$.)\end{exercise}

\end{document}
\end
