WUT_Computer_Science/ENUME/projectA/projectA.tex

\documentclass[12pt]{report}

\usepackage{mathtools}
\usepackage{amsmath}
\usepackage{systeme}
\usepackage{siunitx}
\usepackage[T1]{fontenc}
\usepackage{hyperref}
\usepackage{bigfoot}
\usepackage[numbered, framed]{matlab-prettifier}
\usepackage{filecontents}
\usepackage{graphicx}
\hypersetup{
colorlinks,
citecolor=black,
filecolor=black,
linkcolor=black,
urlcolor=black
linkto=all,
}

\newenvironment{simplechar}{%
   \catcode`\^=12
}{}

\title{Numerical Methods, project A, Number 31}
\author{Krzysztof Rudnicki\\ Student number: 307585 \\ Advisor: dr Adam Krzemieniowski}
\date{\today}

\let\ph\mlplaceholder % shorter macro
\lstMakeShortInline"

\lstset{
  style              = Matlab-editor,
  basicstyle         = \mlttfamily,
  escapechar         = ",
  mlshowsectionrules = true,
}

\begin{document}


\maketitle
\tableofcontents

\chapter{Problem 1 - Finding machine epsilion}

\section{Problem}
Write a program finding macheps in the MATLAB environment
\section{Theoretical Introduction}
\subsection{Definition of machine epsilion}
Machine epsilion is the maximal possible relative error of the floating-point representation. (Tatjewski, p.14)
Machine epsilion is equal to $2^{-t}$ where t is number of bits in the mantissa.
In our case when We use IEEE Standard 754, mantissa is 53 bits long with first bit omitted as it is always equal to '1', so We technicaly work with 52 bits mantissa which makes the machine epsilion equal to: $2^{-52} = 2.220446\mathrm{e}{-16}$

\subsection{Practical applications of machine epsilion}
Since macheps is connected to IEEE754 standard it is always equal to the same number, which means that We can safely compare results from different machines without worrying about their individual errors.

Macheps is also essential when We calculate cumulation of errors of given mathematical operation.

\newpage
\section{Solution}

\subsection{Matlab code}
\begin{lstlisting}
macheps = 1;
while 1.0 + macheps / 2 > 1.0
    macheps = macheps/2;
end
\end{lstlisting}
Code above shifts macheps one bit to the right each iteration (by dividing by 2), it ends when We run out of mantissa bits which renders us unable to save smaller number. Due to underflow the value of macheps becomes 0 and therefore 1.0 > (macheps / 2) > 1.0 will become false.

\section{Discussion of the result}
\begin{simplechar}
\begin{lstlisting}
format long
disp("Display calculated macheps:")
disp(macheps);
disp("Display actual eps:")
disp(eps);
disp("Display 2^-52")
disp(2^-52)
disp("Display difference between calculated macheps and actual eps:")
disp(macheps - eps)
disp("Display difference between 2^-52 and actual eps:")
disp(2^-52 - eps) \
disp("Display difference between calculated macheps and 2^-52:")
disp(macheps - 2^-52)
\end{lstlisting}
\end{simplechar}
Display calculated macheps:
     \[2.220446049250313\mathrm{e}{-16}\]

Display actual eps:
     \[2.220446049250313\mathrm{e}{-16}\]

Display $2^{-52}$:
     \[2.220446049250313\mathrm{e}{-16}\]

Display difference between calculated macheps and actual eps:
     \[0\]

Display difference between $2^{-52}$ and actual eps:
          \[0\]

Display difference between calculated macheps and $2^{-52}$:
         \[0\]

As expected they are all equal to eachother. It means that our method of calculating macheps was correct.

\chapter{Problem 2 - Solving a system of n linear equations - indicated method}

\section{Problem}
Write a program solving a system of \textit{n} linear equations Ax = b using the indicated method (Gaussian elimination with partial pivoting).

\section{Theoretical Introduction}
Gaussian elimination with partial pivoting consists of 3 main steps:
\subsection{Transform matrix into upper-triangular matrix}

\subsubsection{Starting conditions}
We start with the system of linear equations looking like this:

\[
\begin{matrix}

&a_{11}x_1 &{}+&a_{12}x_2&+&\dots&+&a_{1n}x_n &=&b_1,\\

&a_{21}x_1 &{}+&a_{22}x_2&+&\dots&+&a_{2n}x_n &=&b_2,\\

&\vdots    &&\vdots      & &     & &  \vdots  & &\vdots\\

&a_{n1}x_1&{}+&a_{n2}x_2&+&\dots &+&a_{nn}x_n&=&b_n.

\end{matrix}
\]

In order for this method to work all the elements of "diagonal" line:
\[ a_{11}, a_{22}, \dots, a_{nn} \]
Must be different from zero since We will be dividing by them.

We will denote rows as '$w_i$' where 'i' is number of the row.

\subsubsection{Zeroing first column}
We start transforming the system by "zeroing" elements in first column excluding first row element. We do it by multiplying first row by $l_{i1}$, where:
\[ l_{i1} = \frac{ a_{i1}^{(1)} }
{ a_{11}^{(1)} }  \]
And then substracting what We got ($ l_{i1}w_1 $), from \textit{i} row.

Doing so We obtain a system of linear equations:

\[
\begin{matrix}

&a_{11}x_1 &{}+&a_{12}x_2&+&\dots&+&a_{1n}x_n &=&b_1,\\

&0 &{}+&(a_{22} - a_{12}l_{21})x_2&{}+&\dots&{}+&(a_{2n} - a_{1n}l_{21})x_n &=&b_2 - b_{1}l_{21},\\

&\vdots    &&\vdots      & &     & &  \vdots  & &\vdots\\

&0&{}+&(a_{n2} - a_{12}l_{n1})x_2&+&\dots &+&(a_{nn} - a_{1n}l_{n1})x_n&=&b_n - b_{1}l_{n1}.

\end{matrix}
\]

\subsubsection{Zeroing second column}
We continue onto the second column, this time We will zero all elements except first and second rows.
Row multiplier becomes:
\[ l_{i2} = \frac{ a_{i2}^{(2)} }{ a_{22}^{(2)} } \]

Where:
\[ a_{22}^{(2)} = (a_{22} - a_{12}l_{21}) \]
And:
\[ a_{i2}^{(2)} = (a_{i2} - a_{12}l_{i1}) \]
They are modified values obtained from previous step.
We continue as in the first step and We end up with:

\[
\begin{matrix}

&a_{11}x_1 &{}+&a_{12}x_2&+&\dots&+&a_{1n}x_n &=&b_1,\\

&0 &{}+& a_{22}^{(2)}x_2&{}+&\dots&{}+& a_{2n}^{(2)}x_n &=&b_2^{(2)},\\

&\vdots    &&\vdots      & &     & &  \vdots  & &\vdots\\

&0 &{}+& 0 &{}+&\dots&{}+& a_{nn}^{(3)}x_n &=&b_2^{(3)},

\end{matrix}
\]

\subsubsection{Zeroing next columns}
We repeat this process $n-1$ times and We end up with upper triangular matrix:

\[
\begin{matrix}

&a_{11}x_1 &{}+&a_{12}x_2&+&\dots&+&a_{1n}x_n &=&b_1,\\

&0 &{}+& a_{22}^{(2)}x_2&{}+&\dots&{}+& a_{i2}^{(2)}x_n &=&b_2^{(2)},\\

&\vdots    &&\vdots      & &     & &  \vdots  & &\vdots\\

&0 &{}+& 0 &{}+&\dots&{}+& a_{nn}^{(n)}x_n &=&b_2^{(n)},

\end{matrix}
\]

\subsection{Backward substitution}
After transforming the system We solve the system from last to first. \\
First We calculate value of last element:
\[ x_n = \frac{b_n}{a_{nn}} \]
Then one "above":
\[ x_{n-1} = \frac{ b_{n-1} - a_{n-1, n}x_n}{a_{n-1, n-1}} \]
And so on, for $x_k$:
\[ x_{k} = \frac{b_k - \sum_{j = k + 1}^n a_{kj}x_j}{a_{kk}} \]

\subsection{Partial Pivoting}
Gaussian elimination method has one flaw, where it can come into halt if:
\[ a_{kk}^{(k)} = 0 \]
To avoid it We use method of pivoting, in our case We will use partial pivoting method.
We do it before each Gaussian elimination step since this will lead to smaller error.
\\We first find a row $i$ such that:
\[ |{a_{ik}^{k}}| = \underset{j}{max} \{ |{a_{kk}^{k}}|, |{a_{k+1, k}^{k}}|, \cdots, |{a_{nk}^{k}}|\} \]

Then We swap this row with k-th row. Since the matrix We use is assumed to be nonsingular then $|{a_{ik}^{k}}| \neq 0$ will be always true. After that We continue with the Gaussian elimination method.

\section{Discussion of the result}
Solutions vectors for matrix A and vector A and n = 10:
\[ x_{algorithm} = \left(\begin{array}{cc}
-0.930024655110760 \\
-1.223407298665613 \\
-1.273530574219411 \\
-1.230517757325955 \\
-1.151356031091789 \\
-1.056883669282743 \\
-0.952628310089775 \\
-0.834334594319914 \\
-0.683708806203301 \\
-0.450125157623323
\end{array} \right)
%
x_{Matlab Method} = \left(  = \begin{array}{cc}
-0.930024655110760 \\
-1.223407298665613 \\
-1.273530574219411 \\
-1.230517757325955 \\
-1.151356031091788 \\
-1.056883669282743 \\
-0.952628310089775 \\
-0.834334594319914 \\
-0.683708806203301 \\
-0.450125157623323
\end{array} \right)
\]

Error for 'A' method for algorithm: \[ 3.383918772654241 \]
Error for 'A' method for matlab method: \[ 3.383918772654241 \]
\newpage
Solutions vectors for matrix B and vector B and n = 16 (Both multiplied by $ 1.0e+17 $):
\[ x_{algorithm} = \left( \begin{array}{cc}
  0.000001960155675 \\
  -0.000102773501571 \\
  0.001959454282882 \\
  -0.018894079120425 \\
  0.104895022735396 \\
  -0.352396798852209 \\
  0.705545645736628 \\
  -0.728747526649489 \\
  0.112818247768452 \\
  0.261440075930356 \\
  0.953713133491034 \\
  -3.080986443185790 \\
  3.765178552913233 \\
  -2.440218622397594 \\
  0.834768953546100 \\
  -0.118974790826378
\end{array} \right)
%
x_{Matlab Method} = \left(  = \begin{array}{cc}
  0.000001102587209 \\
  -0.000066546298462 \\
  0.001476714765758 \\
  -0.016859999627589 \\
  0.113920718370153 \\
  -0.488374161741872 \\
  1.368641128884513 \\
  -2.495283439985873 \\
  2.793405296264694 \\
  -1.547642305352008 \\
  -0.035332172403445 \\
  0.154726025421297 \\
  0.807694426359552 \\
  -1.133485136703852 \\
  0.592810708065954 \\
  -0.115632364630554
\end{array} \right)
\]

Error for 'B' method for algorithm: \[ 5.699979882700911e+17 \]
Error for 'B' method for matlab method: \[ 4.569118543317684e+17 \]

\begin{center}
   \includegraphics[scale=0.75]{errorsNoResidual.eps}
\end{center}





\addtocontents{toc}{\protect\newpage}
\chapter{Problem 3 - Solving a system of n linear equations - iterative algorithm}

\section{Problem}
Write a general program for solving the system of n linear equations Ax = b using the Gauss-Seidel \textbf{and} Jacobi iterative algorithms.

We are given following system:
\[
\systeme{10x_1-4x_2+x_3+2x_4=-8, 2x_1-6x_2+3x_3-x_4=-12, x_1+4x_2-12x_3+x_4=4, 2x_1+3x_2-3x_3-10x_4=1}
\]

Then We need to compare the results of iterations plotting norm of the solution error versus the iteration number \textbf{k}, untill We get accuracy better than $10^{-10}$.

We should also try to solve the equations from problem 2a) and 2b) for n = 10 using iterative method of our choice.


\section{Theoretical introduction}

 We should also answer the question what happens if the sufficient condition is not fullfiled.

Itertaive methods differ from the Gauss elimination method since they are iterative, which means that our solution will improve with each iteration. Building on that We can cnclude that the number of iterations will depend on what accuracy We want to achieve. Since We are using iterative method We don't have the guarantee of how many iterations will be neeeded before We reach the solution,

In general:
We start with:
 \textbf{$x^{(0)}$ } - being the best known approximation of the solution point

And We generate next vectors  \textbf{$x^{i+1}$}  in such way:
\[ \mathbf{x^{i+1}} = \mathbf{M}\mathbf{x^{(i)}} + \mathbf{w} \]

Where $\mathbf{M}$ is some matrix.

\subsection{Procedure}

\subsubsection{Decomposing matrix}
For both Jacobi and Gauss-Seidel method We first decompose starting matrix $ \mathbf{A} $ to:
\[ \mathbf{A} = \mathbf{L} + \mathbf{D} + \mathbf{U} \]
where:
$ \mathbf{L} $ - Subdiagonal matrix
$ \mathbf{D} $ - Diagonal matrix
$ \mathbf{U} $ - Matrix with entries over the diagonal.

For example:
For:
\[ \mathbf{A} = \begin{bmatrix}
2 & 3 & 3\\
1 & 2 & 3\\
1 & 1 & 2
\end{bmatrix}
\]

We can get:
\[ \mathbf{L} = \begin{bmatrix}
0 & 0 & 0\\
1 & 0 & 0\\
1 & 1 & 0
\end{bmatrix}
\]
\[ \mathbf{D} = \begin{bmatrix}
2 & 0 & 0\\
0 & 2 & 0\\
0 & 0 & 2
\end{bmatrix}
\]
\[ \mathbf{U} = \begin{bmatrix}
0 & 3 & 3\\
0 & 0 & 3\\
0 & 0 & 0
\end{bmatrix}
\]

so:

\[ \mathbf{A} = \mathbf{L} + \mathbf{D} + \mathbf{U} \]

\[
\begin{bmatrix}
2 & 3 & 3\\
1 & 2 & 3\\
1 & 1 & 2
\end{bmatrix}
=
\begin{bmatrix}
0 & 0 & 0\\
1 & 0 & 0\\
1 & 1 & 0
\end{bmatrix}
+
\begin{bmatrix}
2 & 0 & 0\\
0 & 2 & 0\\
0 & 0 & 2
\end{bmatrix}
+
\begin{bmatrix}
0 & 3 & 3\\
0 & 0 & 3\\
0 & 0 & 0
\end{bmatrix}
\]

\subsubsection{Jacobi's method}
After decomposing matrix We can write the system of equations
\[ \mathbf{A}\mathbf{x} = \mathbf{b} \]
in the form:
\[ \mathbf{D}\mathbf{x} = -(\mathbf{L}+\mathbf{U})\mathbf{x}+\mathbf{b}
\]

If We assume that diagonal entries of matrix \textbf{A} are nonzero, then matrix \textbf{D} is nonsingular therefore We can propose such an iterative method:
\[ \mathbf{x}^{i+1} = -\mathbf{D}^{-1}(\mathbf{L}+\mathbf{U})\mathbf{x}^{(i)}+\mathbf{D}^{-1}\mathbf{x}
\]
This is the Jacobi's method.
We can rewritte this equation in the form of \textit{n} independent scalar equations:

\[
{x}_j^{i+1} = -\frac{1}{d_{jj}}(
\sum^n_{k=1} (l_{jk} + u_{jk})x_k^{(i)}+b_j)
) \]
Where $d_{jj}$; $l_{jk}$;  $u_{jk}$ are the elements of the respective matrixes \textbf{D}, \textbf{L}, \textbf{U}

Thanks to this We can do those computations in paraller, totally or partially if We are using a computer that enables a parallelization of the computations.

\paragraph{Converging}
Jacobi's method is convergent if We have strong diagonal dominance of the matrix \textbf{A}


\subsubsection{Gauss-Seidel method}
After decomposing matrix We can write the system of equations
\[ \mathbf{A}\mathbf{x} = \mathbf{b} \]
in the form:
\[ (\mathbf{L}+\mathbf{D})\mathbf{x} = -\mathbf{U}\mathbf{x}+\mathbf{b}
\]
Again We assume that \textbf{D} is nonsingular, in doing so We propose following iterative method:
\[ \mathbf{D}\mathbf{x}^{(i+1)} = -\mathbf{L}\mathbf{x}^{(i+1)}-\mathbf{U}\mathbf{x}^{(i)}+\mathbf{b}
\]
Since matrix \textbf{L} is subdiagonal, provided that We organse the calculation of elements of the vector $x^{(i_1)}$ in a proper way, it does not hurt that $x^{(i_1)}$ is on the right side of the equation.
In order to organise the calculation in the correct way We:
First take into account the structure of matrixes \textbf{D} and \textbf{L}:
\[
\begin{bmatrix}
d_{11}x_1^{(i_1)}\\
d_{22}x_2^{(i_1)}\\
d_{33}x_3^{(i_1)}\\
\vdots\\
d_{nn}x_n^{(i_1)}
\end{bmatrix}
=
-
\begin{bmatrix}
0 & 0 & 0 & \cdots & 0\\
l_{21} & 0 & 0 & \cdots & 0\\
l_{32} & l_{32} & 0 & \cdots & 0\\
\vdots & \vdots & \vdots & \vdots & \vdots\\
l_{n1} & l_{n2} & l_{n3} & \cdots & 0
\end{bmatrix}
\begin{bmatrix}
x_1^{(i_1)}\\
x_2^{(i_1)}\\
x_3^{(i_1)}\\
\vdots\\
x_n^{(i_1)}
\end{bmatrix}
-
\mathbf{w}^{(i)}
\]
Where
\[ \mathbf{w}^{(i)} = \mathbf{U}\mathbf{x}^{(i)} - \mathbf{b} \]

\newpage
So the order of calculations is as follows:
\[ x_1^{(i+1)} = -\frac{w_1^{(i)}}{d_{11}} \]
\[ x_2^{(i+1)} = -\frac{-l_{21}x_1^{(i+1)} - w_2^{(i)}}{d_{22}} \]
\[ x_3^{(i+1)} = -\frac{-l_{31}x_1^{(i+1)} -l_{32}x_2^{(i+1)} - w_3^{(i)}}{d_{33}} \]

And so on

As opposed to Jacobi's method, Gauss-Seidel method computations must be performed sequentially. Every subsequent scalar equations uses results from the computation of the previous equations.
\paragraph{Converging}
Gauss-Seidel method is convergent if the matrix \textbf{A} is strongly row or column diagonnaly dominant. If the matrix is symmetric, the method is also convergent if the matrix \textbf{A} is positive definite. This method is also usually faster convergent compared to Jacobi's method.

\subsubsection{Stop tests}
There are two ways to check when to terminate iterations of the methods We just discussed:
\begin{enumerate}
\item Check differences between two subsequent iteration points
\[
\| \mathbf{x}^{(i+1)} - \mathbf{x}^{(i)} \| \leq \delta
\]
Where $\delta$ is an assumed tolerance, (in our case $10^{-10}$). What We are really interested in though is whether the solution of the system of equation has required accuracy. If We want to check that We can additionaly check (higher level, more computationally demanding test):
\item Check differences between two subsequent iteration points
\[
\| \mathbf{A}\mathbf{x}^{(i+1)} - \mathbf{b}\| \leq \delta_2
\]
Where $\delta_2$ is an assumed tolerance. If this test is not passed then We can diminish the value of $\delta_2$ and continue with the iterations. Value of $\delta_2$ can not be too small since We are limited by the numerical errors.
\end{enumerate}

\subsubsection{\textbf{A} and \textbf{b}}
We have been given the following system:
\[
\systeme{10x_1-4x_2+x_3+2x_4=-8, 2x_1-6x_2+3x_3-x_4=-12, x_1+4x_2-12x_3+x_4=4, 2x_1+3x_2-3x_3-10x_4=1}
\]
Therefore our matrices will look like this:
\[
\textbf{A} = \begin{bmatrix}
10 & -4 & 1 & 2 \\
2 & -6 & 3 & -1 \\
1 & 4 & -12 & 1 \\
2 & 3 & -3 & -10
\end{bmatrix}
\textbf{b} = \begin{bmatrix}
-8 \\
-12 \\
4 \\
11 \\
\end{bmatrix}
\]

So:
\[
\textbf{L} = \begin{bmatrix}
0 & 0 & 0 & 0 \\
2 & 0 & 0 & 0 \\
1 & 4 & 0 & 0 \\
2 & 3 & -3 & 0
\end{bmatrix}
\textbf{D} = \begin{bmatrix}
10 & 0 & 0 & 0 \\
0 & -6 & 0 & 0 \\
0 & 0 & -12 & 0 \\
0 & 0 & 0 & -10
\end{bmatrix}
\textbf{U} = \begin{bmatrix}
0 & -4 & 1 & 2 \\
0 & 0 & 3 & -1 \\
0 & 0 & 0 & 1 \\
0 & 0 & 0 & 0
\end{bmatrix}
\]
\[
\textbf{D}^{-1} = \begin{bmatrix}
\frac{1}{10} & 0 & 0 & 0 \\
0 & - \frac{1}{6} & 0 & 0 \\
0 & 0 & - \frac{1}{12} & 0 \\
0 & 0 & 0 & - \frac{1}{10}
\end{bmatrix}
\]

\section{Results}

\subsection{Jacobi method result}
For system of equations We got in this task We got following results:
\\
Without the change in demanded tolerance:
\[ x = \left( \begin{array}{cc}
  -0.076776098668341 \\
   2.105784262642568 \\
   0.395344797635474 \\
   0.397776619764909
\end{array} \right)
\]
Error:
\[ r = \| \mathbf{A}\mathbf{x} - \mathbf{b}\| = 1.154375287358407e-10 \]
We managed to do this in \textbf{38} iterations of our loop, and the demanded tolerance did not change. (This required small change in code where We ommited the part of code responsible for changing demandedTolerance if $ \| \mathbf{A}x-b \| > \delta_2) $ )

With the change in demanded tolerance:
\[ x = \left( \begin{array}{cc}
  -0.076776098668341 \\
   2.105784262642568 \\
   0.395344797635474 \\
   0.397776619764909
\end{array} \right)
\]
Error:
\[ r = \| \mathbf{A}\mathbf{x} - \mathbf{b}\| = 5.770361548895147e-11 \]
We got this result in \textbf{37} iterations and demanded tolerance was equal to $2*10^{-10}$

Compared to matlab function
\[ x_{matlab} = \left( \begin{array}{cc}
  -0.076776098662498 \\
   2.105784262636790 \\
   0.395344797637659 \\
   0.397776619767240 \\
\end{array} \right)
\]
Matlab error:
\[ r = \| \mathbf{A}\mathbf{x} - \mathbf{b}\| = 4.070144838902081e-15 \]

For data from task 2a We got: \\
Without change in demanded tolerance:
\[ x_a = \left( \begin{array}{cc}
-0.930024655108186 \\
-1.223407298660663 \\
-1.273530574212508 \\
-1.230517757317628 \\
-1.151356031082747 \\
-1.056883669273682 \\
-0.952628310081466 \\
-0.834334594312996 \\
-0.683708806198363 \\
-0.450125157620744 \\
\end{array} \right)
\]
Error:
\[ r = \| \mathbf{A}\mathbf{x} - \mathbf{b}\| = 6.955194519943778e-11 \]
We managed to do this in \textbf{59} iterations of our loop, and the demanded tolerance did not change.

With change in demanded tolerance:
\[ x_a = \left( \begin{array}{cc}
-0.930024655104470 \\
-1.223407298653515 \\
-1.273530574202540 \\
-1.230517757305602 \\
-1.151356031069692 \\
-1.056883669260597 \\
-0.952628310069469 \\
-0.834334594303006 \\
-0.683708806191233 \\
-0.450125157617020 \\
\end{array} \right)
\]
Error:
\[ r = \| \mathbf{A}\mathbf{x} - \mathbf{b}\| = 1.699812218689508e-10 \]
We managed to do this in \textbf{57} iterations of our loop, and the demanded tolerance changed to $4*10^{-10}$

Compared to matlab $ A \ b $ function
\[ x_{matlab} = \left( \begin{array}{cc}
  -0.930024655110760 \\
  -1.223407298665612 \\
  -1.273530574219411 \\
  -1.230517757325956 \\
  -1.151356031091789 \\
  -1.056883669282743 \\
  -0.952628310089775 \\
  -0.834334594319914 \\
  -0.683708806203301 \\
  -0.450125157623323 \\
\end{array} \right)
\]
Matlab error:
\[ r = \| \mathbf{A}\mathbf{x} - \mathbf{b}\| = 3.662053438817790e-15 \]

For Matrix and Vector from task 2b) error of
\[ \| x^{(i+1)} - x^{(i)} \| \]
grew to infinity, therefore We could never achieve demanded tolerance, therefore the program executed infinite loop.

\subsubsection{Minimizing the demanded error}
We tried to minimize the demanded error using this steps:
\begin{enumerate}
\item We copied error from matlab function and pasted it into demanded tolerance.
\item If We did not get infinite loop We copied the newly acquired error and pasted it into demanded tolerance.
\item If We got inifinite loop We used the previous error as "minimal" demanded error.
\end{enumerate}
\paragraph{For original system of equations:}
We managed to get results with error as low as $1.776356839400250e-15$ with demanded tolerance = $3.202372833989376e-15$ for lower values program went into infinite loop.
Results for demanded tolerance = $3.202372833989376e-15$
For given matrix:
\[ x = \left( \begin{array}{cc}
  -0.076776098662498 \\
   2.105784262636790 \\
   0.395344797637659 \\
   0.397776619767240
\end{array} \right)
\]
Error:
\[ r = \| \mathbf{A}\mathbf{x} - \mathbf{b}\| = 3.108624468950438e-15 \]
We got this result in \textbf{53} iterations and demanded tolerance did not change.

\paragraph{For task 2a) system of equations:}
We managed to get results with error as low as
\[ 3.108624468950438e-15 \] with demanded tolerance:
\[ 3.202372833989376e-15 \]
for lower values program went into infinite loop.

For demanded tolerance = $3.202372833989376e-15$:
Results for 2a) system of equation

\[ x_a = \left( \begin{array}{cc}
-0.930024655110760 \\
-1.223407298665613 \\
-1.273530574219411 \\
-1.230517757325955 \\
-1.151356031091788 \\
-1.056883669282743 \\
-0.952628310089775 \\
-0.834334594319914 \\
-0.683708806203301 \\
-0.450125157623323
\end{array} \right)
\]
Error:
\[ r = \| \mathbf{A}\mathbf{x} - \mathbf{b}\| = 3.108624468950438e-15 \]
We managed to do this in \textbf{84} iterations of our loop, and the demanded tolerance did not change.
We managed to achieve slightly better (as in, the error was smaller) results than Matlab custom function.

\subsection{Gauss-Seidel method result}
For system of equations We got in this task We got following results:
\\
\[ x = \left( \begin{array}{cc}
  -0.076776098752996 \\
   2.105784262542701 \\
   0.395344797589652 \\
   0.397776619735316
\end{array} \right)
\]
Error:
\[ r = \| \mathbf{A}\mathbf{x} - \mathbf{b}\| = 6.999102842719934e-10 \]
We managed to do this in \textbf{19} iterations of our loop, and the demanded tolerance did not change.
Error:
\[ r = \| \mathbf{A}\mathbf{x} - \mathbf{b}\| = 6.999102842719934e-10 \]
We got this result in \textbf{19} iterations and demanded tolerance was equal to $2*10^{-10}$

For data from task 2a We got: \\
\[ x_a = \left( \begin{array}{cc}
-0.930024655049330 \\
-1.223407298590161 \\
-1.273530574151901 \\
-1.230517757273947 \\
-1.151356031055568 \\
-1.056883669259564 \\
-0.952628310076145 \\
-0.834334594312669 \\
-0.683708806199986 \\
-0.450125157622217 \\
\end{array} \right)
\]
Error:
\[ r = \| \mathbf{A}\mathbf{x} - \mathbf{b}\| = 5.311669113699570e-10 \]
We managed to do this in \textbf{28} iterations of our loop, and the demanded tolerance did not change.

For Matrix and Vector from task 2b) error of
\[ \| x^{(i+1)} - x^{(i)} \| \]
grew to infinity, therefore We could never achieve demanded tolerance, therefore the program executed infinite loop.

\subsubsection{Minimizing the demanded error}
I tried to minimize the demanded error using this steps:
\begin{enumerate}
\item I copied error from matlab function and pasted it into demanded tolerance.
\item If I did not get infinite loop I copied the newly acquired error and pasted it into demanded tolerance.
\item If I got inifinite loop I used the previous error as "minimal" demanded error.
\end{enumerate}
\paragraph{For original system of equations:}
We managed to get results with error as low as $1.776356839400250e-15$ with demanded tolerance = $ 1.986027322597818e-15 $ for lower values program went into infinite loop.
Results for demanded tolerance = $1.986027322597818e-15$
For given matrix:
\[ x = \left( \begin{array}{cc}
  -0.076776098662498 \\
   2.105784262636790 \\
   0.395344797637659 \\
   0.397776619767240
\end{array} \right)
\]
Error:
\[ r = \| \mathbf{A}\mathbf{x} - \mathbf{b}\| = 1.776356839400250e-15 \]
We got this result in \textbf{30} iterations and demanded tolerance did not change.

\paragraph{For task 2a) system of equations:}
We managed to get results with error as low as
\[ 1.538370149106851e-15 \] with demanded tolerance:
\[ 1.538370149106851e-15 \]
for lower values program went into infinite loop.

For demanded tolerance = $1.538370149106851e-15$:
Results for 2a) system of equation

\[ x_a = \left( \begin{array}{cc}
-0.930024655110760 \\
-1.223407298665613 \\
-1.273530574219411 \\
-1.230517757325955 \\
-1.151356031091788 \\
-1.056883669282743 \\
-0.952628310089775 \\
-0.834334594319914 \\
-0.683708806203301 \\
-0.450125157623323 \\
\end{array} \right)
\]
Error:
\[ r = \| \mathbf{A}\mathbf{x} - \mathbf{b}\| = 1.538370149106851e-15 \]
We managed to do this in \textbf{42} iterations of our loop, and the demanded tolerance did not change.
We managed to achieve slightly better (as in, the error was smaller) results than Matlab custom function.

\paragraph{Table}

\begin{center}
\resizebox{\textwidth}{!}{
\begin{tabular}{||c c c c c c||}
 \hline
 system of equations & method & demanded tolerance & error & iterations \\
 \hline
 task 3 system & Jacobi method & 10e-10 & 1.154375287358407e-10 & 38 \\
 \hline
 task 3 system & Jacobi method & 10e-10 & 5.770361548895147e-11 & 37 \\
 \hline
 task 3 system & Jacobi method & 3.202372833989376e-15 & 3.108624468950438e-15 & 53 \\
 \hline
 task 3 system & Gaussian-Seidel method & 10e-10 & 6.999102842719934e-10 & 19 \\
 \hline
 task 3 system & Gaussian-Seidel method & 1.776356839400250e-15 & 1.776356839400250e-15 & 30 \\
 \hline
 task 3 system & Matlab function & ? & 4.070144838902081e-15 & ? \\
 \hline
 task 2a) system & Jacobi method & 10e-10 & 6.955194519943778e-11 & 59 \\
 \hline
 task 2a) system & Jacobi method & 10e-10 & 1.699812218689508e-10 & 57 \\
 \hline
 task 2a) system & Jacob method & 3.202372833989376e-15 & 3.108624468950438e-15 & 84 \\
 \hline
 task 2a) system & Gaussian-Seidel method & 10e-10 & 5.311669113699570e-10 & 28 \\
 \hline
 task 2a) system & Gaussian-Seidel method & 1.538370149106851e-15 & 1.538370149106851e-15 & 42 \\
 \hline
 task 2a) system & Matlab function & ? & 3.662053438817790e-15 & ? \\
 \hline

\end{tabular}}
\end{center}

\section{Discussion of results}
\subsection{Comparison based on table}
Gaussian-Seidel method takes less iterations, can achieve smaller error and works under smaller demanded tolerance, that being said Jacobi method has one crucial advantage - it can be run in parallel. That means that even when Jacobi method takes twice or thrice amount of iteration us Gaussian-Seidel one if we just run it in paraller we can diminish this difference. In the examples we worked in, just two jacobi method running in paraller provide us with similar amount of iterations per one process as in Gaussian-Seidel one, and three or more processes give the edge to Jacobi method. With modern CPU's with at least 5 cores and correct implementation of algorithm, jacobi method reigns supreme.

\subsection{Convergence}
The biggest problem is of course the fact that those iterative methods do not work on every system of equation. Like the one from task 2b).
For Jacobi's method the matrix is convergent if it has strong diagonal dominance. \textbf{A}, i.e:
\begin{enumerate}
  \item \[ \| a_{ii} \| > \sum^n_{j=1, j \neq i} \|a_{ij} \| \]
  \item \[ \| a_{jj} \| > \sum^n_{i=1, i \neq j} \|a_{ij} \| \]
\end{enumerate}
For Gaussian-Seidel method the matrix is convergent if it has strong diagonal dominance \textbf{or} if the matrix \textbf{A} is symmetric and positivie definite.
\subsubsection{2b) task convergence }
Let's check strong diagonal dominance from matrix \textbf{A} from task 2b) in matlab:
\begin{simplechar}
\begin{lstlisting}
function d = checkIfDiagonallyDominant(Matrix)
    d = 1;
    [Rows, ~] = size(Matrix);
    for i = 1 : Rows
        rowsSum = 0;
        for j = 1 : Rows
            rowsSum = rowsSum + abs(Matrix(i, j));
        end
        rowsSum = rowsSum - Matrix(i, i);
        if Matrix(i, i) <= rowsSum
            d = 0;
            break
        end
    end
end
\end{lstlisting}
\end{simplechar}
checkIfDiagonallyDominant(matrixB(10)) returns '0', therefore matrix from task 2b) will \textbf{not} converge. And as expected it does not work when put in my jacobiLoop function.
\subsubsection{Iterations as function of size of Matrix}
I have used matrix from task 2a), demanded tolerance equal to 10e-10, and max size of matrix equal to 500. for both methods
\begin{center}
   \includegraphics[scale=0.75]{iterations.eps}
\end{center}
Gauss Seidel method again requires fewer iterations, around half the amount for jacobi method. For both methods number of iterations acts like logarithmic function. Therefore we can assume that for big matrices required number of iterations will not change so drastically.

\chapter{Problem 4 - QR method of finding eigenvalues}

\section{Problem}

\section{Theoretical introduction}

\section{Solution}

\section{Discussion of the result}

\chapter{Code appendix}

\section{Task 2 Code}

\subsection{Main function}
\begin{simplechar}
\begin{lstlisting}
function x = indicatedMethod(Matrix, Vector) % Name of the method as in the textbook
% x stands for obtained result
    [~,Columns] = size(Matrix); % We need to know how big the matrix is in next steps
    % notice the '~', since We assume We use square matrix, We do not need
    % to have another variable for number of rows since it is the same as
    % number of columns
    checkIfMatrixIsSquareMatrix(Matrix);
    [Matrix, Vector] = gaussianEliminationWithPartialPivoting(Columns, Matrix, Vector);
    % Change matrix to upper triangular matrix
    [Matrix, Vector, x] = backSubstitutionPhase(Columns, Matrix, Vector);
    % Get the solution
    x = iterativeResidualCorrection(Matrix, x, Vector); % Improve on the solution
end % end function
\end{lstlisting}
\end{simplechar}

\subsection{checkIfMatrixIsSquareMatrix}
\begin{simplechar}
\begin{lstlisting}
function checkIfMatrixIsSquareMatrix(Matrix)
    [Rows,Columns] = size(Matrix);
    if Rows ~= Columns
        error ('Matrix is not square matrix!');
    end % end if
end % end function
\end{lstlisting}
\end{simplechar}

\newpage
\subsection{gaussianEliminationWithPartialPivoting}
\begin{simplechar}
\begin{lstlisting}
function [Matrix, Vector] = gaussianEliminationWithPartialPivoting(Columns, Matrix, Vector)
    for j = 1 : Columns
            centralElement = max(Matrix(j:Columns,j));
            % We stay in the same row (j) but We change columns, as in the
            % textbook
            [Matrix, Vector] = partialPivoting(Matrix, Vector, j, centralElement, Columns);
            % ensures that a_kk != 0 and reduces errors
            [Matrix, Vector] = gaussianElimination(j, Columns, Matrix, Vector);
            % change matrix into upper triangular matrix
    end % end for
end % end function
\end{lstlisting}
\end{simplechar}

\subsection{partialPivoting}
\begin{simplechar}
\begin{lstlisting}
function [Matrix, Vector] = partialPivoting(Matrix, Vector, j, centralElement, Columns)
    for k = j : Columns
        partialPivotingSwapOneRow(Matrix, Vector, j, k, centralElement);
    end % end for
end % end function
\end{lstlisting}
\end{simplechar}

\subsection{partialPivotingSwapOneRow}
\begin{simplechar}
\begin{lstlisting}
function [Matrix, Vector] = partialPivotingSwapOneRow(Matrix, Vector, j, k, centralElement)
    if Matrix(k,j) == centralElement
    swapRowMatrix(Matrix, j, k); % swap jth row with kth row
    swapValueVector(Vector, j, k); % swap jth value with kth value
    end % end if
end % end function
\end{lstlisting}
\end{simplechar}


\subsection{swapRowMatrix}
\begin{simplechar}
\begin{lstlisting}
function Matrix = swapRowMatrix(Matrix, j, k)
    temp =  Matrix(j , :); % ' : ' denote "all elements in jth row"
    Matrix(j , :) = Matrix(k, :);
    Matrix(k, :) = temp; % temp equal to previous value of jth row
end
\end{lstlisting}
\end{simplechar}

\subsection{swapValueVector}
\begin{simplechar}
\begin{lstlisting}
function Vector = swapValueVector(Vector, j, k)
    temp = Vector(j);
    Vector(j) = Vector(k);
    Vector(k) = temp;  % temp equal to previous value of k element of vector
end % end function
\end{lstlisting}
\end{simplechar}

\subsection{gaussianElimination}
\begin{simplechar}
\begin{lstlisting}
function [Matrix, Vector] = gaussianElimination(j, Columns, Matrix, Vector)
    for i = j + 1 : Columns
        rowMultiplier = Matrix(i,j) / Matrix(j,j);
        [Matrix, Vector] = substractRows(Matrix, Vector, i, rowMultiplier, j, Columns);
    end % end for
end % end function
\end{lstlisting}
\end{simplechar}

\subsection{substractRows}
\begin{simplechar}
\begin{lstlisting}
function [Matrix, Vector] = substractRows(Matrix, Vector, i, rowMultiplier, j, Columns)
    Vector(i) = Vector(i) - rowMultiplier * Vector(j);
    for curentColumn = 1 : Columns
        Matrix(i,curentColumn) = Matrix(i,curentColumn) - rowMultiplier * Matrix(j, curentColumn);
    end % end for
end % end function
\end{lstlisting}
\end{simplechar}

\newpage
\subsection{backSubstitutionPhase}
\begin{simplechar}
\begin{lstlisting}
function [Matrix, Vector, x] = backSubstitutionPhase(Columns, Matrix, Vector)
    for k = Columns : -1 : 1
    % Start at final column and move by -1 each iteration until We reach 1
        equation = 0;
        for j = k+1 : Columns
            equation = equation + Matrix(k,j) * x(j, 1);
            % even though x is a vector We still need to put '1' to ensure
            % that number of columns in the first matrix matches number of
            % rows in second matrix
        end % end for

        x(k, 1) = (Vector(k,1) - equation) / Matrix(k,k);
        % even though x is a vector We still need to put '1' to ensure
        % that We do not exceed array bounds
    end % end for
end % end function
\end{lstlisting}
\end{simplechar}

\subsection{iterativeResidualCorrection}
\begin{simplechar}
\begin{lstlisting}
function x = iterativeResidualCorrection(Matrix, x, Vector)
    residuum = Matrix*x - Vector; % as in the book
    newResiduum = residuum;
    x = improveSolution(x, newResiduum, residuum, Matrix, Vector);
end % end function
\end{lstlisting}
\end{simplechar}

\subsection{improveSolution}
\begin{simplechar}
\begin{lstlisting}
function x = iterativeResidualCorrection(A, x, b)
    r = A*x - b;
    for i = 1 : 100
        [~, ~, deltaX] = solveSystem(A, r);
        newX = x - deltaX;
        r = A*newX - b;
        x = newX;
    end

end % end function

\end{lstlisting}
\end{simplechar}

\section{Task 3 code}
\begin{simplechar}
\begin{lstlisting}
function [x_j, x_g] = iterative(Matrix, Vector)
    [L, D, U, initial_x, whichIterationAreWeOnJ, whichIterationAreWeOnG, demandedToleranceJ, demandedToleranceG, flag, Rows] = initializeValues(Matrix);
    [x_j, whichIterationAreWeOnJ, demandedToleranceJ] = jacobiLoop(Matrix, L, D, U, initial_x, whichIterationAreWeOnJ, demandedToleranceJ, Vector, flag);
    [x_g, whichIterationAreWeOnG, demandedToleranceG] = gaussSeidelLoop(Matrix, L, D, U, initial_x, whichIterationAreWeOnG, demandedToleranceG, Vector, flag, Rows);
    dispFinalResults(x_j, x_g, demandedToleranceJ, demandedToleranceG, whichIterationAreWeOnJ, whichIterationAreWeOnG, Matrix, Vector);
end

\end{lstlisting}
\end{simplechar}

\subsection{initializeValues}
\begin{simplechar}
\begin{lstlisting}
function [L, D, U, initial_x, whichIterationAreWeOnJ, whichIterationAreWeOnG, demandedToleranceJ, demandedToleranceG, flag, Rows] = initializeValues(Matrix)
    [Rows, ~] = size(Matrix);
    [L, D, U] = decomposeMatrix(Matrix);
    initial_x = zeros(Rows, 1);
    whichIterationAreWeOnJ = 0;
    whichIterationAreWeOnG = 0;
    demandedToleranceJ = 10e-10; % as per task description
    demandedToleranceG = 10e-10; % as per task description
    flag = 0;
end
\end{lstlisting}
\end{simplechar}

\subsection{decomposeMatrix}
\begin{simplechar}
\begin{lstlisting}
function [L, D, U] = decomposeMatrix(Matrix)
    D = diag(diag(Matrix));
    U = triu(Matrix, 1); % Generates upper triangular part of matrix
    % where the second variable denotes on which diagonal of matrix should we
    % start
    L = tril(Matrix, -1); % Generates lower triangular part of matrix
    % where the second variable denotes on which diagonal of matrix should we
    % start
end
\end{lstlisting}
\end{simplechar}

\subsection{jacobiLoop}
\begin{simplechar}
\begin{lstlisting}
function [x_j, whichIterationAreWeOn, demandedTolerance]  = jacobiLoop(Matrix, L, D, U, initial_x, whichIterationAreWeOn, demandedTolerance, Vector, flag)
    while flag ~= 1 % flag denotes whether norm(Matrix*x_g-Vector) <= demandedTolerance
        [x_j, whichIterationAreWeOn, demandedTolerance, flag, initial_x] = jacobiInsideLoop(Matrix, L, D, U, initial_x, whichIterationAreWeOn, demandedTolerance, Vector);
    end
end
\end{lstlisting}
\end{simplechar}

\subsection{jacobiInsideLoop}
\begin{simplechar}
\begin{lstlisting}
function [x_j, whichIterationAreWeOn, demandedTolerance, flag, initial_x] = jacobiInsideLoop(Matrix, L, D, U, initial_x, whichIterationAreWeOn, demandedTolerance, Vector)
    x_j = jacobiEquation(D, L, U, initial_x, Vector);
    [flag, demandedTolerance] = checkError(x_j, initial_x, demandedTolerance, Matrix, Vector);
    [initial_x, whichIterationAreWeOn] = endOfLoop(x_j, whichIterationAreWeOn);
end
\end{lstlisting}
\end{simplechar}

\subsection{jacobiEquation}
\begin{simplechar}
\begin{lstlisting}
function x = jacobiEquation(D, L, U, initial_x, Vector)
    x = - D \ ( L + U ) * initial_x + D \ Vector; % As per formula
    % We will be using D \ Vector and D \ ( ) instead of inverseD since
    % this is faster according to matlab
end
\end{lstlisting}
\end{simplechar}

\subsection{gaussSeidelLoop}
\begin{simplechar}
\begin{lstlisting}
function [x_g, whichIterationAreWeOn, demandedTolerance]  = gaussSeidelLoop(Matrix, L, D, U, initial_x, whichIterationAreWeOn, demandedTolerance, Vector, flag, Rows)
    while flag ~= 1 % flag denotes whether norm(Matrix*x_g-Vector) <= demandedTolerance
        [x_g, whichIterationAreWeOn, demandedTolerance, flag, initial_x] = gaussiInsideLoop(Matrix, L, D, U, initial_x, whichIterationAreWeOn, demandedTolerance, Vector, Rows);
    end
end
\end{lstlisting}
\end{simplechar}

\subsection{gaussiInsideLoop}
\begin{simplechar}
\begin{lstlisting}
function [x_j, whichIterationAreWeOn, demandedTolerance, flag, initial_x] = gaussiInsideLoop(Matrix, L, D, U, initial_x, whichIterationAreWeOn, demandedTolerance, Vector, Rows)
    x_j = gaussSeidelEquation(D, L, U, initial_x, Vector, Rows);
    [flag, demandedTolerance] = checkError(x_j, initial_x, demandedTolerance, Matrix, Vector);
    [initial_x, whichIterationAreWeOn] = endOfLoop(x_j, whichIterationAreWeOn);
end
\end{lstlisting}
\end{simplechar}

\subsection{gaussSeidelEquation}
\begin{simplechar}
\begin{lstlisting}
function x_g = gaussSeidelEquation(D, L, U, initial_x, Vector, Rows)
    W = U*initial_x - Vector;
    x_g(1, 1) = -W(1, 1) / D(1,1);
    for i = 2 : Rows
        x_g(i, 1) = calculateNominator(i, L, x_g, W) / D(i, i);
    end
end
\end{lstlisting}
\end{simplechar}

\subsection{checkError}
\begin{simplechar}
\begin{lstlisting}
function [flag, demandedTolerance] = checkError(x, initial_x, demandedTolerance, Matrix, Vector)
    flag = 0;
    currentError = norm(x - initial_x);
    if currentError <= demandedTolerance
        currentError = norm(Matrix*x-Vector);
        if currentError <= demandedTolerance % if sequence as per textbook
            flag = 1;
        else
            demandedTolerance = demandedTolerance * 2; % arbitrary value
        end
    end
end
\end{lstlisting}
\end{simplechar}

\subsection{endOfLoop}
\begin{simplechar}
\begin{lstlisting}
function [initial_x, whichIterationAreWeOn, flag] = endOfLoop(x, whichIterationAreWeOn)
    initial_x = x;
    whichIterationAreWeOn = whichIterationAreWeOn + 1;
    flag = 0;
end
\end{lstlisting}
\end{simplechar}

\subsection{dispFinalResults}
\begin{simplechar}
\begin{lstlisting}
function dispFinalResults(x_j, x_g, demandedToleranceJ, demandedToleranceG, whichIterationAreWeOnJ, whichIterationAreWeOnG, Matrix, Vector)
    disp("Final demandedTolerance for Jacobi method");
    disp(demandedToleranceJ);
    disp("Final demandedTolerance for Gaussian-Seidel method:");
    disp(demandedToleranceG);
    disp("Final Iteration for Jacobi method: ");
    disp(whichIterationAreWeOnJ);
    disp("Final Iteration for Gaussian-Seidel method: ");
    disp(whichIterationAreWeOnG);
    disp("Error for Jacobi method:");
    disp(norm(Matrix*x_j - Vector));
    disp("Error for Gaussian-Seidel method:");
    disp(norm(Matrix*x_g - Vector));
    disp("A\b error:");
    disp(norm(Matrix * (Matrix\Vector) - Vector));
    disp("Answer for Jacobi method: ");
    disp(x_j);
    disp("Answer for Gaussian-Seidel method: ");
    disp(x_g);
end
\end{lstlisting}
\end{simplechar}

\subsection{plotIterations}
\begin{simplechar}
\begin{lstlisting}
function plotIterations()
    maxMatrixSize = 500;
    iterationsJ = zeros(maxMatrixSize);
    iterationsG = zeros(maxMatrixSize);
    for i = 1 : maxMatrixSize
        [~, ~, whichIterationAreWeOnJ, whichIterationAreWeOnG] = iterative(matrixA(i), vectorA(i));
        iterationsJ(i) = whichIterationAreWeOnJ;
        iterationsG(i) = whichIterationAreWeOnG;
    end
    nexttile
    plot(iterationsJ, '.');
    title('Number of iterations for different sizes of matrices for Jacobi method');
    xlabel('Size of matrix A');
    ylabel('Number of iterations');
    nexttile
    plot(iterationsG, '.');
    title('Number of iterations for different sizes of matrices for Gauss-Seidel method');
    xlabel('Size of matrix A');
    ylabel('Number of iterations');

end
\end{lstlisting}
\end{simplechar}

\begin{thebibliography}{9}
\bibitem{texbook}
Piotr Tatjewski (2014) \emph{Numerical Methods}, Oficyna Wydawnicza Politechniki Warszawskiej
\end{thebibliography}

\end{document}