%2multibyte Version: 5.50.0.2953 CodePage: 1252
%\def\paperauthorB{Author_B_fullname}
% you can add more authors using alphabets after \paperauthor
%------------------------------------------------------------------------------------------
% Do not delete this part starting from here and up to .....
%\setcounter{page}{1}
%\input{../../expages.tex}\setpagenumber{1}
%\input{expages.tex}\setpagenumber{1}
% Delete if you cannot get PS fonts working on your system.
% pdf-tex settings: detect automatically if run by latex or pdflatex
% compiling with pdflatex
% compiling with latex
% hyperrefs are active in the pdf file after conversion
%------------------------------------------------------------------------------------------
% This command replaces \name{The DAFx Crew} 
%\usepackage{layout}
%------------------------------------------------------------------------------------------
%%%%%%%%%%%%%%%%%%%%%%% to here %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


\documentclass[oneside]{article}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\usepackage{amsfonts}
\usepackage{icma2021,amssymb,amsmath,mathrsfs}
\usepackage{mathptmx}
\usepackage[pdftex,
pdftitle={\papertitle},
colorlinks=false, bookmarksnumbered, pdfstartview= XYZ	]{hyperref}
\usepackage[pdftex]{graphicx}
\usepackage[figure,table]{hypcap}
\usepackage[dvips]{graphicx}
\usepackage[dvips,
colorlinks=false,           bookmarksnumbered, pdfstartview= XYZ	]{hyperref}
\usepackage[figure,table]{hypcap}

\setcounter{MaxMatrixCols}{10}
%TCIDATA{OutputFilter=LATEX.DLL}
%TCIDATA{Version=5.50.0.2953}
%TCIDATA{Codepage=1252}
%TCIDATA{<META NAME="SaveForMode" CONTENT="1">}
%TCIDATA{BibliographyScheme=Manual}
%TCIDATA{LastRevised=Tuesday, October 12, 2021 10:29:14}
%TCIDATA{<META NAME="GraphicsSave" CONTENT="32">}

\def\papertitle{Full title of the paper}
\def\paperauthorA{AuthorAfullname}
\ninept
\newif\ifpdf
\ifx\pdfoutput\relax
\else
\ifcase\pdfoutput
\pdffalse
\else
\pdftrue
\fi
\ifpdf 
\pdfcompresslevel=9
\else 
\fi
\newcommand\dsum{\displaystyle\sum}
\newcommand\dprod{\displaystyle\prod}
\affiliation{Khelladi Samia, Benterki Djamel}
{Laboratory of Fundamental and Numerical Mathematics LMFN,\\ Faculty of Sciences, University of Setif-1, Ferhat Abbas,\\ 19000 Setif, Algeria}
\newtheorem{theorem}{Theorem}[section]
\newtheorem{lemma}[theorem]{Lemma}
\input{tcilatex}

\begin{document}

\title{Hybrid conjugate gradient-BFGS methods based on Wolfe line search}
\author{Khelladi Samia and Benterki Djamel \and {Laboratory of Fundamental
an Numerical Mathematics, }Faculty of Sciences, University of Setif-1,
Ferhat Abbas, 19000 Setif, Algeria}
\maketitle

\begin{abstract}
In this paper, we present some hybrid methods for solving unconstrained
optimization problems. These methods are defined using proper combinations
of the search directions and included parameters in conjugate gradient and
quasi-Newton method of Broyden--Fletcher--Goldfarb--Shanno (CG-BFGS). Their
global convergence under the Wolfe line search is analyzed for general
objective functions. Numerical experiments show the superiority of the
modified hybrid (CG-BFGS) method with respect to some existing methods.

\textbf{Keywords:} Unconstrained optimization, Global convergence, Conjugate
gradient methods, Quasi-Newton methods, Wolfe line search.
\end{abstract}

%\layout

% more pdf-tex settings:
\ifpdf % used graphic file format for pdflatex
\DeclareGraphicsExtensions{.png,.jpg,.pdf} \else  
% used graphic file format for latex
\DeclareGraphicsExtensions{.eps} \fi

\section{Introduction}

Conjugate gradient methods are very important ones for solving unconstrained
optimization problems, especially for large scale problems. It is well known
that Fletcher-Reeves (FR) \cite{FR64}, Conjugate Descent (CD) \cite{F87} and
Dai-Yuan (DY) \cite{DY99} conjugate gradient methods have strong convergence
properties, but they may not perform well in practice. On the other hand,
Hestnes-Stiefel (HS) \cite{HS52}, Polak-Ribiere-Polyak (PRP) \cite{PR69,P69}
and Liu-Storey (LS) \cite{LS91} conjugate gradient methods may not converge
in general, but they often perform better than FR, CD and DY. To combine the
best numerical performances of the LS method and the global convergence
properties of the CD method, Yang et al. \cite{YL2013} proposed a hybrid
LS-CD method. Dai and Liao \cite{DL2001} proposed an efficient conjugate
gradient method (Dai-Liao type method). Later, some more efficient Dai-Liao
type conjugate gradient method, known as DHSDL and DLSDL were proposed in 
\cite{ZH17} .

The rest of this paper is organized as follows. In Section 2, we give
various possibilities to determine the step size and the search direction. A
hybridization of the conjugate gradient method (CG) and the BFGS method will
also be presented. In Section 3, we consider the modification of LSCD
method, termed as MLSCD and the modification of (DHSDL and DLSDL) termed as
MMDL \cite{SB2018} and we prove the global convergence using the Wolfe line
search instead of backtracking line search used by the authors in \cite%
{SB2018}. In Section 4, we consider the hybrid method BFGS-CG termed as
H-BFGS-CG1 in \cite{SB2018} and we prove the global convergence with the
Wolfe line search termed WH-BFGS-CG. In section 5, we report some numerical
results and compare the performance of the different considered methods.
Finally, we give some conclusions to end this paper.

\bigskip

\section{Preliminaries}

Consider the following unconstrained optimization problem%
\begin{equation}
\min f(x),\text{ \ \ \ }x\in 
%TCIMACRO{\U{211d} }%
%BeginExpansion
\mathbb{R}
%EndExpansion
^{n},  \label{1}
\end{equation}

where $f:$ $%
%TCIMACRO{\U{211d} }%
%BeginExpansion
\mathbb{R}
%EndExpansion
^{n}\longrightarrow 
%TCIMACRO{\U{211d} }%
%BeginExpansion
\mathbb{R}
%EndExpansion
$ is a continuously differentiable function. Let $g_{k}$ be the gradient of $%
f(x)$ at the current iterative point $x_{k}$, then the classical conjugate
gradient method for $(\ref{1})$ is given by%
\begin{equation}
x_{k+1}=x_{k}+\alpha _{k}d_{k}\text{ ,}  \label{2}
\end{equation}

in which $\alpha _{k}>0$ is the step size found by one of the line search
methods, and $d_{k}$ is the search direction defined by%
\begin{equation}
d_{k}=\left\{ 
\begin{array}{l}
-g_{0},\text{ \ \ \ \ \ \ \ \ \ \ \ \ \ \ }k=0, \\ 
-g_{k}+\beta _{k}d_{k-1},\text{ \ }k\geq 1,%
\end{array}%
\right.  \label{3}
\end{equation}

where $\beta _{k}$ is an appropriately defined real scalar, known as the
conjugate gradient parameter.

Since Fletcher and Reeves introduced the nonlinear conjugate gradient method
in $1964$, many formulae have been proposed using various modifications of
the conjugate gradient direction $d_{k}$ and the parameter $\beta _{k}.$ The
most popular parameters\ $\beta _{k}$ are:%
\begin{eqnarray}
\beta _{k}^{FR} &=&\frac{\parallel g_{k}\parallel ^{2}}{\parallel
g_{k-1}\parallel ^{2}},\text{ \ \ \ }\beta _{k}^{CD}=-\frac{\parallel
g_{k}\parallel ^{2}}{g_{k-1}^{T}d_{k-1}},\text{ \ \ }\beta _{k}^{DY}=\frac{%
\parallel g_{k}\parallel ^{2}}{y_{k-1}^{T}d_{k-1}}\text{,\ }  \notag \\
\beta _{k}^{HS} &=&\frac{g_{k}^{T}y_{k-1}}{y_{k-1}^{T}d_{k-1}},\text{ \ \ }%
\beta _{k}^{PRP}=\frac{g_{k}^{T}y_{k-1}}{\parallel g_{k-1}\parallel ^{2}},%
\text{ \ \ }\beta _{k}^{LS}=-\frac{g_{k}^{T}y_{k-1}}{g_{k-1}^{T}d_{k-1}}, 
\notag \\
\beta _{k}^{DHSDL} &=&\frac{\parallel g_{k}\parallel ^{2}-\frac{\parallel
g_{k}\parallel }{\parallel g_{k-1}\parallel }\mid g_{k}^{T}g_{k-1}\mid }{\mu
\mid g_{k}^{T}d_{k-1}\mid +d_{k-1}^{T}y_{k-1}}-t\frac{g_{k}^{T}s_{k-1}}{%
d_{k-1}^{T}y_{k-1}},\text{ \ }\mu >1,\text{ }t>0,  \notag \\
\beta _{k}^{DLSDL} &=&\frac{\parallel g_{k}\parallel ^{2}-\frac{\parallel
g_{k}\parallel }{\parallel g_{k-1}\parallel }\mid g_{k}^{T}g_{k-1}\mid }{\mu
\mid g_{k}^{T}d_{k-1}\mid -d_{k-1}^{T}g_{k-1}}-t\frac{g_{k}^{T}s_{k-1}}{%
d_{k-1}^{T}y_{k-1}},\text{ \ }\mu >1,\text{ }t>0,  \notag
\end{eqnarray}

where $y_{k-1}=g_{k}-g_{k-1},$ $s_{k-1}=x_{k}-x_{k-1}$ and $\parallel
.\parallel $ denotes the Euclidean vector norm.

In this paper, the step size $\alpha _{k}$ is determined using the following
Wolfe line search conditions%
\begin{equation}
f(x_{k}+\alpha _{k}d_{k})\leq f(x_{k})+\rho \alpha _{k}g_{k}^{T}d_{k},
\label{4}
\end{equation}%
\begin{equation*}
g_{k+1}^{T}d_{k}\geq \sigma g_{k}^{T}d_{k},\text{ }0<\rho <\sigma <1.
\end{equation*}

We consider hybrid CG methods where the search direction $d_{k}$, $k\geq 1,$
from $(\ref{3})$ is modified using one of the following tow rules \cite%
{SB2018}%
\begin{equation}
d_{k}=\mathcal{D}(\beta _{k},g_{k},d_{k-1})=-\left( 1+\beta _{k}\frac{%
g_{k}^{T}d_{k-1}}{\left\Vert g_{k}\right\Vert ^{2}}\right) g_{k}+\beta
_{k}d_{k-1}  \label{5}
\end{equation}

\begin{equation}
d_{k}=\mathcal{D}_{1}(\beta _{k},g_{k},d_{k-1})=-B_{k}g_{k}+\mathcal{D}%
(\beta _{k},g_{k},d_{k-1})  \label{6}
\end{equation}

and the conjugate gradient parameter $\beta _{k}$ is defined using some
proper combinations of the parameters $\beta _{k}$ given above and already
defined hybridizations of these parameters.

On the other hand, the search direction $d_{k}$ in quasi-Newton methods is
obtained as a solution of the linear algebraic system%
\begin{equation*}
B_{k}d_{k}=-g_{k}.
\end{equation*}

Where $B_{k}$ is an approximation of the Hessian. The initial approximation
is the identity matrix $(B_{0}=I)$ and the subsequent updates $B_{k}$ are
defined by an appropriate formula.

Here, we are interested in the BFGS update formula, defined by%
\begin{equation}
B_{k+1}=B_{k}+\frac{y_{k}y_{k}^{T}}{s_{k}^{T}y_{k}}-\frac{%
B_{k}s_{k}s_{k}^{T}B_{k}}{s_{k}^{T}B_{k}s_{k}},  \label{7}
\end{equation}

where $s_{k}=x_{k+1}-x_{k},$ \ \ $y_{k}=g_{k+1}-g_{k}.$ The next secant
equation must hold%
\begin{equation}
B_{k+1}s_{k}=y_{k},  \label{8}
\end{equation}

which is possible only if the curvature condition%
\begin{equation}
y_{k}^{T}s_{k}>0  \label{9}
\end{equation}

is satisfied.

\section{ Modification of LSCD, DHSDL and DLSDL methods}

\subsection{ A modified LSCD conjugate gradient method}

We consider the modification of LSCD method, defined in \cite{YL2013} by%
\begin{equation*}
\beta _{k}^{LSCD}=\max \left\{ 0,\min \left\{ \beta _{k}^{LS},\beta
_{k}^{CD}\right\} \right\} ,
\end{equation*}

\begin{equation*}
d_{k}=\left\{ 
\begin{array}{ll}
-g_{0}\text{ } & k=0 \\ 
d_{k}=-g_{k}+\beta _{k}^{LSCD}d_{k-1} & k\geq 1,%
\end{array}%
\right.
\end{equation*}

and define the MLSCD method \cite{SB2018} with the search direction%
\begin{equation*}
d_{k}=\mathcal{D}(\beta _{k}^{LSCD},g_{k},d_{k-1}).
\end{equation*}

Now, we give the algorithm of this method using the Wolfe line search.

\subsubsection{Algorithm WMLSCD}

\begin{itemize}
\item Step0: Given a starting point $x_{0}$ and a parameter $0<\varepsilon
<1.$

\item Step1: Set \textbf{\ }$k=0$ and compute $d_{0}=-g_{0}.$

\item Step2: If $\left\Vert g_{k}\right\Vert \leq \varepsilon,$ STOP; else
go to Step3.

\item Step3: Find the step size $\alpha _{k}\in ]0,1]$ using the Wolfe line
search.

\item Step4: Compute $x_{k+1}=x_{k}+\alpha _{k}d_{k}.$

\item Step5: Compute $y_{k}=g_{k+1}-g_{k}$ and go to Step6.

\item Step6: Compute 
\begin{eqnarray*}
\beta _{k+1}^{LS} &=&-\dfrac{y_{k}^{T}g_{k+1}}{g_{k}^{T}d_{k}}, \mbox{  }%
\beta _{k+1}^{CD}=-\dfrac{\left\Vert g_{k+1}\right\Vert 
%TCIMACRO{\U{b2} }%
%BeginExpansion
{{}^2}
%EndExpansion
}{g_{k}^{T}d_{k}}, \\
\beta _{k+1}^{LSCD} &=&\max \left\{ 0,\min \left\{ \beta _{k+1}^{LS},\beta
_{k+1}^{CD}\right\} \right\}.
\end{eqnarray*}

\item Step7: Compute the search direction $d_{k+1}=\mathcal{D}(\beta
_{k+1}^{LSCD},g_{k+1},d_{k})$.

\item Step8: Let $k:=k+1$ and go to Step2.
\end{itemize}

\bigskip

\subsubsection{Convergence of the WMLSCD conjugate gradient method}

It is easy to prove the next theorem.

\begin{theorem}
\label{th3.1} Let $\beta _{k}$ be any CG parameter. Then, the search
direction $d_{k}=\mathcal{D}(\beta _{k},g_{k},d_{k-1})$ satisfies 
\begin{equation*}
g_{k}^{T}d_{k}=-\left\Vert g_{k}\right\Vert ^{2}.
\end{equation*}
\end{theorem}

To prove the global convergence of the WMLSCD method, we need the following
assumptions.

\textbf{Assumption 3.1} The level set $\mathcal{L}=\left\{ x\in 
%TCIMACRO{\U{211d} }%
%BeginExpansion
\mathbb{R}
%EndExpansion
^{n}/f(x)\leq f(x_{0})\right\} $ is bounded.

\textbf{Assumption 3.2} The function $f$ is continuously differentiable in
some neighbourhood $\mathcal{N}$ of $\mathcal{L}$ and its gradient is
Lipschitz continuous. Namely, there exists a constant $L>0$ such that

\begin{equation}
\left\Vert g(x)-g(y)\right\Vert \leq L\left\Vert x-y\right\Vert ,\text{ for
all }x,y\in \mathcal{N}.  \label{10}
\end{equation}

It is well known that if Assumption 3.2 holds, then there exists a positive
constant $\gamma $, such that

\begin{equation}
\left\Vert g_{k}\right\Vert \leq \gamma ,\forall k  \label{11}
\end{equation}

The next lemma, often called the Zoutendijk condition \cite{ZT70}, is used
to prove the global convergence of nonlinear CG method.

\begin{lemma}
\cite{SB2018} \label{lem3.1} Let the Assumption 3.1 and Assumption 3.2 be
satisfied. Let the sequence $\left\{ x_{k}\right\} $ be generated by the
MLSCD method with the Wolfe line search. Then it holds that 
\begin{equation}
\overset{\infty }{\underset{k=1}{\sum }}\frac{\left\Vert g_{k}\right\Vert
^{4}}{\left\Vert d_{k}\right\Vert ^{2}}<+\infty  \label{12}
\end{equation}
\end{lemma}

\begin{theorem}
\label{th3.2} Let the Assumption 3.1 and Assumption 3.2 hold. Then, the
sequence $\left\{ x_{k}\right\} $ generated by the WMLSCD method with the
Wolfe line search satisfies

\begin{equation}
\liminf\limits_{k\rightarrow \infty }\left\Vert g_{k}\right\Vert =0
\label{13}
\end{equation}
\end{theorem}

\subsection{A modified DHSDL and DLSDL conjugate gradient method}

In this part, we have the hybrid MMDL method, proposed in \cite{SB2018},
which is defined by the search direction $d_{k}$ as follows

\begin{equation*}
\beta _{k}^{MMDL}=\max \left\{ 0,\min \left\{ \beta _{k}^{DHSDL},\beta
_{k}^{DLSDL}\right\} \right\}
\end{equation*}

\begin{equation*}
d_{k}=\mathcal{D}(\beta _{k}^{MMDL},g_{k},d_{k-1}).
\end{equation*}

We give the algorithm of this method where we have changed the backtracking
line search by the Wolfe line search.

\subsubsection{Algorithm WMMDL}

\begin{itemize}
\item Step0: Given a starting point $x_{0}$, a parameter $0<\varepsilon <1$
and $\mu >1.$

\item Step1: Set \textbf{\ }$k=0$ and compute $d_{0}=-g_{0}.$

\item Step2: If $\left\Vert g_{k}\right\Vert \leq \varepsilon ,$ STOP; else
go to Step3.

\item Step3: Find the step size $\alpha _{k}\in ]0,1]$ using the Wolfe line
search.

\item Step4: Compute $x_{k+1}=x_{k}+\alpha _{k}d_{k}.$

\item Step5: Compute $y_{k}=g_{k+1}-g_{k}$, $s_{k}=x_{k+1}-x_{k}$ and go to
Step6.

\item Step6: Compute 
\begin{equation*}
\beta _{k+1}^{DHSDL}=\frac{\parallel g_{k+1}\parallel ^{2}-\frac{\parallel
g_{k+1}\parallel }{\parallel g_{k}\parallel }\mid g_{k+1}^{T}g_{k}\mid }{\mu
\mid g_{k+1}^{T}d_{k}\mid +d_{k}^{T}y_{k}}-\alpha _{k}\frac{g_{k+1}^{T}s_{k} 
}{d_{k}^{T}y_{k}}
\end{equation*}

\begin{equation*}
\beta _{k+1}^{DLSDL}=\frac{\parallel g_{k+1}\parallel ^{2}-\frac{\parallel
g_{k+1}\parallel }{\parallel g_{k}\parallel }\mid g_{k+1}^{T}g_{k}\mid }{\mu
\mid g_{k+1}^{T}d_{k}\mid -d_{k}^{T}g_{k}}-\alpha _{k}\frac{g_{k+1}^{T}s_{k} 
}{d_{k}^{T}y_{k}}
\end{equation*}

\begin{equation*}
\beta _{k+1}^{MMDL}=\max \left\{ 0,\min \left\{ \beta _{k+1}^{DHSDL},\beta
_{k+1}^{DLSDL}\right\} \right\}.
\end{equation*}

\item Step7: Compute the search direction $d_{k+1}=\mathcal{D}(\beta
_{k+1}^{MMDL},g_{k+1},d_{k})$.

\item Step8: Let $k:=k+1$ and go to Step2.
\end{itemize}

\bigskip

\subsubsection{Convergence of the WMMDL conjugate gradient method}

The following theorem prove the global convergence of the WMMDL method.

\begin{theorem}
\label{th4.1} Let the Assumption 3.1 and Assumption 3.2 be satisfied. Then
the sequence $\left\{ x_{k}\right\} $ generated by the WMMDL method with the
Wolfe line search satisfies 
\begin{equation*}
\liminf\limits_{k\rightarrow \infty }\left\Vert g_{k}\right\Vert =0
\end{equation*}
\end{theorem}

\section{Hybrid BFGS-CG methods}

It is known that conjugate gradient method are better compared to the
quasi-Newton method in terms of the CPU time. In addition, BFGS is more
costly in terms of the memory storage requirements than CG. On the other
hand, the quasi-Newton methods are better in terms of the number of
iterations and the number of function evaluations. For this purpose, various
hybridizations of quasi-Newton methods and CG methods have been proposed by
various researchers.

\subsection{WH-BFGS-CG method}

P. S. Stanimirovic et al. proposed in \cite{SB2018} a three-term hybrid
BFGS-CG method, called H-BFGS-CG, defined by the search direction%
\begin{equation*}
d_{k}=\left\{ 
\begin{array}{ll}
-B_{k}g_{k}\text{ ,} & k=0 \\ 
\mathcal{D}_{1}(\beta _{k+1}^{LSCD},g_{k},d_{k-1}), & k\geq 1%
\end{array}%
\right.
\end{equation*}

\bigskip

The following algorithm correspond to this method, where we have changed the
backtracking line search by the Wolfe line search.

\subsubsection{Algorithm WH-BFGS-CG}

\begin{itemize}
\item Step0: Given a starting point $x_{0}$ and a parameter $0<\varepsilon
<1.$

\item Step1: Set \textbf{\ }$k=0$ and compute $g_{0},$ $B_{0}=I,$ $%
d_{0}=-B_{0}g_{0}.$

\item Step2: If $\left\Vert g_{k}\right\Vert \leq \varepsilon,$ STOP; else
go to Step3.

\item Step3: Find the step size $\alpha _{k}\in ]0,1]$ using the Wolfe line
search.

\item Step4: Compute $x_{k+1}=x_{k}+\alpha _{k}d_{k}.$

\item Step5: Compute $y_{k}=g_{k+1}-g_{k}$, $s_{k}=x_{k+1}-x_{k} $ and go to
Step6.

\item Step6: Compute 
\begin{equation*}
\beta _{k+1}^{LS}=-\dfrac{y_{k}^{T}g_{k+1}}{g_{k}^{T}d_{k}},\text{ }\beta
_{k+1}^{CD}=-\dfrac{\left\Vert g_{k+1}\right\Vert 
%TCIMACRO{\U{b2} }%
%BeginExpansion
{{}^2}
%EndExpansion
}{g_{k}^{T}d_{k}},
\end{equation*}

\begin{equation*}
\beta _{k+1}^{LSCD}=\max \left\{ 0,\min \left\{ \beta _{k+1}^{LS},\beta
_{k+1}^{CD}\right\} \right\}.
\end{equation*}

\item Step7: Compute $B_{k+1}$ using $(\ref{7}).$

\item Step8: Compute the search direction $d_{k+1}=\mathcal{D}_{1}(\beta
_{k+1}^{LSCD},g_{k+1},d_{k}).$

\item Step9: Let $k:=k+1$ and go to Step2.
\end{itemize}

\subsection{Convergence analysis of WH-BFGS-CG method}

\textbf{Assumption 4.1}:

$H1:$ The objective function $f$ is twice continuously differentiable.

$H2:$ The level set $\mathcal{L}$ is convex. Moreover, there exist positive
constants $c_{1}$ and $c_{2}$ such that%
\begin{equation*}
c_{1}\left\Vert z\right\Vert ^{2}\leq z^{T}H(x)z\leq c_{2}\left\Vert
z\right\Vert ^{2}, \mbox{ for all } z\in \mathbb{R}^{n} \mbox{ and } x\in 
\mathcal{L},
\end{equation*}

where $H(x)$ is the Hessian of $f$.

$H3:$ The gradient $g$ is Lipschitz continuous at the point $x^{\ast } $,
that is, there exists a positive constant $c_{3}$ satisfying 
\begin{equation*}
\left\Vert g(x)-g(x^{\ast })\right\Vert \leq c_{3}\left\Vert x-x^{\ast
}\right\Vert,
\end{equation*}

for all $x$ in a neighbourhood of $x^{\ast }.$

\begin{theorem}
\label{th5.1} \cite{BN89} Let $\left\{ B_{k}\right\} $ be generated by the
BFGS update formula $(\ref{7})$, where $s_{k}=x_{k+1}-x_{k},$ $%
y_{k}=g_{k+1}-g_{k}.$ Assume that the matrix $B_{k}$ is symmetric positive
definite and satisfies $(\ref{8})$ and $(\ref{9})$ for all $k.$ Furthermore,
assume that $\left\{ s_{k}\right\} $ and $\left\{ y_{k}\right\} $ satisfy
the inequality 
\begin{equation*}
\frac{\left\Vert y_{k}-G_{\ast }s_{k}\right\Vert }{\left\Vert
s_{k}\right\Vert }\leq \epsilon _{k},
\end{equation*}

for some symmetric positive definite matrix $G_{\ast }$ and for some
sequence $\left\{ \epsilon _{k}\right\} $ possessing the property 
\begin{equation*}
\overset{\infty }{\underset{k=1}{\sum }}\epsilon _{k}<\infty,
\end{equation*}

then 
\begin{equation*}
\underset{k\longrightarrow \infty }{\lim }\frac{\left\Vert \left(
B_{k}-G_{\ast }\right) s_{k}\right\Vert }{\left\Vert s_{k}\right\Vert }=0,
\end{equation*}

and the sequences $\left\{ \left\Vert B_{k}\right\Vert \right\} ,$ $\left\{
\left\Vert B_{k}^{-1}\right\Vert \right\} $ are bounded.
\end{theorem}

\begin{theorem}
\label{th5.2}(Sufficient descent and global convergence) Consider Algorithm
WH-BFGS-CG. Assume that the conditions H1, H2 and H3 in Assumption 4.1 are
satisfied as well as conditions of Theorem5.1. Then 
\begin{equation*}
\lim\limits_{k\rightarrow \infty }\left\Vert g_{k}\right\Vert ^{2}=0.
\end{equation*}
\end{theorem}

\section{Numerical results}

In this section, some numerical results are reported to illustrate the
behaviours of WMLSCD, WMMDL and WH-BFGS-CG methods. The step size $\alpha
_{k}$ is determined using the Wolfe line search.

We use the Matlab Langage with a precision $\varepsilon =10^{-6}$.

We designate by:

\begin{itemize}
\item $k$: The number of iterations required to obtain the solution.

\item $Time$: The execution time in second.
\end{itemize}

\begin{example}
We take the function

\begin{equation*}
f(x)=\sum\limits_{i=1}^{n}(\exp (x_{i})-x_{i}).
\end{equation*}
\end{example}

We take as starting point $x_{0}=(1,1,\ldots,1)^{T}.$

The minimum of this function is reached at the point $x^{\ast
}=(0,0,\ldots,0)^{T}$ and $f(x^{\ast })=n.$

The results obtained are summarised in the following tables:

For $n=100$, we have 
\begin{equation*}
\begin{tabular}{|l|l|l|l|}
\hline
Methods & $k$ & $Time$ & $\parallel g_{k}\parallel $ \\ \hline
WMLSCD & $22$ & $3.883876$ & $5.8263e-07$ \\ \hline
WMMDL & $22$ & $3.803220$ & $5.8263e-07$ \\ \hline
WH-BFGS-GC & $5$ & $1.622640$ & $8.2976e-08$ \\ \hline
\end{tabular}%
\end{equation*}

For $n=500$, we have 
\begin{equation*}
\begin{tabular}{|l|l|l|l|}
\hline
Methods & $k$ & $Time$ & $\parallel g_{k}\parallel $ \\ \hline
WMLSCD & $24$ & $74.325460$ & $6.4631e-07$ \\ \hline
WMMDL & $24$ & $70.101070$ & $6.4631e-07$ \\ \hline
WH-BFGS-GC & $5$ & $21.087659$ & $1.8554e-07$ \\ \hline
\end{tabular}%
\end{equation*}

\begin{example}
We take the function 
\begin{equation*}
f(x)=\sum\limits_{i=1}^{n}\ln (\exp (x_{i})+\exp (-x_{i})).
\end{equation*}
\end{example}

We take as starting point $x_{0}=(1.1,1.1,\ldots,1.1)^{T}$

The minimum of this function is reached at the point $x^{\ast
}=(0,0,\ldots,0)^{T}$ and $f(x^{\ast })=n\ln (2).$

The results obtained are summarised in the following tables:

For $n=100$, we have 
\begin{equation*}
\begin{tabular}{|l|l|l|l|}
\hline
Methods & $k$ & $Time$ & $\parallel g_{k}\parallel $ \\ \hline
WMLSCD & $104$ & $40.083872$ & $9.9132e-07$ \\ \hline
WMMDL & $104$ & $83.918822$ & $9.9369e-07$ \\ \hline
WH-BFGS-GC & $66$ & $20.465962$ & $8.4827e-07$ \\ \hline
\end{tabular}%
\end{equation*}

For $n=200$, we have 
\begin{equation*}
\begin{tabular}{|l|l|l|l|}
\hline
Methods & $k$ & $Time$ & $\parallel g_{k}\parallel $ \\ \hline
WMLSCD & $107$ & $83.209667$ & $9.1391e-07$ \\ \hline
WMMDL & $108$ & $80.273199$ & $9.2334e-07$ \\ \hline
WH-BFGS-GC & $69$ & $52.410529$ & $8.2027e-07$ \\ \hline
\end{tabular}%
\end{equation*}

For $n=300$, we have 
\begin{equation*}
\begin{tabular}{|l|l|l|l|}
\hline
Methods & $k$ & $Time$ & $\parallel g_{k}\parallel $ \\ \hline
WMLSCD & $109$ & $171.535865$ & $9.8675e-07$ \\ \hline
WMMDL & $111$ & $205.430203$ & $9.5399e-07$ \\ \hline
WH-BFGS-GC & $70$ & $110.807414$ & $7.9846e-07$ \\ \hline
\end{tabular}%
\end{equation*}

\bigskip \textbf{Commentaries:} The numerical tests show clearly that the
proposed hybrid algorithm WH-BFGS-GC Wolfe based on line search is more
efficient in terms of number of iterations and computation time than WMLSCD
and WMMDL methods.

\section{Conclusion}

We have considered the hybrid conjugate gradient methods, MLSCD, MMDL and
H-BFGS-CG, for solving unconstrained optimization problems where we have
changed the backtracking line search given in \cite{SB2018} by the Wolfe
line search. Firstly, we have shown that the obtained WMLSCD, WMMDL and
WH-BFGS-CG algorithms are globally convergent for general functions.

Secondly, the numerical simulations confirm the effectiveness of the
approach WH-BFGS-CG. In fact, the WH-BFGS-CG method is the most efficient in
terms of number of iterations and computation time compared to WMLSCD and
WMMDL methods which was not the case with backtracking line search, where
the computation time of H-BFGS-GC was greater than MLSCD and MMDL \cite%
{SB2018}.

\begin{thebibliography}{99}
\bibitem{BN89} Byrd, R. H., Nocedal, J., \emph{A tool for the analysis of
quasi-Newton methods with application to unconstrained minimization}, SIAM
J. Numer. Anal., \textbf{26}(1989), no. 3, 727-739.

\bibitem{DL2001} Dai, Y.H., Liao, L.Z., \emph{New conjugacy conditions and
related nonlinear conjugate gradient methods}, Appl. Math. Optim., \textbf{43%
}(2001), no. 1, 87-101.

\bibitem{DY99} Dai, Y.H., Yuan, Y., \emph{A nonlinear conjugate gradient
method with a strong global convergence property}, SIAM J. Optim., \textbf{10%
}(1999), no. 1, 177-182.

\bibitem{F87} Fletcher, R., \emph{Practical Methods of Optimization.
Unconstrained Optimization}, vol. 1. Wiley New York, 1987.

\bibitem{FR64} Fletcher, R., Reeves, C.M., \emph{Function minimization by
conjugate gradients}, Comput. J., \textbf{7}(1964), no. 2, 149-154.

\bibitem{HS52} Hestenes, M.R., Stiefel, E., \emph{Methods of conjugate
gradients for solving linear systems}, J. Res. Natl. Bur. Stand., \textbf{49}%
(1952), no. 6, 409-436.

\bibitem{LS91} Liu, Y., Storey, C., \emph{Efficient generalized conjugate
gradient algorithms, part 1: theory}, J. Optim. Theory Appl., \textbf{69}%
(1991), no. 1, 129-137.

\bibitem{PR69} Polak, E., Ribiere, G., \emph{Note sur la convergence des
m\'ethodes de directions conjugu\'{e}es}, Rev. Fran\c{c}aise d'Informatique
et de Recherche Op\'erationnelle \textbf{3}(1969), no. R1, 35-43.

\bibitem{P69} Polyak, B.T., \emph{The conjugate gradient method in extreme
problems}, Comput. Math. Phys., \textbf{9}(1969), 94-112.

\bibitem{SB2018} Stanimirovi\'{c}, P. S., Ivanov, B., Djordjevi\'{c}, S.,
Brajevi\'{c}, I., \emph{New Hybrid Conjugate Gradient and
Broyden--Fletcher--Goldfarb--Shanno Conjugate Gradient Methods}, J. Optim.
Theory Appl., \textbf{178}(2018), no. 3, 860-884.

\bibitem{YL2013} Yang, X., Luo, Z., Dai, X., \emph{A global convergence of
LS-CD hybrid conjugate gradient method}, Adv. Numerical Anal. (2013).
Hindawi Publishing Corporation, Article ID 517452, 5 pp.

\bibitem{ZH17} Zheng, Y., Zheng, B., \emph{Two new Dai-Liao-type conjugate
gradient methods for unconstrained optimization problems}, J. Optim. Theory
Appl., \textbf{175}(2017), 502-509.

\bibitem{ZT70} Zoutendijk, G., \emph{Nonlinear programming, computational
methods}, Abadie, J. (ed.) Integer and Nonlinear Programming, (1970), 37-86.
\end{thebibliography}

\end{document}
