\documentclass[reqno]{amsart}
\usepackage{hyperref}

\AtBeginDocument{{\noindent\small
\emph{Electronic Journal of Differential Equations},
Vol. 2015 (2015), No. 176, pp. 1--17.\newline
ISSN: 1072-6691. URL: http://ejde.math.txstate.edu or http://ejde.math.unt.edu
\newline ftp ejde.math.txstate.edu}
\thanks{\copyright 2015 Texas State University - San Marcos.}
\vspace{9mm}}

\begin{document}
\title[\hfilneg EJDE-2015/176\hfil Orbital stability]
{Orbital stability of solitary waves for a 2D-Boussinesq system}

\author[A. M. Montes, J. R. Quintero \hfil EJDE-2015/176\hfilneg]
{Alex M. Montes, Jos\'e R. Quintero}

\address{Alex M. Montes \newline
 Universidad del Cauca, Carrera 3 3N-100, Popay\'an, Colombia}
\email{amontes@unicauca.edu.co}

\address{Jos\'e R. Quintero \newline
 Universidad del Valle, A. A. 25360, Cali, Colombia}
\email{jose.quintero@correounivalle.edu.co}

\thanks{Submitted June 2, 2015. Published June 24, 2015.}
\subjclass[2010]{35B35, 76B25, 35Q35}
\keywords{Solitary waves; orbital stability; ground state solutions;
\hfill\break\indent  Boussinesq system}

\begin{abstract}
 In this article, using a variational approach, we establish the
 nonlinear orbital stability of ground state solitary waves for a 2D
 Boussinesq-Benney-Luke system that models the evolution of three
 dimensional long water waves with small amplitude in the presence of
 surface tension.
\end{abstract}

\maketitle
\numberwithin{equation}{section}
\newtheorem{theorem}{Theorem}[section]
\newtheorem{lemma}[theorem]{Lemma}
\allowdisplaybreaks

\section{Introduction}

The focus of the present work is the two-dimensional
Boussinesq-Benney-Luke type system
\begin{equation}  \label{bbl}
\begin{gathered}
(I-\frac\mu2\Delta) \eta_t
+\Delta\Phi-\frac{2\mu}3\Delta^2\Phi+\epsilon\nabla\cdot(\eta\nabla\Phi)
 =0, \\
\left(I-\frac\mu2\Delta\right) \Phi_t  + \eta -\mu\sigma\Delta\eta
+\frac{\epsilon}{2}|\nabla\Phi|^2=0,
\end{gathered}
\end{equation}
that arises in the study of the evolution of small amplitude long
water waves in the presence of surface tension (see  Quintero and
 Montes \cite{JQAM2013}). Here $\mu, \epsilon$ are small
positive parameters, $\sigma^{-1}$ is the Bond number (associated
with the surface tension) and the functions $\eta(t,x,y)$ and
$\Phi(t,x,y)$ denote the wave elevation and the potential velocity
on the bottom $z=0$, respectively. The aspect that we study about
the system \eqref{bbl} is the orbital stability of solitary wave
solutions. It is well know that the study  of this kind of states of
motion is very important to understand the behavior of many physical
systems.

A special feature on the Boussinesq system \eqref{bbl} is that the
Benney-Luke equation (see \cite{JQRP1999}) and the
Kadomtsev-Petviashvili (KP) equation can be derived up to some order
with respect to $\mu$ and $\epsilon$ from system \eqref{bbl}.
Moreover, for small wave speed and large surface tension, is showed
in \cite{JQAM2013} (see also \cite{AMM}) that a suitable
renormalized family of solitary waves of the Boussinesq system
\eqref{bbl} converges to a nontrivial solitary wave for the (KP-I)
equation. We will use this fact in the stability analysis.

One of the main characteristics behind water wave systems is the
existence of a Hamiltonian structure which characterizes travelling
waves as critical points of the action functional and also provides
relevant information for the stability of travelling waves. In our
particular Boussinesq system \eqref{bbl}, the Hamiltonian structure
is given by
\[
\begin{pmatrix}\eta_t  \\
\Phi_t
\end{pmatrix}=\mathcal{B}\mathcal{H}'\begin{pmatrix}\eta \\ \Phi \end{pmatrix},  \   \    \  \
\mathcal{B}=\left(I-\frac\mu2\Delta\right)^{-1}
\begin{pmatrix}0 &1\\ -1&0 \end{pmatrix},
\]
where the Hamiltonian $\mathcal{H}$ is defined as
\[ %\label{ham}
\mathcal{H}\begin{pmatrix}\eta \\ \Phi
\end{pmatrix}=\frac{1}{2}\int_{\mathbb{R}^2}\Bigl(|\nabla\Phi|^2+\eta^2
+\frac{2\mu}{3}|\Delta\Phi|^2+\mu\sigma|\nabla\eta|^2
+\epsilon\eta|\nabla\Phi|^2\Bigr)\,dx\,dy.
\]
On the other hand, by Noether's Theorem, there is a functional
$\mathcal Q$ (named the Charge) which is conserved in time for
classical solutions defined formally as
\begin{equation*}\label{Q}
\mathcal{Q}\begin{pmatrix}\eta \\ \Phi \end{pmatrix}=\frac12
\big\langle \mathcal{B}^{-1}\partial_x
\begin{pmatrix}\eta \\ \Phi \end{pmatrix},
\begin{pmatrix}\eta \\ \Phi \end{pmatrix}\big\rangle
= -\frac12\int_{\mathbb{R}^2}\Bigl(2\eta \Phi_x+ \mu \eta_x \Delta \Phi \Bigr)
\,dx\,dy.
\end{equation*}
We will see that travelling waves of wave speed $c$ for the
Boussinesq system \eqref{bbl} corresponds to stationary solutions of
the modulated system
\[
\begin{pmatrix}\eta_t  \\
\Phi_t
\end{pmatrix}=\mathcal{B}{\mathcal{H}_c}'\begin{pmatrix}\eta \\ \Phi \end{pmatrix},
\]
where $\mathcal{H}_c(Y)=\mathcal{H}(Y)+c\mathcal Q(Y)$. In other
words, solutions of the system
\[
\mathcal{H}'(Y)+c\mathcal Q'(Y)=0.
\]
We note from the Hamiltonian structure that the well definition of
the functionals $\mathcal{H}$ and $\mathcal{H}_c$ require having
$\eta, \nabla\Phi\in H^1(\mathbb{R}^2)$. These conditions already
characterize the natu\-ral space (energy space) to look for
travelling waves solutions of the system Bousinesq-Benney-Luke, as
shown in the preliminary section. It is important to mention that
using the conservation in time of the Hamiltonian, A. Montes
\textit{et.} \textit{al.} (see \cite{JQAM2013c}) established the
existence of global solutions for the Cauchy problem associated with
the system \eqref{bbl} and the initial condition in the energy
space. On the other hand,  J. Quintero and A. Montes in
\cite{JQAM2013} showed the existence of solitary waves (travelling
wave solutions in the energy space) by using a variational approach
in which weak solutions correspond to critical points of an energy
under a special constrain.

Regarding the stability issue, we need to recall that M. Grillakis,
J. Shatah and W. Strauss in \cite{GSS} gave a general result used to
establish orbital stability of solitary waves for a class of
abstract Hamiltonian systems. In this case, solitary waves of least
energy $Y_c$ are minimums of the action functional $\mathcal{H}_c$
and the stability analysis depends on the positiveness of the
symmetric operator $ \mathcal{H}''_c(Y_c)$ in a neighborhood of the
solitary wave $Y_c$, except possibly in two directions, and also the
strict convexity of the real function
$$
d_1(c)=\inf \{\mathcal{H}_c(Y): Y \in \mathcal M_c\},
$$
where $\mathcal M_c$ is a suitable set. The verification of the
positiveness of $\mathcal{H}''_c(Y_c)$ is much simpler for
one-dimensional spatial problems since the spectral analysis for the
operator $ \mathcal{H}''_c(Y_c)$ is reduced to studying the
eigenvalues of a ordinary differential equation which at $\pm$
infinity becomes to a   ordinary differential equation with constant
coefficients (see \cite{BSS,Q1,jq-jm}). The key fact
to obtain stability in those cases is that in the one dimensional
case solitary waves are unique up to translations, and $d_1$ can be
rescaled allowing to establish   the strict convexity  $d_1(c)$ in a
direct way (see \cite{BSS,Q1,jq-jm}). In the
two-dimensional spatial case, we have a harder task to overcome
using Grillakis \emph{et al.} approach since the spectral
analysis is not straightforward for our problem. In order to avoid
making the spectral analysis required in Grillakis \emph{et al.}
work, we used a direct approach to prove orbital stability of ground
state solitary wave solutions of the system \eqref{bbl} in the case
of wave speed $c$ near $1^{-}$, using strongly the variational
characterization of $d_1$, as done for other 2D models: see
Shatah for nonlinear Klein Gordon equations \cite{Shatah},
Quintero  for the 2D Benney-Luke equation \cite{JQ2005} and also in
the case of a 2D Boussinesq-KdV type system  \cite{JQ2010b},
Saut  for the KP equation \cite{DBJC},  Fukuizumi for the
nonlinear Schr\"odinger equation with harmonic potential
\cite{Fukuizumi} and  Liu for the generalized KP equation
\cite{YLXP}, among others.

This article is organized as follows. In section \ref{pre}, we present
preliminaries related with the existence of solitons (solitary
wave solutions) for the system Boussinesq-Benney-Luke and the link
between solitons for the system \eqref{bbl} and the (KP) equation.
In section \ref{var}, we prove the strict convexity of $d_1$ for
$c\in(0,1)$, but near 1. In section \ref{sta}, we establish the
orbital stability result.

\section{Preliminaries} \label{pre}

To simplify the computation, we rescale the parameters
$\mu$ and $\epsilon$ from the system  \eqref{bbl} by defining
$$
\widetilde\eta(t,x,y)
=\frac{1}{\epsilon}\eta
\Big(\frac{t}{\sqrt{\mu}},\frac{x}{\sqrt{\mu}},\frac{y}{\sqrt{\mu}}
\Big), \quad  \widetilde{\Phi}(t,x,y)
=\frac{\sqrt{\mu}}{\epsilon}\Phi
\Big(\frac{t}{\sqrt{\mu}},\frac{x}{\sqrt{\mu}},\frac{y}{\sqrt{\mu}}
\Big).
$$
So,  by a solitary wave solution for the system \eqref{bbl} we mean
a solution for the rescaled system of the form
\begin{equation*}
\eta(t,x,y)=u\left(x-ct,y\right), \quad
\Phi(t,x,y)=v\left(x-ct,y\right),
\end{equation*}
where $c$ denotes the speed of the wave. Then, one sees that the
solitary wave profile $(u,v)$ should satisfy the system
\begin{equation} \label{trav-eqs}
\begin{gathered}
\frac{2}{3}\Delta^2v-\Delta v+c\Big(I-\frac{1}{2}\Delta\Big)u_x
-\nabla\cdot(u\nabla v) =0,
\\
u-\sigma\Delta u-c\Big(I-\frac{1}{2}\Delta\Big)v_x
+\frac{1}{2}|\nabla v|^2=0.
\end{gathered}
\end{equation}
Our stability analysis of the solitary wave solutions will be
perform in the following appropriate spaces. Recall that the
standard Sobolev space $H^k(\mathbb{R}^2)$, $k\in \mathbb{Z}^+,$ is
the Hilbert space defined as the closure of
$C_0^\infty(\mathbb{R}^2)$ with inner product
$$
\langle u,v\rangle_{H^{k}}
=\sum_{|\alpha| \leq k}\int_{\mathbb{R}^2}D^\alpha u\cdot D^\alpha v\,dx.
$$
We denote $\mathcal V$  the closure of $C_0^{\infty}(\mathbb{R}^2)$
with respect to the norm given by
\[
 \|v\|^2_{\mathcal V}
:= \int_{{\mathbb{R}}^2}\Bigl(|\nabla v|^2+|\Delta
v|^2\Bigr)\,dx\,dy=\int_{{\mathbb{R}}^2} \left(v_x^2 +v_y^2 + v_{xx}^2 +
2v_{xy}^2 +v_{yy}^2\right)\,dx\,dy.
\]
Note that $\mathcal V$ is a Hilbert space with respect to the inner
product
$$
\langle v,w\rangle_{\mathcal
V}=\langle v_x,w_x\rangle_{H^{1}(\mathbb{R}^2)}
+\langle v_y,w_y\rangle_{H^{1}(\mathbb{R}^2)}.
$$
Also, we define the energy space $\mathcal
X=H^1(\mathbb{R}^2)\times\mathcal V$, which is a Hilbert space with
respect to the norm
\[
\|(u,v)\|_{\mathcal
X}^2=\|u\|_{H^{1}(\mathbb{R}^2)}^2+\|v\|_{\mathcal V}^2=
\int_{{\mathbb{R}}^2} \left(u^2+|\nabla u|^2+|\nabla v|^2+|\Delta
v|^2\right)\,dx\,dy.
\]
We can see that solutions $(u,v)$ of system \eqref{trav-eqs} are
critical points of the functional $J_c= 2\mathcal{H}_c$ given by
$$
J_{c}(u, v)=I_c(u, v)+ G(u, v),
$$
where the functionals $I_c$  and $G$ are defined on the space
$\mathcal X$ by
\begin{gather*}
I_c(u,v)=I_1(u,v)+I_{2,c}(u,v),\\
I_1(u,v)=\int_{{\mathbb{R}}^2} \left(u^2+\sigma|\nabla u|^2+|\nabla
v|^2+\tfrac{2}{3}(\Delta v)^2\right)\,dx\,dy,\\
I_{2,c}(u,v)=-c\int_{\mathbb{R}^2} \left(2uv_x+u_x\Delta
v\right)\,dx\,dy,\\
G(u,v)=\int_{\mathbb{R}^2}u|\nabla v|^2\,dx\,dy.
\end{gather*}
In fact, note that $I_c, G \in C^1(\mathcal X, \mathbb{R})$ and its
derivatives in $(u,v)$ in the direction of $(U, V)$ are given by
\begin{gather*}
\begin{aligned}
\langle I'_c(u, v), (U, V)\rangle
&= 2 \int_{\mathbb{R}^2} \left( uU+
\sigma\nabla u \cdot \nabla U + \nabla v \cdot
\nabla V + \tfrac2{3} \Delta v \Delta V \right)\,dx\,dy \\
&\quad -c\int_{\mathbb{R}^2}\left(2uV_x+ 2v_xU+u_x\Delta V
+\Delta vU_x\right)\,dx\,dy,
\end{aligned} \\
\langle G'(u, v), (U, V)\rangle
=\int_{\mathbb{R}^2}\left(|\nabla
v|^2U+2u\nabla v\cdot\nabla V\right)\,dx\,dy.
\end{gather*}
Then we see that
\[
J_{c}'(u, v)= 2\begin{pmatrix}u-\sigma\Delta u
-c(I-\frac12\Delta)v_x+\frac12|\nabla v|^2\\
\frac{2}3\Delta^2v-\Delta v
+c(I-\frac12\Delta)u_x-\nabla\cdot(u\nabla v)
\end{pmatrix},
\]
meaning that critical points of the functional $J_{c}$ satisfy the
solitary wave system \eqref{trav-eqs}.


\subsection{Existence of solitary waves}

Quintero and Montes \cite{JQAM2013} established the \\
existence of solitary wave solutions for the Boussinesq-Benney-Luke
system \eqref{bbl} for $\sigma>0$ and
$0<c<\min\{1,\tfrac{8\sigma}{3}\}$, by using the
Concentration-Compactness principle and the existence of a local
compact embedding result. The strategy was to consider the following
minimization problem
\begin{equation}\label{mp}
\mathcal I_c:=\inf\{I_c(u,v)  :     (u,v)\in\mathcal
X \text{ with } G(u,v)=1 \}.
\end{equation}
The existence of solitary waves is consequence of the following
results \cite{JQAM2013}, which we will use throughout this
work. Next, we assume that $\sigma>0$ and
$0<c<\min\{1,\tfrac{8\sigma}{3}\}$.

\begin{lemma} \label{lem2.1}
The functional $I_c$ is nonnegative and there are positive
cons\-tants $C_1(\sigma,c)<C_2(\sigma,c)$ defined as
$$
C_1(\sigma,c)=\min\bigl\{1-c, \sigma(1-c),
\frac23-\frac{c}{4\sigma}\bigr\}, \quad
C_2(\sigma,c)=\max\bigl\{1+c,\frac23+\frac{c}2, \sigma+\frac{c}{2}\bigr\}
$$
such that
\begin{equation}\label{eqn}
C_1(\sigma,c)I_c(u,v)\leq\|(u,v)\|^2_{\mathcal X}
\leq C_2(\sigma,c) I_c(u,v).
\end{equation}
Furthermore, $\mathcal I_c$ is finite and positive.
\end{lemma}

\begin{theorem}\label{wsol}
If $(u_0,v_0)$ is a minimizer for problem \eqref{mp}, then
$(u,v)=-k(u_0,v_0)$ is a nontrivial solution of \eqref{trav-eqs} for
$k=\tfrac{2}{3}\mathcal I_c$.
\end{theorem}

\begin{theorem}\label{pt}
If $\{(u_m,v_m)\}$ is a minimizing sequence for \eqref{mp}, then
there is a subsequence  (which we denote the same), a sequence of
points $(x_m,y_m)\in \mathbb{R}^2 $, and a minimizer 
$(u_0,v_0)\in \mathcal X$ of \eqref{mp}, such that the translated functions
$$
(\tilde{u}_m,\tilde{v}_m)=(u_m( \cdot+x_m,\cdot+y_m),v_m(
\cdot+x_m,\cdot+y_m))
$$ 
converge to $(u_0,v_0)$ strongly in $\mathcal X$.
\end{theorem}

\subsection{Link between solitary waves for \eqref{bbl} and the KP equation}
\label{interela}
 Assuming $\sigma>3/8$, $c$ is close to $1^-$, and balancing the effects of
nonlinearity and dispersion,  Quintero and  Montes \cite{JQAM2013}
 established that a renormalized family of solitons
of the Boussinesq-Benney-Luke system converges to a nontrivial
soliton for a KP-I type equation. More precisely, set $\sigma>0$,
$\epsilon>0$, $\mu=\epsilon$, $c^2=1-\epsilon$ and for a given
couple $(u,v)\in\mathcal X$ define the functions $z$ and $w$ by
\begin{equation}\label{defuz}
u(x,y)=\epsilon^{1/2}z(X,Y), \quad v(x,y)=w(X,Y), \quad
X=\epsilon^{1/2}x,\quad  Y=\epsilon y.
\end{equation}
Then a simple calculation shows that
\begin{gather*}
I_1(u,v)=\epsilon^{1/2}I^{1,\epsilon}(z,w), \quad
I_{2,c}(u,v)=\epsilon^{1/2}I^{2,\epsilon}(z,w), \\
I_{c(\epsilon)}(u,v)=\epsilon^{1/2}I^\epsilon(z,w), \quad
G(u,v)=G^\epsilon(z,w),
\end{gather*}
where $I^1$, $I^{2,\epsilon}$, $I^\epsilon$ and $G^\epsilon$ are
given by
\begin{gather*}
I^\epsilon(z,w)=I^{1,\epsilon}(z,w)+I^{2,\epsilon}(z,w),\\
\begin{aligned}
I^{1,\epsilon}(z,w)&=\int_{\mathbb{R}^2}\left(\epsilon^{-1}z^2
 +\sigma(z_x^2+\epsilon z_y^2)+ \epsilon^{-1}w_x^2+w_y^2\right)\,dx\,dy\\
&\quad +\frac{2}{3}\int_{\mathbb{R}^2}\left(w_{xx}^2+2\epsilon
  w_{xy}^2+\epsilon^2w_{yy}^2\right)\,dx\,dy,
\end{aligned}\\
I^{2,\epsilon}(z,w)=-c\int_{\mathbb{R}^2}\left(2\epsilon^{-1}zw_x+z_x(w_{xx}
 + \epsilon w_{yy})\right)\,dx\,dy,\\
G^\epsilon(z,w) =\int_{{\mathbb{R}}^2}z\left(w_x^2+\epsilon
w_y^2\right)\,dx\,dy.
\end{gather*}
Note that if $\sigma>3/8$ then  there is a family
$\{(u_c,v_c)\}_c$ such that
$$
I_{c}(u_c,v_c)=\mathcal I_{c},  \quad  G(u_c,v_c)=1,  \quad 0<c<1.
$$
Then, if we denote
$$
\mathcal I^\epsilon:=\inf\{I^\epsilon(z,w)   :   (z,w)\in\mathcal X
\text{ with }  G^\epsilon(z,w)=1 \},
$$
there is a correspondent family
$\{(z^\epsilon,w^\epsilon)\}_\epsilon$ such that
\begin{equation}\label{Ic=eIe}
\mathcal I^\epsilon=I^\epsilon(z^\epsilon,w^\epsilon), \quad
G^\epsilon(z^\epsilon,w^\epsilon)=1, \quad
\mathcal I_{c}=\epsilon^{1/2}\mathcal I^\epsilon\,.
\end{equation}
We have the following results (see \cite{JQAM2013}).

\begin{lemma}\label{limite}
Let $\sigma>3/8$. Then we have 
\begin{equation}\label{lim}
\lim_{\epsilon\to 0^+}\mathcal
I^\epsilon=\lim_{\epsilon\to 0^+}I^\epsilon(z^\epsilon,w^\epsilon)
=\mathcal{J}^0>0,
\end{equation}
where
\begin{gather*}
\mathcal J^{0} = \inf \{J^0(w): w \in \mathcal V, \, G^{0}(w)=1 \},\\
J^{0}(w)  = \int_{{\mathbb{R}}^2} \left(
w_{x}^2 + w_y^2 + \left(\sigma - \tfrac{1}{3}\right)w_{xx}^2 \right)\,dx\,dy,  \\
G^{0}(w)  =  \int_{{\mathbb{R}^{2}}}w_{x}^{3} \,dx\,dy.
\end{gather*}
\end{lemma}

\begin{lemma}\label{conver}
Let $\sigma>3/8$. Then we have 
\begin{equation*}
\lim_{\epsilon\to 0^+}\left(z^{\epsilon}-\partial_xw^\epsilon\right)=0
\quad \text{in }  L^2(\mathbb{R}^2).
\end{equation*}
Moreover, there is a nontrivial distribution $w_0\in\mathcal V$ such
that
\begin{equation*}
\lim_{\epsilon\to 0^+}\partial_xw^{\epsilon}=\partial_xw_0  
\quad \text{in }   L^2(\mathbb{R}^2).
\end{equation*}
Furthermore,
\begin{gather*}
\|z^{\epsilon}\|_{L^2(\mathbb{R}^2)}+\|\partial_xz^{\epsilon}\|_{L^2(\mathbb{R}^2)}=
O(1), \quad 
\|\partial_{yy}w^\epsilon\|_{L^2(\mathbb{R}^2)}=O(\epsilon^{-1}), \\
\|\partial_xw^{\epsilon}\|_{L^2(\mathbb{R}^2)}
 +\|\partial_{xx}w^{\epsilon}\|_{L^2(\mathbb{R}^2)}=O(1).
\end{gather*}
\end{lemma}

Using the previous lemmas,  Quintero \emph{et al.} showed
that there are nontrivial distributions $w_0\in\mathcal V$, $z_0\in
H^1(\mathbb{R}^2)$ such that as $\epsilon\to 0^+$,
\begin{equation*}\label{zcon}
w^\epsilon\to w_0  \quad  \text{in }\mathcal{V}, \quad
z^{\epsilon} \to z_0 \quad \text{in }H^1(\mathbb{R}^2),
\end{equation*}
and $\partial_x w_0$ being a solution of the solitary wave equation
for the (KP-I) type equation
\begin{equation*}
\left(u_{x}-(\sigma-\tfrac{1}{3})u_{xxx}+3 u
u_{x}\right)_{x} +u_{yy}=0.
\end{equation*}

We shall use  Lemmas \ref{limite} and  \ref{conver} in our
proof of stability.

\section{Variational approach for stability}\label{var}

Recall that the solitary waves for the Boussinesq-Benney-Luke system
\eqref{bbl} are characterized as critical points of the functional
defined on $\mathcal X$ by
$$
J_c(u, v)=I_c(u, v)+G(u, v).
$$
In particular, if
$$
K_c(u,v)=\langle J_{c}'(u, v), (u, v)\rangle
$$ 
we have 
\begin{align*}
K_c(u,v)=2I_c(u, v)+3G(u, v)= 2J_{c}(u, v) + G(u, v).
\end{align*}
Thus, on any critical point $(u, v)$ of the functional $J_c$ we have
that
\begin{gather}\label{critical1}
J_{c}(u, v) = \frac{1}{3}I_c(u, v),\\
J_{c}(u, v) = -\frac{1}{2}G(u, v), \label{critical2}\\
I_c(u, v) = -\frac{3}{2}G_(u, v). \label{critical3}
\end{gather}
Now, define the set
\begin{equation*}
\mathcal{M}_c=\{ (u,v)\in\mathcal X   :   K_c(u,v)=0,  \,
(u,v)\neq0\}.
\end{equation*}
Note that $\mathcal{M}_c$ is just the ``artificial constrain'' for
minimizing the functional $J_c$ on $\mathcal X$. We will see that
the analysis of the orbital stability of ground states solutions
depends upon some properties of the function $d$ defined by
\begin{align*}
d(c)= \inf\{J_c(u,v) :   (u,v)\in\mathcal{M}_c\}.
\end{align*}
A ground state solution is a solitary wave  which minimizes the
action functional $J_c$ among all the nonzero solutions of
\eqref{trav-eqs}. Moreover, the set of ground state solutions
\begin{equation*}
\mathcal{G}_c=\{(u,v)\in \mathcal{M}_c  :  d(c)=J_c(u,v)\}
\end{equation*}
can be characterized as
\begin{equation*}
\mathcal{G}_c=\{(u,v)\in \mathcal X \setminus\{0\}   : 
d(c)=\frac13 I_c(u,v)=-\frac12 G(u,v)\}\subset\mathcal M_c.
\end{equation*}
We note that there is a simple relationship between $d_1$ and $d$,
and so regarding the convexity of them. In fact,
\begin{align*}
d(c)& =\inf\{J_c(u,v) :    (u,v)\in\mathcal{M}_c\} \\
& =2\inf\{\mathcal{H}_c(u,v)  :    (u,v)\in\mathcal{M}_c\} 
= 2 d_1(c).
\end{align*}

In the next lemmas we present important variational properties of
$d(c)$.

\begin{lemma}
Let $0<c<1$ and $\sigma>3/8$. Then
\begin{itemize}
\item[(1)] $d(c)$ exist and is positive.

\item[(2)]  $d(c)=\inf\{\frac13 I_c(u,v)   : K_c(u,v)\leq0, \,  (u,v)\neq0\}$.
\end{itemize}
\end{lemma}

\begin{proof}
(1) Let $(u,v)\in \mathcal{M}_c$, then we have that
$$
J_c(u,v)=\frac13I_c(u,v)\geq0.$$ This implies that $d(c)$ exists.
Now, Using the Young inequality and that the embedding
$H^1(\mathbb{R}^2) \hookrightarrow L^q(\mathbb{R}^2)$ is
continuous for $q\geq2$, we see that there is a constant $C>0$ such
that
\begin{equation}\label{Ge}
|G(u,v)| \leq C\Bigl(\|u\|^3_{H^1({\mathbb{R}}^2)}+\|\nabla
v\|^3_{H^1({\mathbb{R}}^2)}\Bigr).
\end{equation}
Thus, using \eqref{eqn} we see that
$$
J_c(u,v)=\frac13I_c(u,v)=-\frac12G(u,v)\leq C\|(u,v)\|_{\mathcal
X}^3\leq C\left( I_c(u,v)\right)^{3/2}.
$$
Then follows that $\frac13I_c(u,v)\geq C$, and this implies that
$d(c)\geq C>0$.


(2) For $(u,v)\in\mathcal X$ such that $K_c(u,v)\leq0$ we have that
$G(u,v)<0$. Define $\alpha\in[0,1)$ by
$$
\alpha=-\frac{2I_c(u,v)}{3G(u,v)}.
$$
Then a direct computation shows that $K_c(\alpha(u,v))=0$. In other
words, $\alpha(u,v)\in\mathcal{M}_c$. So that,
$$
d(c)\leq J_c(\alpha(u,v))=\frac{\alpha^2}{3}I_c(u,v)\leq\frac13
I_c(u,v).
$$
Hence, we obtain 
$$
d(c)\leq\inf\bigl\{\frac13 I_c(u,v) :K_c(u,v)\leq0 \bigr\}.
$$
If $(u,v)\in\mathcal{M}_c$, we see that $J_c(u,v)=\frac13 I_c(u,v)$
and 
\[
\inf\bigl\{\tfrac13 I_c(u,v) :
K_c(u,v)\leq0, \ (u,v)\neq0\bigr\} 
\leq\inf\bigl\{J_c(u,v) : (u,v)\in\mathcal{M}_c\bigr\}=d(c).
\]
Then the statement 2 of the lemma follows.
\end{proof}

\begin{lemma}\label{d1dc}
Let $0<c<1$ and $\sigma>3/8$. Then

(1) If $\{(u_m,v_m)\}$ is a minimizing sequence of $d(c)$, then
there is a subsequence, which we denote the same, a sequence of
points $(x_m,y_m)\in \mathbb{R}^2 $, and $(u^c,v^c)\in \mathcal
X\setminus\{0\}$ such that the translated functions
$$
(u_m( \cdot+x_m,\cdot+y_m),v_m( \cdot+x_m,\cdot+y_m))
$$
converge to $(u^c,v^c)$ strongly in $\mathcal X$,
$(u^c,v^c)\in\mathcal M_c$, $d(c)=J_c(u^c,v^c)$ and $(u^c,v^c)$ is a
solution of \eqref{trav-eqs}. Moreover,
\begin{equation}\label{dc=Ic}
d(c)=\frac4{27}\mathcal I_c^3,
\end{equation}
where $\mathcal I_c=\inf\{I_c(u,v) :G(u,v)=1, \
(u,v)\in\mathcal X\}$.

(2) Let $\{(u_m,v_m)\}$ be a sequence in $\mathcal X$ such that
$$
\frac13 I_c(u_m,v_m)\to  d(c) \quad\text{and}\quad
J_c(u_m,v_m)\to  \tilde d\leq d(c).
$$
Then there exist a subsequence of $\{(u_m,v_m)\}$ which denote the
same, a sequence $(x_m,y_m)\in\mathbb{R}^2$ and
$(u^c,v^c)\in\mathcal{M}_c$ such that the translated functions
$$
(u_m(\cdot+x_m, \cdot+y_m), v_m(\cdot+x_k, \cdot+y_k))
$$
converge to $(u^c,v^c)$ strongly in $\mathcal X$ and $\tilde
d=d(c)=\frac13 I_c(u^c,v^c)$.
\end{lemma}

\begin{proof}
The first part of this result is consequence of the Theorem
\ref{wsol}, Theorem \ref{pt} and the following argument. Let
$(u,v)\in\mathcal X\setminus\{0\}$ be such that $K_c(u,v)=0$, then
$$
I_c(u,v)=-\frac32G(u,v)=\frac32|G(u,v)|=3J_c(u,v).
$$
Consider the couple
$$
(z,w)=\frac{1}{G^{1/3}(u,v)}(u,v).
$$
Then $G(z,w)=1$. Thus,
\begin{align*}
\mathcal I_c\leq
I_c(z,w)=\frac{1}{G^{\frac{2}{3}}(u,v)}I_c(u,v)
=\left(\frac32\right)^{2/3}I_c^{1/3}(u,v)
=\left(\frac32\right)^{2/3}\Bigl(3J_c(u,v)\Bigr)^{1/3}.
\end{align*}
So that, we concluded
$$
\frac4{27}\mathcal I_c^3\leq d(c).
$$
Now, suppose that $(u,v)\neq0$ such that $G(u,v)=1$. Take $t$ such
that
$K_{c}(tu, tv)=0$.
In this case, $2I_c(u,v)+3t=0$. Therefore
$$
t^2=\frac49I^2_c(u,v).
$$
Then we obtain,
\[
d(c)\leq J_c(tu,tv)=t^2\left(I_c(u,v)+t\right)=\frac4{27}I^3_c(u,v).
\]
Thus, we have shown that
$$
d(c)\leq\frac4{27}\left(\mathcal I_c\right)^3.
$$
This proves \eqref{dc=Ic}. Now, we show the second part. Since
$K_c=2I_c+3G$ then we see that
$$
J_c(u_m,v_m)=\frac13\left(I_c(u_m,v_m)+K_c(u_m,v_m)\right)\to
\tilde d\leq d(c).
$$
Then for $m$ large enough we have that $K_c(u_m,v_m)\leq0$. This
fact implies that the sequence $\{(u_m,v_m)\}$ is a minimizing
sequence for $d(c)$. Then using the part 1 we have that there exist
a subsequence of $\{(u_m,v_m)\}$,  which denote the same, a sequence
$(x_m,y_m)\in\mathbb{R}^2$ and $(u^c,v^c)\in\mathcal{M}_c$ such that
$$
(u_m(\cdot+x_m, \cdot+y_m), v_m(\cdot+x_k, \cdot+y_k))\to (u^c,v^c)
$$  
in $\mathcal X$.
In particular $K_c(u^c,v^c)=0$ and $ \tilde
d=d(c)=\frac13I_c(u^c,v^c)$.
\end{proof}

\begin{lemma}\label{estd}
Let $0<c<1$ and $\sigma>3/8$. Then

(1) If $0<c_1<c_2<1$ and $(u,v)\in\mathcal{G}_c$, then
we have that $d(c)$ and $I_{2,c}(u,v)$ are uniformly bounded
functions on $[c_1,c_2]$.

(2) If $c_1<c_2$ and $(u^{c_i},v^{c_i})\in\mathcal{G}_{c_i}$, we have the following
inequalities
\begin{align*}
d(c_1)&\leq
d(c_2)-\Big(\frac{c_2-c_1}{c_2}\Big)I_{2,c_2}(u^{c_2},v^{c_2})+o(c_2-c_1),\\
d(c_2)&\leq
d(c_1)+\Big(\frac{c_2-c_1}{c_1}\Big)I_{2,c_1}(u^{c_1},v^{c_1})+o(c_2-c_1).
\end{align*}

(3) If $0<c_1<c_2<1$, $(u^{c_1},v^{c_1})\in\mathcal{G}_{c_1}$
and $I_{2,c_1}(u^{c_1},v^{c_1})\leq0$, then
$$
d(c_2)\leq
d(c_1)+\Big(\frac{2(c_2-c_1)}{3c_1}\Big)I_{2,c_1}(u^{c_1},v^{c_1}).
$$
In particular, $d$ is a strictly decreasing function on $(c_1,1)$.
\end{lemma}

\begin{proof}
(1) Let $c_1,c_2$ be such that $0<c_1<c_2<1$ and let
$(u,v)\in\mathcal X$ be such that $G(u,v)\neq0$. Define $t_c$ by
$$
t_c=-\frac{2}3\frac{I_c(u,v)}{G(u,v)}.
$$
Then we have that $K_c(t_c(u,v))=0$ and $J_c(t_c(u,v))=\frac{t_c^2}3
I_c(u,v)$. Using \eqref{eqn} we see that there exist $C>0$ that
depends only on $\sigma$ such that for all $c\in[c_1,c_2]$,
$$
d(c)\leq J_c(t_c(u,v))=\frac{4}{27}\frac{I^3_c(u,v)}{G^2(u,v)}\leq
C\frac{\|(u,v)\|^6_{\mathcal X}}{G^2(u,v)}.
$$
Now, let $(z,w)\in\mathcal{G}_c$, then we have that
$2I_c(z,w)+3G(z,w)=0$. Moreover,
$$
C_1(\sigma,c_1,c_2)\|(z,w)\|^2_{\mathcal
X}\leq2I_c(z,w)=3|G(z,w)|\leq C\|(z,w)\|_{\mathcal X}^3.
$$
Then we conclude that
$$
C_1(\sigma,c_1,c_2)\leq\|(z,w)\|_{\mathcal X}\leq
C_2(\sigma,c_1,c_2)\Big(\frac13I_c(z,w)\Big)^{1/2}.
$$
Thus, we have shown that
$$
d(c)\geq
\Big(\frac{C_1(\sigma,c_1,c_2)}{C_2(\sigma,c_1,c_2)}\Big)^2.
$$
Hence, if $(u,v)\in\mathcal{G}_c$ we see that $I_c(u,v)$ and
$G(u,v)$ are uniformly bounded on $[c_1,c_2]$ since
$$
d(c)=\frac13I_c(u,v)=-\frac12G(u,v),
$$
which implies that $I_{2,c}(u,v)$ is also uniformly bounded because
$K_c(u,v)=0$ and $$I_1(u,v)\cong\|(u,v)\|^2_{\mathcal X}.$$

(2) Let $(z,w)$ be defined by $(z,w)=t(u^{c_2},v^{c_2})$. We want $t$
such that $K_{c_1}(z,w)=0$. Note that
\begin{align*}
K_{c_1}(z,w)
&=2t^{2}I_{c_1}(u^{c_2},v^{c_2})+3t^3G(u^{c_2},v^{c_2})\\
&=t^2\Big(2I_{c_2}(u^{c_2},v^{c_2})-\frac{2(c_2-c_1)}{c_2}I_{2,c_2}(u^{c_2},v^{c_2})
\Big)+ 3t^3G(u^{c_2},v^{c_2})\\
&=t^2\Big(3tG(u^{c_2},v^{c_2})-3G(u^{c_2},v^{c_2})
-\frac{2(c_2-c_1)}{c_2}I_{2,c_2}(u^{c_2},v^{c_2}\Big).
\end{align*}
Thus, $t$ has to be such that
$$
tG(u^{c_2},v^{c_2})=G(u^{c_2},v^{c_2})
+\frac{2(c_2-c_1)}{3c_2}I_{2,c_2}(u^{c_2},v^{c_2})
$$
or equivalently
$$
t=1+\frac{2(c_2-c_1)}{3c_2}
\Big(\frac{I_{2,c_2}(u^{c_2},v^{c_2})}{G(u^{c_2},v^{c_2})}\Big)
= 1-\frac{(c_2-c_1)}{3c_2}\Big(\frac{I_{2,c_2}(u^{c_2},v^{c_2})}{d(c_2)}\Big).
$$
Then for this $t$, we conclude that $K_{c_1}(z,w)=0$. Now,
\begin{align*}
d(c_1)
&\leq J_{c_1}(w,z)
=t^2\Big(I_{c_1}(u^{c_2},v^{c_2})+tG(u^{c_2},v^{c_2})\Big)\\
&=t^2\Big(I_{c_2}(u^{c_2},v^{c_2})+\frac{c_1-c_2}{c_2}I_{2,c_2}(u^{c_2},v^{c_2})
+tG(u^{c_2},v^{c_2})\Big)\\
&=t^2\Big(d(c_2)-\frac{c_2-c_1}{3c_2}I_{2,c_2}(u^{c_2},v^{c_2})\Big).
\end{align*}
But we have that
\begin{align*}
t^2&= \Big(1-\frac{(c_2-c_1)}{3c_2}\Big(\frac{I_{2,c_2}(u^{c_2},v^{c_2})}{d(c_2)}
\Big)\Big)^2\\
&=1-\frac{2(c_2-c_1)}{3c_2}\Big(\frac{I_{2,c_2}(u^{c_2},v^{c_2})}{d(c_2)}\Big)
+O\left((c_2-c_1)^2\right).
\end{align*}
Then we see that
\begin{align*}
&t^2\Big(d(c_2)-\frac{(c_2-c_1)}{3c_2}I_{2,c_2}(u^{c_2},v^{c_2})\Big)\\
&= d(c_2)-\frac{(c_2-c_1)}{c_2}I_{2,c_2}(u^{c_2},v^{c_2})+O\left((c_2-c_1)^2\right),
\end{align*}
which implies the desired result,
$$
d(c_1)\leq
d(c_2)-\Big(\frac{c_2-c_1}{c_2}\Big)I_{2,c_2}(u^{c_2},v^{c_2})+o(c_2-c_1).
$$
Now, let $(z,w)$ be defined by $(z,w)=t(u^{c_1},v^{c_1})$. As
before, we want $t$ such that $K_{c_2}(z,w)=0$. In this case,
$$
t=1-\frac{2(c_2-c_1)}{3c_1}
\Big(\frac{I_{2,c_1}(u^{c_1},v^{c_1})}{G(u^{c_1},v^{c_1})}\Big)
= 1+\frac{(c_2-c_1)}{3c_1}\Big(\frac{I_{2,c_1}(u^{c_1},v^{c_1})}{d(c_1)}\Big).
$$
Since $K_{c_1}(z,w)=0$, we see that
$$
d(c_2)\leq
J_{c_2}(z,w)=t^2\Big(d(c_1)+\frac{c_2-c_1}{3c_1}I_{2,c_1}(u^{c_1},v^{c_1})\Big).
$$
Then, as above, we have that
\begin{align*}
t^2&=1+\frac{2(c_2-c_1)}{3c_1}\Big(\frac{I_{2,c_1}(u^{c_1},v^{c_1})}{d(c_1)}\Big)
+O\left((c_2-c_1)^2\right).
\end{align*}
Using this we conclude that
\begin{align*}
&t^2\Big(d(c_1)+\frac{(c_2-c_1)}{3c_1}I_{2,c_1}(u^{c_1},v^{c_1})\Big)\\
&= d(c_1)+\frac{(c_2-c_1)}{c_1}I_{2,c_1}(u^{c_1},v^{c_1})+O\left((c_2-c_1)^2\right),
\end{align*}
which implies the other inequality.

(3) Assume that $K_{c_1}(u^{c_1},v^{c_1})=0$. Hence we see that
$G(u^{c_1},v^{c_1})\leq0$. Now, if $I_{2,c_1}(u^{c_1},v^{c_1})\leq0$
then for $c_1<c_2$ we have that
$$
K_{c_2}(u^{c_1},v^{c_1})
=K_{c_1}(u^{c_1},v^{c_1})+\frac{2(c_2-c_1)}{c_1}I_{2,c_1}(u^{c_1},v^{c_1})\leq0.
$$
Thus, we obtain
\begin{align*}
d(c_2)&\leq\frac13I_{c_2}(u^{c_1},v^{c_1})\\
&=\frac{1}{3}\Big(I_{c_1}(u^{c_1},v^{c_1})
+\frac{c_2-c_1}{c_1}I_{2,c_1}(u^{c_1},v^{c_1})\Big)\\
&\leq d(c_1)+\frac{c_2-c_1}{3c_1}I_{2,c_1}(u^{c_1},v^{c_1}).
\end{align*}
This also implies that $d(c_2)<d(c_2)$, provided that $0<c_1<c_2<1$.
\end{proof}


\subsection*{Convexity of $d$}
 Now, we  prove that the function
$d$ is strictly convex on $(c_0,1)$ with $c_0>0$ near 1. To do this,
we compute $d'$ and analyze the behavior of $d$ and $d'$ near $1^-$.
We have the following results.

\begin{lemma} \label{lem3.4}
If $(u^c,v^c)\in\mathcal{G}_c$, then we have that
\begin{equation} \label{deri}
d'(c)=\frac{I_{2,c}(u^c,v^c)}{c}.
\end{equation}
\end{lemma}

\begin{proof} Note that $d'$ can be computed by taking approptiate limits in
part 2 of Lemma \ref{estd} 
\end{proof}

\begin{theorem}\label{behd}
Let $\sigma>3/8$ and $(u^c,v^c)\in\mathcal{G}_c$. Then we have
that
$$
\lim_{c\to 1^-}d(c)=0  \quad \text{and} \quad 
I_{2,c}(u^c,v^c)<0  \quad \text{for }  c  \text{near } 1^-.
$$
\end{theorem}

\begin{proof}
From Equations \eqref{defuz}-\eqref{lim} and \eqref{dc=Ic} we obtain
the first part. Now, using the same notation as Section
\ref{interela} we have 
\begin{align*}
\epsilon I^{2,\epsilon}(z^\epsilon,w^\epsilon)
&=-c\int_{\mathbb{R}^2}\Bigl(2z^\epsilon\partial_{x}w^\epsilon+\epsilon\partial_xz^\epsilon\left(\partial_{xx}w^\epsilon+
\epsilon\partial_{yy}w^\epsilon\right)\Bigr)\,dx\,dy\\
&=-2c\int_{\mathbb{R}^2}\left(z^\epsilon-\partial_{x}w^\epsilon\right)\partial_{x}w^\epsilon
\,dx\,dy\\
&-c\epsilon\int_{\mathbb{R}^2}\partial_xz^\epsilon\left(\partial_{xx}w^\epsilon+
\epsilon
z^\epsilon\partial_{yy}w^\epsilon\right)\,dx\,dy-2c\int_{\mathbb{R}^2}\left(\partial_xw^\epsilon\right)^2\,dx\,dy.
\end{align*}
Then using Lemma \ref{conver} we see that
$$
\lim_{\epsilon\to 0^+}\epsilon
I^{2,\epsilon}(z^\epsilon,w^\epsilon)<0,
$$
meaning that for $\epsilon$ near $0^+$ we have
$I^{2,\epsilon}(z^\epsilon,w^\epsilon)<0$, which implies that for $c$ near
$1^-$, we ahve
$I_{2,c}(u_c,v_c)<0$.
\end{proof}

\begin{theorem} \label{thm3.2}
Let $\sigma>3/8$. Then there exist $0<c_0<1$ enough near $1$
such that $d$ is a decreasing function on $(c_0,1)$. Furthermore, 
$\lim_{c\to 1^-}d'(c)=0$.
\end{theorem}

\begin{proof}
Using \eqref{deri} and Theorem \ref{behd} we have that $d$ is a
decreasing function for $c$ near $1^-$ and we also have that
$\lim_{c\to 1^-}\|(u^c,v^c)\|_{\mathcal X}=0$ for any
$(u^c,v^c)\in\mathcal X$ such that $d(c)=\frac13 I_{c}(u^c,v^c)$,
since from \eqref{eqn} we see that
$$
\|(u^c,v^c)\|_{\mathcal X}^2\leq
C(\sigma)I_c(u^c,v^c)=C(\sigma)d(c).
$$
Thus, from \eqref{deri} and definition of $I_{2,c}$ we conclude that
$$
|d'(c)|\leq2\|u^c\|_{L^2(\mathbb{R}^2)}\|v^c_x\|_{L^2(\mathbb{R}^2)}
+\|u^c_x\|_{L^2(\mathbb{R}^2)}\|\Delta
v^c\|_{L^2(\mathbb{R}^2)}\leq3\|(u^c,v^c)\|^2_{\mathcal X}.
$$
Therefore,
$\lim_{c\to 1^-}d'(c)=0$.
\end{proof}

From the previous results we have the following lemma.

\begin{lemma} \label{lem3.5}
Let $\sigma>3/8$, then $d$ and $d_1$ are strictly convex for $c$
near $1^-$.
\end{lemma}

We will use the following result by Shatah  \cite{Shatah}.

\begin{lemma} \label{shlem}
Suppose that $h$ is a strictly convex function in a neighborhood of
$c_0$. Then given $\varepsilon>0$, there exist $N(\varepsilon)>0$
such that for $|c_\varepsilon-c_0|=\varepsilon$,
\begin{enumerate}
\item If $c_{\varepsilon}<c_0<c$ and $|c-c_0|<\varepsilon/2$,
$$
\frac{h(c_\varepsilon)-h(c)}{c_\varepsilon-c}
\leq\frac{h(c_0)-h(c)}{c_0-c}-\frac1{N(\varepsilon)}.
$$
\item If $c<c_0<c_{\varepsilon}$ and $|c-c_0|<\varepsilon/2$,
$$
\frac{h(c_\varepsilon)-h(c)}{c_\varepsilon-c}
\geq\frac{h(c_0)-h(c)}{c_0-c}+\frac1{N(\varepsilon)}.
$$
\end{enumerate}
\end{lemma}

\begin{theorem}\label{dc-dc0}
Let $\sigma>3/8$. If $0<c_0<1$ with $c_0$ near 1 and
$(u^{c_0},v^{c_0})\in\mathcal{G}_{c_0}$, then for $c$ close to $c_0$,
there exist $\rho(c)>0$ such that $\rho(c_0)=0$ and
$$
d(c)-d(c_0)\geq\Big(\frac{c-c_0}{c_0}\Big)I_{2,c_0}(u^{c_0},v^{c_0})+\rho(c).
$$
\end{theorem}

\begin{proof}
Let $c<c_0$, $c$ close to $c_0$. Then by Lemma \ref{shlem}, for
$c<c_0<c_1$ we see that
$$
\frac{d(c)-d(c_1)}{c-c_1}\leq\frac{d(c_0)-d(c_1)}{c_0-c_1}-\frac1{N(c)}.
$$
From Lemma \ref{estd} we have 
$$
d(c_1)\leq
d(c_0)+\Big(\frac{c_1-c_0}{c_0}\Big)I_{2,c_0}(u^{c_0},v^{c_0})+o(c_1-c_0).
$$
Then we obtain 
$$
\frac{d(c)-d(c_1)}{c-c_1}\leq\frac{d(c_1)-d(c_0)}{c_1-c_0}-\frac1{N(c)}
\leq\frac{I_{2,c_0}(u^{c_0},v^{c_0})}{c_0}+\frac{o(c_1-c_0)}{c_1-c_0}-\frac1{N(c)}.
$$
Using the continuity of $d$, we have as $c_1\to  c_0$ that
$$
\frac{d(c)-d(c_0)}{c-c_0}\leq\frac{I_{2,c_0}(u^{c_0},v^{c_0})}{c_0}-\frac1{N(c)}.
$$
As a consequence of this inequality follows that
$$
d(c)-d(c_0)\geq\Big(\frac{c-c_0}{c_0}\Big)I_{2,c_0}(u^{c_0},v^{c_0})
+\frac{c_0-c}{N(c)}.
$$
Now, let $c_0<c$ be $c$ close to $c_0$. If $c_1<c_0<c$, then by
using Lemma \ref{shlem},
$$
\frac{d(c)-d(c_1)}{c-c_1}\geq\frac{d(c_0)-d(c_1)}{c_0-c_1}+\frac1{N(c)}.
$$
Then from Lemma \ref{estd},
$$
d(c_1)\leq
d(c_0)-\Big(\frac{c_0-c_1}{c_0}\Big)I_{2,c_0}(u^{c_0},v^{c_0})+o(c_1-c_0).
$$
Thus, we obtain 
$$
\frac{d(c)-d(c_1)}{c-c_1}\geq\frac{d(c_1)-d(c_0)}{c_1-c_0}+\frac1{N(c)}
\geq\frac{I_{2,c_0}(u^{c_0},v^{c_0})}{c_0}+\frac{o(c_1-c_0)}{c_1-c_0}+\frac1{N(c)}.
$$
Again, using the continuity of $d$, we have as $c_1\to  c_0$
that
$$
\frac{d(c)-d(c_0)}{c-c_0}\geq\frac{I_{2,c_0}(u^{c_0},v^{c_0})}{c_0}+\frac1{N(c)}.
$$
As a consequence of this inequality holds 
$$
d(c)-d(c_0)\geq\Big(\frac{c-c_0}{c_0}\Big)I_{2,c_0}(u^{c_0},v^{c_0})
+\frac{c-c_0}{N(c)},
$$
and the result follows.
\end{proof}


\section{Orbital stability of the solitary waves}\label{sta}

We first consider the modulated system associated with the system
\eqref{trav-eqs} on $\mathcal X$. In other words, we assume that the
solution $(\eta(t), \Phi(t))$ of the system \eqref{bbl} has the form
$$
\eta(t,x,y)= z(t,x-ct,y) , \quad \Phi(t,x,y)=w(t,x-ct,y)
$$
Then we see that $(z(t),w(t))$ satisfies the modulated system
\begin{equation} \label{modsys}
\begin{gathered}
\Big(I-\frac{1}{2}\Delta\Big)z_t-c\Big(I-\frac{1}{2}\Delta\Big)z_x
-\frac{2}{3}\Delta^2w+\Delta w+\nabla\cdot(z\nabla w) =0,
\\ 
\Big(I-\frac{1}{2}\Delta\Big)w_t
-c\Big(I-\frac{1}{2}\Delta\Big)w_x+z-\sigma\Delta
z +\frac{1}{2}|\nabla w|^2=0.
\end{gathered}
\end{equation}
Observe that the modulated  Hamiltonian for this system has the form
$$
\mathcal{H}_c(z,w)=\frac12 J_c(z,w)=\mathcal{H}(z,w)+\frac12I_{2,c}(z,w),
$$
We also observe that $\mathcal{H}_c$ is conserved in time on
solutions since
\begin{gather*}
\Bigl(I-\frac12\Delta\Bigr)z_t=\partial_w\mathcal{H}_c(z,w)=c\Bigl(I-\frac{1}{2}\Delta\Bigr)z_x
+\frac{2}{3}\Delta^2w-\Delta w-\nabla\cdot\left(z\nabla w\right),\\
-\Bigl(I-\frac12\Delta\Bigr)w_t=\partial_z\mathcal{H}_c(z,w)=-c\Bigl(I-\frac{1}{2}\Delta\Bigr)w_x+z-\sigma\Delta z
+\frac{1}{2}|\nabla w|^2.
\end{gather*}

Now we introduce the regions $\mathcal{R}_c^i, i=1,2$, in the energy
space $\mathcal X$ by
\begin{gather*}
\mathcal{R}_c^1=\{(z,w)\in \mathcal X  :\mathcal{H}_c(z,w)<\frac12d(c), \;  \frac13I_c(z,w)<d(c)\}\\
\mathcal{R}_c^2=\{(z,w)\in \mathcal X  :\mathcal{H}_c(z,w)<\frac12d(c), \; \frac13I_c(z,w)>d(c)\},
\end{gather*}
and have the following result.

\begin{lemma}\label{inv}
$\mathcal{R}_c^1, \mathcal{R}_c^2$ are invariant regions under the
flow for the modulated system \eqref{modsys}.
\end{lemma}

\begin{proof}
Let $(u_0,v_0)\in \mathcal{R}_c^1$. Suppose that $(z(t),w(t))$
satisfies the modulated system \eqref{modsys} with initial condition
$$
z(0)=u_0,   \quad   w(0)=v_0.
$$
By characterization of $d(c)$ and definition of $\mathcal{R}_c^1$, we
must have that $$K_c(u_0,v_0)>0.$$ In fact, suppose that
$K_c(u_0,v_0)\leq0$. Then we see that $d(c)\leq\frac13I_c(u_0,v_0)$.
Moreover, if $(z(t),w(t))\in\mathcal{R}_c^1$ for some $t>0$, we have
that $K_c(z(t),w(t))>0$. Now, suppose that there exists a minimum
$t_0$ such that $K_c(z(t),w(t))>0$ for $t\in[0, t_0)$ and
$K_c(z(t_0),w(t_0))=0$. Observe that
\begin{align*}
d(c)&\leq\frac13I_c(z(t_0),w(t_0))\\
&\leq\liminf_{t\to  t_0^-}\Big(\frac13I_c(z(t),w(t))+\frac13K_c(z(t),w(t))\Big)\\
&\leq\liminf_{t\to  t_0^-}J_c(z(t),w(t))\\
&\leq2\liminf_{t\to  t_0^-}\mathcal{H}_c(z(t),w(t))\\
&\leq2\mathcal{H}_c(z_0,w_0)
<d(c).
\end{align*}
This is a contradiction, which shows that $\mathcal{R}_c^1$ is
invariant under the flow for the modulated system \eqref{modsys}. In
a similar fashion we have that  $\mathcal{R}_c^2$ is also invariant
under the flow for the modulated system \eqref{modsys}.
\end{proof}

The following lemma will be used to obtain stability with respect to
the ground state solutions.

\begin{lemma}\label{prelem}
Let $\sigma>3/8$ and $0<c_0<1$ be near $1$. If $U(t)=(\eta(t),
\Phi(t))$ is a global solution of the Boussinesq-Benney-Luke system
\eqref{bbl} with initial condition $U(0)=U_0\in\mathcal X$, then for
every M, there is $\delta(M)$ such that if
$$
\|U_0-U^{c_0}\|_{\mathcal X}<\delta(M).
$$
Then we have
$$
d\Big(c_0+\frac1M\Big)\leq\frac13I_{c_0}(U(t))\leq
d\Big(c_0-\frac1M\Big), \quad \text{for all }  t\in\mathbb{R}.
$$
\end{lemma}

\begin{proof}
Let $M$ be fixed and define $c_1=c_0-\frac1M$ and $c_2=c_0+\frac1M$.
Now, let $(z^{i}(t),w^{i}(t))$ be defined by the formulas
$$
\eta(t,x,y)= z^{i}(t,x-c_it,y) , \quad
\Phi(t,x,y)=w^{i}(t,x-c_it,y), \quad   i=1,2.
$$
Then the couple $(z^{i}(t),w^{i}(t))$ satisfies the modulated system
\eqref{modsys} with initial condition
$$
(z^{i}(0),w^{i}(0))=U(0).
$$
For this solution we have that the modulated Hamiltonian  is
conserved in time, in other words
$$
\mathcal{H}_{c_i}(U(t))=\mathcal{H}_{c_i}(U(0)).
$$
Now, using hypothesis and inequality \eqref{eqn} we conclude for
small $\delta$ that
$$
I_{c_i}(U^{c_0})=I_{c_i}(U(0))+O(\delta).
$$
Since $d$ is a strictly decreasing function such that
$d(c_0)=\frac13I_{c_0}(U^{c_0})$, we can choose $\delta$ small
enough in such a way that
$$
d(c_2)<\frac{1}3I_{c_0}(U(0))<d(c_1).
$$
We also have that
\begin{align*}
J_{c_i}(U(0))
&=J_{c_i}(U^{c_0})+O(\delta)\\
&=J_{c_0}(U^{c_0})+\frac{c_i-c_0}{c_0}I_{2,c_0}(U^{c_0})+O(\delta)\\
&=d(c_0)+\frac{c_i-c_0}{c_0}I_{2,c_0}(U^{c_0})+O(\delta)\\
&\leq d(c_i)-\rho(c_i)+O(\delta),
\end{align*}
where we have make used of Theorem \ref{dc-dc0}. Next, let $\delta$
be small enough such that
$$
2\delta<\min\{\rho\bigl(c_0-\frac1M\bigr),
\rho\bigl(c_0+\frac1M\bigr)\}.
$$
This implies 
\begin{equation}\label{2H=J}
2\mathcal{H}_{c_i}\left(U(0)\right)=J_{c_i}\left(U(0)\right)<d(c_i).
\end{equation}
Then, using Lemma \ref{inv}, we have for all $t\in\mathbb{R}$ that
$$
\mathcal{H}_{c_i}\left(U(t)\right)<\frac12d(c_i), \  \  \
d\Bigl(c_0+\frac1M\Bigr)\leq\frac13I_{c_0}\left(U(t)\right)\leq
d\Bigl(c_0-\frac1M\Bigr).
$$
\end{proof}

Finally we establish the main result in this work.

\begin{theorem}[Orbital stability]
Let $\sigma>3/8$ and $0<c_0<1$ be near 1. Then the ground state
solitary wave solutions of the Boussinesq-Benney-Luke system
\eqref{bbl} are stable in the following sense: Given
$\varepsilon>0$, there exist $\delta(\varepsilon)>0$ such that if
$U_0\in\mathcal X$ satisfies
$$
\|U_0-U^{c_0}\|_{\mathcal X}<\delta(\varepsilon),
$$
then there exist a unique solution $U(t)$ of the
Boussinesq-Benney-Luke system \eqref{bbl} with initial condition
$U_0$ such that
$$
\inf_{V\in\mathcal{G}_{c_0}}\|U(t)-V\|_{\mathcal X}<\varepsilon, \quad
\text{for all } t\in\mathbb{R}.
$$
\end{theorem}

\begin{proof}
We will argue by contradiction. Suppose that there exist a positive
number $\varepsilon_0$, and sequences $\{t_k\}\subset\mathbb{R}$ and
$\{U_0^k\}\subset\mathcal X$, such that
$$
\lim_{k\to \infty}\|U_0^k-U^{c_0}\|_{\mathcal X}=0, \quad
\inf_{V\in\mathcal{G}_{c_0}}\|U^k(t_k)-V\|_{\mathcal
X}>\varepsilon_0,
$$
where $U^k$ denotes the unique solution of system \eqref{bbl} with
initial condition $U^k(0)=U_0^k$. Now, from the Lemma \ref{prelem}
and the assumption, given $m>0$ we have the existence of $\delta(m)$
and a subsequence $k_m$ such that
$$
\|U_0^{k_m}-U^{c_0}\|_{\mathcal X}<\delta(m)
$$
and
$$
d\Bigl(c_0+\frac1{k_m}\Bigr)\leq\frac13I_{c_0}\left(U^{k_m}(t_{k_m})\right)\leq
d\Bigl(c_0-\frac1{k_m}\Bigr),
$$
meaning that there exist a subsequence of $\{U^k(t_k)\}$, which we
denote the same, such that
$$
d\Bigl(c_0+\frac1{k}\Bigr)\leq\frac13I_{c_0}\left(U^{k}(t_{k})\right)\leq
d\Bigl(c_0-\frac1{k}\Bigr).
$$
In particular, we have that
$$
\frac13I_{c_0}\left(U^{k}(t_{k})\right)\to d(c_0) \,\,\,
\text{ as}\,\,\,\, k\to \infty.
$$
Now, we consider $c_2=c_0+\frac1k$ and $V^{k,2}(t)$ defined by
$$U^k(t,x,y)=V^{k,2}(t,x-c_2t,y).$$
Then as in proof of previous lemma (see \eqref{2H=J}), we obtain
that
$$
2\mathcal{H}_{c_2}\left(U^{k}(t_{k})\right)=J_{c_2}\left(U^{k}(t_{k})\right)
<d(c_2)<d(c_0) <d\Bigl(c_0-\frac1{k}\Bigr).
$$
On the other hand,
\begin{align*}
J_{c_2}\left(U^{k}(t_{k})\right)
&=J_{c_0}\left(U^{k}(t_{k})\right)
+\Big(\frac{c_2-c_0}{c_0}\Big)I_{2,c_0}\left(U^{k}(t_{k})\right)\\
&=J_{c_0}\left(U^{k}(t_{k})\right)
+\Big(\frac{1}{kc_0}\Big)I_{2,c_0}\left(U^{k}(t_{k})\right).
\end{align*}
But note that
$$
\lim_{k\to \infty}
\Big(\frac{1}{kc_0}\Big)\left|I_{2,c_0}\left(U^{k}(t_{k})\right)\right|
\leq\lim_{k\to \infty}\frac1k\|U^{k}(t_{k})\|^2_{\mathcal
X}\leq\lim_{k\to \infty}\Bigl(\frac1k\,C\Bigr)=0,
$$
since we have that
$$
\|U^{k}(t_{k})\|^2_{\mathcal
X}\cong\frac13I_{2,c_0}\left(U^{k}(t_{k})\right)\to  d(c_0).
$$
Using these facts, we conclude that
$$
J_{c_0}\left(U^{k}(t_{k})\right)\to \tilde d\leq d(c_0).
$$
Then by Corollary \ref{d1dc}, there exist $U_{c_0}\in\mathcal{G}_{c_0}$ 
such that as $k\to \infty$,
$$
U^k(t_k)\to U_{c_0} \quad \text{in }  \mathcal X,  \quad
\frac13I_{c_0}\left(U^{k}(t_{k})\right)\to d(c_0)=\tilde d,
$$
also  $J_{c_0}\left(U^{k}(t_{k})\right)\to
d(c_0)$. But this contradicts the assumption of instability
$$
\inf_{V\in\mathcal{G}_{c_0}}\|U^k(t_k)-V\|_{\mathcal
X}>\varepsilon_0.
$$
\end{proof}

\subsection* {Acknowledgments} 
A. M. Montes was supported by the
Universidad del Cauca (Colombia) under the project I.D. 3982. J. R.
Quintero was supported by the Mathematics Department at Universidad
del Valle (Colombia) under the project CI 7001. A. M. and J. Q. are
supported by Colciencias grant No 42878.

\begin{thebibliography}{00}

\bibitem{BSS} J. Bona, P. Souganidis, W. Strauss;
\emph{Stability and instability of solitary waves of Korteweg de Vries
type equation}. Proc. Roy. Soc. London. Ser  A 411 (1987), 395-412.

\bibitem{DBJC} A. de Bouard, J. C. Saut;
\emph{Remarks on the stability of generalized KP solitary waves}. 
Comtemp. Math. 200 (1996), 75-84.

\bibitem{Fukuizumi} R. Fukuizumi;
\emph{Stability and instability of standing waves for the nonlinear
Schr\"odinger equation with harmonic potential}. Discrete contin.
Dynam. Syst.  7 (2001), 525-544.

\bibitem{GSS} M. Grillakis, J. Shatah, W. Strauss;
\emph{Stability Theory of Solitary Waves in Presence of Symmetry, I.}
Functional Anal. 74 (1987), 160-197.

\bibitem{YLXP} Y. Liu, X-P. Wang;
\emph{Nonlinear stablitity of solitary waves of a generalized
 Kadomtsev-Petviashvili equation}. Comm. Math.
Phys. 183 (1997), 253-266.

\bibitem{AMM} A. M. Montes;
\emph{Boussinesq-Benney-Luke type systems related with water wave
models}. Doctoral Thesis, Universidad del Valle (Colombia), 2013.

\bibitem{JQAM2013c} A. M. Montes, J. R. Quintero;
\emph{On the nonlinear scattering and well-posedness
for a 2D-Boussinesq-Benney-Luke type system}.  Submmited to Journal
Diff. Eq. (2013).

\bibitem{Q1} J. R. Quintero;
\emph{Nonlinear stability of a one-dimensional Boussinesq equation}.
Dynamic of PDE 15 (2003), 125-142.

\bibitem{JQ2005} J. R. Quintero;
\emph{Nonlinear stability of solitary waves for a 2-D Benney-Luke equation}.
 Discrete Contin. Dynam. Systems 13 (2005), 203-218.

\bibitem{JQ2010b} J. R. Quintero;
\emph{The Cauchy problem and stability of solitary waves for a 2D
Boussinesq-KdV type system}. Diff. and Int. Eq. 23 (2010), 325-360.

\bibitem{JQAM2013} J. R. Quintero, A. M. Montes;
\emph{Existence, physical sense and analyticity
of solitons for a 2D Boussinesq-Benney-Luke System}. Dynamics of PDE
10 (2013), 313-342.

\bibitem{jq-jm} J. R. Quintero, J. C. Mu\~noz;
\emph{Solitons and nonlinear stability/instability for a 1D
Benney-Luke-Paumond Equation}. In preparation (2014).

\bibitem{JQRP1999} J. R. Quintero,  R. L. Pego;
\emph{Two-dimensional solitary waves for a Benney-Luke equation}. Physica
D 45 (1999), 476-496.

\bibitem{Shatah} J. Shatah;
\emph{Stable standing waves of nonlinear Klein-Gordon equations}. 
Comm. Math. Phys. A.  91 (1983), 313-327.

\end{thebibliography}

\end{document}