\documentclass[reqno]{amsart}

\AtBeginDocument{{\noindent\small
{\em Electronic Journal of Differential Equations},
Vol. 2003(2003), No. 56, pp. 1--8.\newline
ISSN: 1072-6691. URL: http://ejde.math.swt.edu or http://ejde.math.unt.edu
\newline ftp ejde.math.swt.edu  (login: ftp)}
\thanks{\copyright 2003 Southwest Texas State University.}
\vspace{9mm}}

\begin{document}
\title[\hfilneg EJDE--2003/56\hfil Existence of solutions for elliptic systems]
{Existence of solutions for a class of elliptic systems in $\mathbb{R}^N$
involving the $p$-Laplacian}

\author[Ali Djellit \& Saadia Tas\hfil EJDE--2003/56\hfilneg]
{Ali Djellit \& Saadia Tas}

\address{Ali Djellit \hfill\break 
University of Annaba, Faculty of Sciences \\
Department of Mathematics \\
B.P. 12, 23000, Annaba, Algeria}
\email{a\_djellit@hotmail.com}

\address{Saadia Tas \hfill\break 
University of Annaba, Faculty of Sciences \\
Department of Mathematics \\
B.P. 12, 23000, Annaba, Algeria}
\email{tas\_saadia@yahoo.fr}

\date{}
\thanks{Submitted March 24, 2003. Published Maay 8, 2003.}
\subjclass[2000]{35P65, 35P30}

\keywords{$p$-Laplacian, nonlinear elliptic system, mountain pass theorem, 
\hfill\break\indent Palais-Smale condition}

\begin{abstract}
  Using a variational approach, we study a class of nonlinear elliptic 
  systems derived from a potential and involving the $p$-Laplacian.  
  Under suitable assumptions on the nonlinearities, we show the 
  existence  of nontrivial solutions.
\end{abstract}

\maketitle

\numberwithin{equation}{section} 
\newtheorem{theorem}{Theorem}[section] 
\newtheorem{lemma}[theorem]{Lemma}

\section{Introduction}

In this paper, we deal with the nonlinear elliptic system 
\begin{equation}
\begin{gathered} -\Delta_pu=\frac{\partial F}{\partial
u}(x,u,v)\quad\text{in } \mathbb{R}^N,\\ -\Delta_qv=\frac{\partial
F}{\partial v}(x,u,v)\quad\text{in } \mathbb{R}^N .\end{gathered}  \label{S}
\end{equation}
The nonlinearities on the right hand side are the gradient of a $C^{1}$%
-functional $F$ and $\triangle_p$ is the so-called $p$-Laplacian operator
i.e. $\Delta_pu=\mathop{\rm div}(|\nabla u|^{p-2}\nabla u)$; $u$ and $v$ are
unknown real-valued functions defined in $\mathbb{R}^N $ and belonging to
appropriate function spaces; $1<p$, $q<N$. Many authors studied the
existence of solutions for such problems (equations or systems) for which
explicit solutions generally can not be given.

We observe that there exists a vast literature on the use of the Mountain
Pass Theorem. Before stating our main theorem, we recall some work about
some nonlinear problems: Rabinowitz \cite{r2} investigated a class of
superlinear Schr\"{o}dinger equations of the form $-\Delta u+q(x)u=f(x,u)$
defined in $\mathbb{R}^{N}$ using a variational approach based on a variant
of the Mountain Pass Theorem. The nonlinear function verifies 
$|{\partial f/}{\partial u}|\leq a|u|^{p-1}+b$; $a$ and $b$ are positive 
constants; $1<p<2^{\ast }-1$ (in this case, the equation is said superlinear). 
Yu \cite{y1} obtained sufficient conditions on the nonlinearity for the existence of
positive solutions for some nonlinear equations of the form 
$-\mathop{\rm div}(\left| a(x)\nabla u\right| ^{p-2}\nabla u)
=b(x)|u|^{p-2}u+f(x,u)$
defined on a smooth exterior domains; $a(x)$ and $b(x)$ are smooth
functions. Costa \cite{c1} studied a class of elliptic systems $-\Delta
u+a(x)u=f(x,u,v);$ $-\Delta v+b(x)v=g(x,u,v)$ in $\mathbb{R}^{N}$; with 
$(f,g)=\nabla F$, the potential $F$ is nonquadratic at infinity. The partial
derivatives satisfy the conditions $|\nabla f(x,U)|+|\nabla g(x,U)|\leq
c(1+|U|^{p-1});1<p<2^{\ast }-1$ if $N\geq 3$ $($or $1\leq p<+\infty $ if $%
N=1,2)$. Do O \cite{o1} considered a non-autonomous perturbed eigenvalue
problem involving the $p$-Laplacian of the form $-\Delta _{p}u=f(x,u)$
defined in $\mathbb{R}^{N};$ the nonlinearity $f$ interacts with the first
eigenvalue of a corresponding problem, and also verifies the following
estimate $|f(x,u)|\leq \varphi (x)|u|^{r}+\psi (x)|u|^{s}$; $\varphi (x)$
and $\psi (x)$ are suitable functions; $0<r\leq p-1\leq s<p^{\ast }-1$.

In this work, we show the existence of nontrivial solutions for System 
\eqref{S} in homogeneous Sobolev spaces under mixed subcritical growth
conditions; the primitive $F$ being intimately connected with the first
eigenvalue of an appropriate system. Using a weak version of the
Palais-Smale condition, due to Cerami \cite{c2}, we can apply the Mountain
Pass Theorem to System \eqref{S}. Our main goal in this article is to
illustrate how the ideas introduced in \cite{c1,o1,r2} can be applied to
handle the problem of existence of nontrivial solutions for System \eqref{S}.

This paper is organized as follows. In section 2, we present some
preliminary results and definitions; we also introduce precise assumptions
under which our problem is studied. We reserve the section 3 for the proof
of the main result.

\section{Notations and Hypotheses}% sec 2

We first recall some standard definitions and notations. Let $Z$ be a
reflexive Banach space endowed with a norm $\|\cdot \|$. Let $I\in
C^{1}(Z,\mathbb{R})$. We say that $I$ satisfies the Cerami condition,
denoted by (C) condition, if every $(w_{n})\in Z$ such that 
\begin{equation*}
|I(w_{n})|\leq c\quad \text{and}\quad (1+\|w_{n}\|)I'(w_{n})\rightarrow 0
\end{equation*}
contains a convergent subsequence in the norm of $Z$.

For $1<m<N$, let $m^{\ast }=\frac{Nm}{N-m}$ be the critical Sobolev exponent
of $m$.

Let $D^{1,m}(\mathbb{R}^N )$ be the closure of $C_{0}^{\infty }(\mathbb{R}^N
)$ with respect to the norm $\| u\| _{1,m}\equiv \| \nabla u\|_m =\Big(
\int_{\mathbb{R}^N }|\nabla u|^{m}dx\Big)^{1/m}$. $D^{1,m}(\mathbb{R}^N )$
is a reflexive Banach space and may be written $D^{1,m}(\mathbb{R}^N
)=\{u\in L^{m^{\ast }}(\mathbb{R}^N ):\nabla u\in (L^{m}(\mathbb{R}^N ))^N \}
$. Moreover, Sobolev imbedding holds; in fact there exists a positive
constant $c$ such that $\| u\|_{m^{\ast }}\leq c\| u\|_{1,m}$ for all $u\in
D^{1,m}(\mathbb{R}^N )$ (see \cite{m1}).

Now we denote by $Z$ the product space $D^{1,p}(\mathbb{R}^N )\times D^{1,q}(%
\mathbb{R}^N )$ ; $Z^{^{\ast }}$ designates the dual space equipped with the
dual norm $\|\cdot\|_{\ast }$.

For $(u,v)$ in $Z$, we define the functionals $I$, $J$, $K$ by 
\begin{gather*}
J(u,v)=\frac{1}{p}\| u\|_{1,p}^p +\frac{1}{q}\| v\|_{1,q}^q , \\
K(u,v)=\int_{\mathbb{R}^N }F(x,u(x),v(x))dx, \\
I(u,v)=J(u,v)-K(u,v).
\end{gather*}

In this article we use the following hypotheses:

\begin{itemize}
\item[(H1)]  $F\in C^{1}(\mathbb{R}^{N}\times \mathbb{R}^{2},\mathbb{R})$
and $F(x,0,0)=0$.

\item[(H2)]  For all $U=(u,v)\in \mathbb{R}^{2}$ and for almost every $x\in 
\mathbb{R}^{N}$ 
\begin{gather*}
|\frac{\partial F}{\partial u}(x,U)|\leq
a_{1}(x)|U|^{p_{1}-1}+a_{2}(x)|U|^{p_{2}-1}, \\
|\frac{\partial F}{\partial v}(x,U)|\leq
b_{1}(x)|U|^{q_{1}-1}+b_{2}(x)|U|^{q_{2}-1}.
\end{gather*}
Here $1<p_{1}$, $q_{1}<\min (p,q)$, $\max (p,q)<p_{2}$, $q_{2}<\min (p^{\ast
},q^{\ast })$, $a_{i}\in L^{\alpha _{i}}(\mathbb{R}^{N})\cap L^{\beta _{i}}(%
\mathbb{R}^{N})$, $b_{i}\in L^{\gamma _{i}}(\mathbb{R}^{N})\cap L^{\delta
_{i}}(\mathbb{R}^{N})$, $i=1,2$. 
\begin{gather*}
\alpha _{i}=\frac{p^{\ast }}{p^{\ast }-p_{i}},\quad \gamma _{i}=\frac{%
q^{\ast }}{q^{\ast }-q_{i}}, \\
\beta _{i}=\frac{p^{\ast }q^{\ast }}{p^{\ast }q^{\ast }-p^{\ast
}(p_{i}-1)-q^{\ast }},\quad \delta _{i}=\frac{p^{\ast }q^{\ast }}{p^{\ast
}q^{\ast }-q^{\ast }(q_{i}-1)-p^{\ast }}.
\end{gather*}

\item[(H3)]  $U.\nabla F(x,U)-F(x,U)\leq 0$, for all $(x,U)\in \mathbb{R}%
^{N}\times \mathbb{R}^{2}-\{(0,0)\}$, where $\nabla F=(\frac{\partial F}{%
\partial u},\frac{\partial F}{\partial v})$. This type of condition has been
introduced by Costa \cite{c1}.

\item[(H4)]  At last we suppose the existence of two positive and bounded
functions $a\in L^{N/p}(\mathbb{R}^{N})$ and $b\in L^{N/q}(\mathbb{R}^{N})$
such that 
\begin{equation*}
\limsup_{|U|\rightarrow 0}\frac{pq|F(x,U)|}{qa(x)|u|^{p}+pb(x)|v|^{q}}%
<\lambda _{1}<\liminf_{|U|\rightarrow +\infty }\frac{pq|F(x,U)|}{%
qa(x)|u|^{p}+pb(x)|v|^{q}}.
\end{equation*}
\end{itemize}
Putting 
\begin{equation*}
\Lambda =\Big\{(u,v)\in Z:\frac{1}{p}\int_{\mathbb{R}^N }a(x)|u|^p dx 
+\frac{1}{q}\int_{\mathbb{R}^N }b(x)|v|^q dx=1\Big\}, 
\end{equation*}
$\lambda_1=\inf_{\Lambda }J(u,v)$ is the first eigenvalue of the system 
\begin{equation}
\begin{gathered} -\Delta_pu=\lambda a( x) | u| ^{p-2}u\quad\text{in
}\mathbb{R}^N,\\ -\Delta_qv=\lambda b( x) | v|^{q-2}v\quad\text{in
}\mathbb{R}^N.\\ \end{gathered}  \label{SVp}
\end{equation}

\noindent\textbf{Remark } The hypothesis (H4) is related with the
interaction of the potential $F$ and $\lambda_1$. Costa \cite{c1} was the
first to introduce such assumption. A variant of this condition appeared in
Do O \cite{o1}. An example of such functions is 
\begin{equation*}
F(x,u,v)=-a(x)| u| ^{\alpha }| v| ^{\beta };\quad \max (p,q)<\alpha ,\;
\beta <\min (p^{\ast },q^{\ast }).
\end{equation*}
It is easy to prove that $F$ satisfies (H1), (H3) and (H4). In order to
obtain (H2), we use Young's inequality.

\section{Existence of solutions}

Taking into account the above hypotheses, we have some assertions.

\begin{lemma} \label{lm1}
Under Hypotheses (H1) and (H2), the functional $K$ is well
defined and is of class $C^{1}$ on $Z$. Moreover, its derivative is
\[
K'(u,v)(w,z)=\int_{\mathbb{R}^N }(\frac{\partial F}{\partial u}%
(x,u,v)w+\frac{\partial F}{\partial v}(x,u,v)z)dx\quad \forall
(u,v),(w,z)\in Z.
\]
\end{lemma}

\begin{proof}
$K$ is well defined on $Z$. Indeed, for all $(u,v)\in Z$ and by virtue of
(H1) and (H2), we have
\begin{align*}
F(x,u,v)& =\int_{0}^{u}\frac{\partial F}{\partial s}(x,s,v)ds+F(x,0,v) \\
& =\int_{0}^{u}\frac{\partial F}{\partial s}(x,s,v)ds+\int_{0}^{v}\frac{%
\partial F}{\partial s}(x,0,s)ds+F(x,0,0)
\end{align*}
and
\begin{align*}
F(x,u,v)\leq &
c_1[a_1(x)(|u|^{p_1}+|v|^{p_1-1}|u|)+a_2 (x)(|u|^{p_2 }+|v|^{p_2 -1}|u|)
\\
& +b_1(x)|v|^{q_1}+b_2 (x)|v|^{q_2 }].
\end{align*}
Using H\"{o}lder's inequality and Sobolev's imbedding and the fact that 
$a_{i}\in L^{\alpha_{i}}(\mathbb{R}^N )\cap L^{\beta_{i}}(\mathbb{R}^N )$,
$b_{i}\in L^{\gamma_{i}}(\mathbb{R}^N )$, we get
\begin{align*}
& \int_{\mathbb{R}^N }F(x,u,v)dx \\
& \leq c_2 (\| a_1\|_{\alpha_1}\| u\|
_{1,p}^{p_1}+\| a_1\|_{\beta_1}\| v\|
_{1,q}^{p_1-1}\| u\|_{1,p}+\| a_2 \|_{\alpha
_2 }\|u\|_{1,p}^{p_2 } \\
& \quad +\| a_2 \|_{\beta_2 }\| v\|
_{1,q}^{p_2 -1}\| u\|_{1,p}+\| b_1\|_{\gamma_1}\|
v\|_{1,q}^{q_1}+\| b_2 \|_{\gamma_2 }\| v\|
_{1,q}^{q_2 })<+\infty .
\end{align*}
Observe that $K'(u,v)$ is also well defined on $Z$ since
\begin{align*}
\int_{\mathbb{R}^N }\frac{\partial F}{\partial u}(x,u,v)wdx\leq & c_{3}
\Big(\| a_1\|_{\alpha_1}\| u\|_{1,p}^{p_1-1}+\|
a_1\|_{\beta_1}\| v\|_{1,q}^{p_1-1} \\
& +\| a_2 \|_{\alpha_2 }\| u\|
_{1,p}^{p_2 -1}+\| a_2 \|_{\beta_2 }\| v\|
_{1,q}^{p_2 -1}\Big)\| w\|_{1,p}<+\infty .
\end{align*}
and
\begin{align*}
\int_{\mathbb{R}^N }\frac{\partial F}{\partial v}(x,u,v)zdx\leq & c_{4}
\Big(\| b_1\|_{\delta_1}\| u\|_{1,p}^{q_1-1}+\|
b_1\|_{\gamma_1}\| v\|_{1,q}^{q_1-1} \\
& +\| b_2 \|_{\delta_2 }\| u\|
_{1,p}^{q_2 -1}+\| b_2 \|_{\gamma_2 }\| v\|
_{1,q}^{q_2 -1}\Big)\| z\|_{1,q}<+\infty .
\end{align*}
Now, we show that $K$ is differentiable in sense of Fr\'{e}chet at
each point $(u,v)$ of $Z$ i.e. $\forall \varepsilon >0$, there
exists $\delta =\delta (\varepsilon ,u,v)>0$ such that $(\|
w\|_{1,p}+\| z\|_{1,q})\leq \delta $ implies
\begin{equation*}
|K(u+w,v+z)-K(u,v)-K'(u,v)(w,z)|\leq \varepsilon (\| w\|
_{1,p}+\| z\|_{1,q}).
\end{equation*}
Let $B_{R}$ be the ball of radius $R$, centered at the origin of
$\mathbb{R}^N $. We put $B_{R}'=\mathbb{R}^N -B_{R}$ and we define a
functional $K_{R}$ on $D^{1,p}(B_{R})\times D^{1,q}(B_{R})$ by
$K_{R}(u,v)=\int_{B_{R}}F(x,u(x),v(x))dx$. Taking (H1) and (H2) into account,
it is well-known that $K_{R}\in C^{1}(D^{1,p}(B_{R})\times D^{1,q}(B_{R}))$
and for any $(w,z)\in D^{1,p}(B_{R})\times D^{1,q}(B_{R})$, we have
\begin{equation*}
K_{R}'(u,v)(w,z)=\int_{B_{R}}(\frac{\partial F}{\partial u}(x,u,v)w+%
\frac{\partial F}{\partial v}(x,u,v)z)dx.
\end{equation*}
Moreover, $K_{R}'$ is compact from $Z$ to $Z^{*}$ (see
\cite{o1,r1,s1}). On the other hand, for all $(u,v)$ , 
$(w,z)\in Z$, we have
\begin{align*}
& \big|K(u+w,v+z)-K(u,v)-K'(u,v)(w,z)\big| \\
& \leq \big|K_{R}(u+w,v+z)-K_{R}(u,v)-K_{R}'(u,v)(w,z)\big| \\
& +\big|\int_{B_{R}'}(F(x,u+w,v+z)-F(x,u,v)
-\frac{\partial F}{\partial u}(x,u,v)w-\frac{\partial F}{\partial v}(x,u,v)z)dx\big|.
\end{align*}
By the Mean-value theorem, we can write
\begin{equation*}
F(x,u+w,v+z)-F(x,u,v)=\frac{\partial F}{\partial u}(x,u+\theta_1w,v)w+%
\frac{\partial F}{\partial v}(x,u,v+\theta_2 z)z,
\end{equation*}
for $\theta_1,\theta_2 \in ]0,1[$.
By the growth condition (H2) and the fact that for $i=1,2$,
\begin{equation}
\begin{gathered} \| a_{i}\|_{L^{\alpha_{i}}(B'_{R})}+\| a_{i}\|
_{L^{\beta_{i}}(B'_{R})}\to 0, \\ \|
b_{i}\|_{L^{\gamma_{i}}(B'_{R})}+\|
b_{i}\|_{L^{\delta_{i}}(B'_{R})}\to 0, \end{gathered}  \label{9}
\end{equation}
as $R\to \infty$, we obtain for $R$ sufficiently large that
\begin{align*}
& \big|\int_{B_{R}'}(F(x,u+w,v+z)-F(x,u,v)-\frac{\partial F}{%
\partial u}(x,u,v)w-\frac{\partial F}{\partial v}(x,u,v)z)dx\big| \\
& \leq \varepsilon \big(\| w\|_{1,p}+\| z\|
_{1,q}\big).
\end{align*}
We have only to show that $K'$ is continuous on $Z$. Let $%
(u_n ,v_n )\to (u,v)$ in $Z$. For $(w,z)\in Z$ , we have
\begin{align*}
& |K'(u_n ,v_n )(w,z)-K'(u,v)(w,z)| \\
& =|K_{R}'(u_n ,v_n )(w,z)-K_{R}'(u,v)(w,z)| \\
& \quad +|\int_{B_{R}'}(\frac{\partial F}{\partial u}%
(x,u_n ,v_n )+\frac{\partial F}{\partial u}(x,u,v))wdx| \\
& \quad +|\int_{B_{R}'}(\frac{\partial F}{\partial v}%
(x,u_n ,v_n )+\frac{\partial F}{\partial v}(x,u,v))zdx|.
\end{align*}
Then $K_{R}'$ is continuous on $D^{1,p}(B_{R})\times
D^{1,q}(B_{R})$ (see \cite{o1,r2}). The first expression on the
right hand side of the above equation tends to 0 as
$n\to +\infty $; we use (H2) and \eqref{9} to prove that both the second
and the third expressions tend also to 0 as $R$ sufficiently
large.
\end{proof}

\noindent\textbf{Remark } The functional $J$ is of class $C^{1}$ on $Z$ and
its derivative is 
\begin{equation*}
J'(u,v)(w,z)=\int_{\mathbb{R}^N }|\nabla u|^{p-2}\nabla u\nabla
wdx+\int_{\mathbb{R}^N }|\nabla v|^{q-2}\nabla v\nabla zdx.
\end{equation*}

\begin{lemma} \label{lm2}
Under assumptions (H1) and (H2), $K'$ is compact from $Z$ to $Z^*$.
\end{lemma}

\begin{proof}
Let $(u_n ,v_n )$ be a bounded sequence in $Z$. Then there is a
subsequence denoted again $(u_n ,v_n )$ weakly convergent to $(u,v)$ in $Z$.
As before, we write
\begin{align*}
& |K'(u_n ,v_n )(w,z)-K'(u,v)(w,z)| \\
& =|K_{R}'(u_n ,v_n )(w,z)-K_{R}'(u,v)(w,z)| \\
& \quad +|\int_{B_{R}'}(\frac{\partial F}{\partial u}%
(x,u_n ,v_n )-\frac{\partial F}{\partial u}(x,u,v))w\,dx| \\
& \quad +|\int_{B_{R}'}(\frac{\partial F}{\partial v}%
(x,u_n ,v_n )-\frac{\partial F}{\partial v}(x,u,v))z\,dx|.
\end{align*}
Since the restriction operator is continuous, we have $(u_n ,v_n )%
\rightharpoonup (u,v)$ in $D^{1,p}(B_{R})\times D^{1,q}(B_{R})$.
Because of the compactness of $K_{R}'$, the first expression on
the right hand side of the equation tends to 0 as $n\to
+\infty $; as above both the second and the third expressions tend
also to 0 as $R$ sufficiently large.
\end{proof}

\begin{lemma} \label{lm3}
If (H1), (H2), and (H3) hold then $I=J-K$ satisfies the condition (C).
\end{lemma}

\begin{proof}
Let $( u_n ,v_n ) \subset Z$ such that
\begin{itemize}
\item[(i)]  $|I(u_n ,v_n )|\leq c$.
\item[(ii)]  $(1+\| u_n \|_{1,p}+\| v_n \|_{1,q})I'(u_n ,v_n )\to 0$ in
$Z^{^{\ast }}$, as $n\to +\infty $.
\end{itemize}

From (ii), we have $I'(u_n ,v_n )(w,z)\leq \varepsilon _n
\to 0$ as $n\to +\infty $, $\forall $ $(w,z)\in
Z$. In particular, for $(w,z)=(u_n ,v_n )$, we get
\begin{align*}
& I'(u_n ,v_n )(u_n ,v_n ) \\
& =\| u_n \|_{1,p}^p +\| v_n \|_{1,q}^q -\int_{\mathbb{R}^N }
(\frac{\partial F}{\partial u}(x,u_n ,v_n )u_n
+\frac{\partial F}{\partial v}(x,u_n ,v_n )v_n )dx\leq \varepsilon_n .
\end{align*}
On the other hand,
\begin{equation*}
I(u_n ,v_n )=\frac{1}{p}\| u_n \|_{1,p}^p +\frac{1}{q}\|
v_n \| _{1,q}^q -\int_{\mathbb{R}^N }F(x,u_n ,v_n )dx\leq c.
\end{equation*}
Then, taking (H2) into account, we get
\begin{align*}
\varepsilon_n +c& \geq I'(u_n ,v_n )(u_n ,v_n )-I(u_n ,v_n )\\
& =(1-\frac{1}{p})\| u_n \|_{1,p}^p +(1-\frac{1}{q})\|
v_n \|_{1,q}^q  \\
& \quad +\int_{\mathbb{R}^N }(F(x,u_n ,v_n )-\frac{\partial F}{\partial u}%
(x,u_n ,v_n )u_n -\frac{\partial F}{\partial v}(x,u_n ,v_n )v_n )dx \\
& \geq (1-\frac{1}{p})\| u_n \|_{1,p}^p
+(1-\frac{1}{q})\| v_n \|_{1,q}^q .
\end{align*}
Hence, $(u_n ,v_n )$ is bounded in $Z$. There is a subsequence
denoted again $(u_n ,v_n )$ weakly convergent in $Z$. Since $K'$
is
compact, $K'(u_n ,v_n )$ is a Cauchy's sequence in $Z^{^{\ast }}$.
We have $J'(u,v)=I'(u,v)+K'(u,v),\forall (u,v)\in Z$ and
\begin{align*}
& \big(J'(u_n ,v_n )-J'(u_m ,v_m )\big)(u_n -u_m ,0)\\
& =\int_{\mathbb{R}^N }(|\nabla u_n |^{p-2}\nabla u_n -|\nabla u_m
|^{p-2}\nabla u_m )(\nabla u_n -\nabla u_m )dx.
\end{align*}
Observe that for all $\lambda ,\mu \in \mathbb{R}^N $,
\begin{equation*}
|\lambda -\mu |^p \leq
\begin{cases}
(|\lambda |^{p-2}\lambda -|\mu |^{p-2}\mu )(\lambda -\mu ) & \text{if }%
p\geq 2, \\
\left[ (|\lambda |^{p-2}\lambda -|\mu |^{p-2}\mu )(\lambda -\mu )\right]
^{p/2}(|\lambda |+|\mu |)^{(2-p)p/2} & \text{if }1<p<2,
\end{cases}
\end{equation*}
(see \cite{d1,z1}). Substituting $\lambda $ and $\mu $ by $\nabla
u_n $ and $\nabla u_m $ respectively and integrating over
$\mathbb{R}^N $, we obtain
\begin{equation*}
\| u_n -u_m \|_{1,p}^p \leq (J'(u_n ,v_n )-J'(u_m ,v_m
))(u_n -u_m ,0),\text{ if }p\geq 2.
\end{equation*}
and
\begin{equation*}
\| u_n -u_m \|_{1,p}^{2}\leq |(J'(u_n ,v_n )-J'(u_m ,v_m
))(u_n -u_m ,0)|(\| u_n \|_{1,p}^p +\| u_m \| _{1,p}^p
)^{\frac{2-p}{2}}\,,
\end{equation*}
if $1<p<2$.

Since $(u_n )$ is bounded in $D^{1,p}(\mathbb{R}^N )$ and
$(J'(u_n ,v_n )-J'(u_m ,v_m ))(u_n -u_m ,0)\to 0$ as $%
n,m\to +\infty $, then $(u_n )$ is a Cauchy's sequence in $D^{1,p}(%
\mathbb{R}^N )$. Hence $(u_n )$ converges in $D^{1,p}(\mathbb{R}^N
)$. In the same way, we prove that $(v_n )$ converges in
$D^{1,q}(\mathbb{R}^N )$.
\end{proof}
Moreover the functional $I=J-K$ verifies the geometric conditions of the
Mountain Pass Theorem, summarized in the following lemma.

\begin{lemma} \label{lm4}
Under Assumptions (H1), (H2), (H3), and (H4), the functional $I$
satisfies
\begin{itemize}
\item[(I1)]  There exist $\rho $, $\sigma >0$ such that $\| u\|
_{1,p}+\| v\|_{1,q}=\rho $ implies $I(u,v)\geq \sigma >0$.

\item[(I2)]  There exists $E\in Z$ such that $\| E\|_{Z}>\rho $ and $%
I(E)\leq 0$.
\end{itemize}
\end{lemma}

\begin{proof}
By (H4), there exists $\rho >0$ such that
\begin{equation*}
\| u\|_{1,p}+\| v\|_{1,q}=\rho \Longrightarrow
F(x,u,v)<\lambda _1(\frac{1}{p}a(x)|u|^p +\frac{1}{q}b(x)|v|^p ).
\end{equation*}
The variational characterization of $\lambda_1$ (see \cite{f1})
gives
\begin{equation*}
\int_{\mathbb{R}^N }F(x,u,v)dx<\frac{1}{p}\| u\|_{1,p}^p +\frac{1}{q%
}\| v\|_{1,q}^q .
\end{equation*}
Then there exist $\rho ,\sigma >0$ such that $\| u\|
_{1,p}+\| v\|_{1,q}=\rho $ implies $I(u,v)\geq \sigma >0$.
Let $(\varphi ,\psi )$ be an eigenfunction associated with
$\lambda_1$. In view of (H4), we get for $\varepsilon >0$ and $t$
sufficiently large,
\begin{equation*}
F(x,t^{\frac{1}{p}}\varphi ,t^{\frac{1}{q}}\psi )\geq (\lambda
_1+\varepsilon )(\frac{t}{p}a(x)|\varphi |^p +\frac{t}{q}b(x)|\psi
|^p ).
\end{equation*}
Hence
\begin{align*}
I(t^{\frac{1}{p}}\varphi ,t^{\frac{1}{q}}\psi )
& =\frac{t}{p}\int_{\mathbb{R}^N }|\nabla \varphi |^p dx
+\frac{t}{q}\int_{\mathbb{R}^N }|\nabla\psi |^q dx
-\int_{\mathbb{R}^N }F(x,t^{\frac{1}{p}}\varphi,t^{\frac{1}{q}}\psi)dx \\
& \leq \frac{t}{p}\int_{\mathbb{R}^N }|\nabla \varphi |^p dx+\frac{t}{q}
\int_{\mathbb{R}^N }|\nabla \psi |^q dx \\
& \quad -(\lambda_1+\varepsilon )\Big(\frac{t}{p}\int_{\mathbb{R}
^N }a(x)|\varphi |^p dx+\frac{t}{q}\int_{\mathbb{R}^N }b(x)|\psi |^p dx\Big) \\
& \leq -t\varepsilon (\frac{1}{p}\int_{\mathbb{R}^N }a(x)|\varphi |^p dx
+\frac{1}{q}\int_{\mathbb{R}^N }b(x)|\psi |^p dx).
\end{align*}
we deduce that $\lim_{t\to +\infty }I(t^{\frac{1}{p}}\varphi ,t^{1/q}
\psi )=-\infty $. So, for $t$ large, $I(t^{1/p}\varphi
,t^{1/q}\psi )\leq 0$.

Consequently, the functional $I$ has a critical value. Note that the
critical points of $I$ are precisely the weak solutions of System \eqref{S}.
\end{proof}

Now, we can state the main theorem.

\begin{theorem}\label{thm1}
System \eqref{S} has at least one nontrivial solution $(u,v)$.
\end{theorem}

\begin{proof}
In view of Lemmas \ref{lm3} and \ref{lm4}, we can apply the Mountain-Pass theorem (see \cite
{k1,r1,m1}) to conclude that system \eqref{S} has a nontrivial weak solution.
\end{proof}

\subsection*{Acknowledgments}
The authors want to express their gratitude to ANDRU for providing support
through the grant $CU39904$, and to the anonymous referee for his/her
helpful comments.

\begin{thebibliography}{99}
\bibitem{b1}  S. M. Berger, \textit{Nonlinearity and Functional Analysis. }%
Academic Press, New York, San Francisco, London, 1977.

\bibitem{b2}  P. Bartolo, V. Benci and D. Fortunato, \textit{Abstract
critical point theorems and applications to some nonlinear problems with
strong resonance at infinity. }Nonlinear Analysis TMA 7 (1983), 981-1012.

\bibitem{b3}  V. Benci and P. H. Rabinowitz, \textit{Critical point theorems
for indefinite functionals. }Invent. Math. 52 (1979), 241-273.

\bibitem{c1}  D. G. Costa, \textit{On a class of elliptic systems in }$%
\mathbb{R}^N $. E. J. D. E, Vol. 1994 (1994), No. 07, 1-14.

\bibitem{c2}  G. Cerami, \textit{Un criterio de esistenza per i punti
critici su variet\`{a} ilimitate.} Istituto Lombardo di Scienze e Lettere
112 (1978), 332-336.

\bibitem{d1}  G. Dinca and P. Jebelean, \textit{Some existence results for a
class of nonlinear equations involving a duality mapping. }Nonlinear
analysis 46 (2001), 347-363.

\bibitem{f1}  J. Ffleckinger, R. F. Man\'{a}sevich, N. M. Stavrakakis, F. de
Th\'{e}lin, \textit{Principal eigenvalues for some quasilinear elliptic
equation on }$R^N $. V.2, N.6, November 1997,pp.981-1003.

\bibitem{k1}  O. Kavian, \textit{Introduction \`{a} la th\'{e}orie des
points critiques et applications aux probl\`{e}mes elliptiques}.
Springer-Verlag France, Paris, 1993.

\bibitem{o1}  J. M. B. do O, \textit{Solutions to perturbed eigenvalue
problems of the p-Laplacian in }$\mathbb{R}^N $. Electronic J. Diff. Eqns.,
Vol. 11 (1997), 1-15.

\bibitem{r1}  P. H. Rabinowitz, \textit{Minimax Methods in Critical Point
Theory with Applications to Differential Equations. }CBMS Regional Conf.
Ser. in Math. 65, AMS, Providence, RI, 1986.

\bibitem{r2}  P. H. Rabinowitz, \textit{On a class of Nonlinear
Schr\"{o}dinger Equations}. z. angew. Math. Phys. 43 (1992), 270-291.

\bibitem{s1}  N. M. Stavrakakis and N. B. Zographopoulos, \textit{Existence
results for quasilinear elliptic systems in }$\mathbb{R}^N $. Electronic J.
Diff. Eqns., Vol. 1999 (1999), No. 39, 1-15.

\bibitem{m1}  M. Willem, \textit{Minimax Theorems, }Bikhauser,
Boston-Basel-Berlin, 1996.

\bibitem{y1}  L. S. Yu, \textit{Nonlinear p-Laplacian Problems on Unbounded
domains}. Proc. Amer. Math. Soc. 115 (1992) No. 4, 1037-1045.

\bibitem{z1}  E. Zeidler, \textit{Nonlinear Functional Analysis and its
Applications, Vol III Variational Methods and Optimization}. Springer
Verlag, Berlin 1990.
\end{thebibliography}

\end{document}