\documentclass[reqno]{amsart}
\usepackage{hyperref}

\AtBeginDocument{{\noindent\small
\emph{Electronic Journal of Differential Equations},
Vol. 2008(2008), No. 121, pp. 1--11.\newline
ISSN: 1072-6691. URL: http://ejde.math.txstate.edu or http://ejde.math.unt.edu
\newline ftp ejde.math.txstate.edu  (login: ftp)}
\thanks{\copyright 2008 Texas State University - San Marcos.}
\vspace{9mm}}

\begin{document}
\title[\hfilneg EJDE-2008/121\hfil Multiple solutions for a elliptic system]
{Multiple solutions for a elliptic system in exterior domain}

\author[H. Gu, J. Yang, X. Yu\hfil EJDE-2008/121\hfilneg]
{Huijuan Gu, Jianfu Yang, Xiaohui Yu}  % in alphabetical order

\address{Huijuan  Gu \newline
Department of Mathematics, Jiangxi Normal University, Nanchang,
Jiangxi 330022,  China}
\email{ruobing411@yahoo.com.cn}

\address{Jianfu  Yang \newline
Department of Mathematics, Jiangxi Normal University, Nanchang, Jiangxi
330022,  China}
\email{jfyang\_2000@yahoo.com}

\address{Xiaohui  Yu \newline
China Institute for Advanced Study, Central University of Finance
and Economics, Beijing 100081, China}
\email{yuxiao\_211@163.com}

\thanks{Submitted June 28, 2008. Published August 28, 2008.}
\subjclass[2000]{35J50, 35B32}
\keywords{Exterior domain; nonlinear elliptic system; existence result}

\begin{abstract}
 In this paper, we study the existence
 of solutions for the nonlinear elliptic system
 \begin{gather*}
 -\Delta u+u=|u|^{p-1}u+\lambda v  \quad \text{in }  \Omega,    \\
 -\Delta v+v=|v|^{p-1}v+\lambda u  \quad \text{in }  \Omega,    \\
 u=v=0  \quad \text{on }  \partial\Omega,
 \end{gather*}
 where $\Omega$ is a exterior domain in $\mathbb{R}^N$, $N\geq 3$.
 We show that the system possesses at least one nontrivial positive solution.
\end{abstract}

\maketitle
\numberwithin{equation}{section}
\newtheorem{theorem}{Theorem}[section]
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{remark}[theorem]{Remark}

\section{Introduction}

This article concerns the existence of solutions to the  semilinear
elliptic problem
\begin{equation}\label{1.1}
\begin{gathered}
-\Delta u+u=|u|^{p-1}u+\lambda v     \quad \text{in }  \Omega,    \\
-\Delta v+v=|v|^{p-1}v+\lambda u    \quad \text{in }  \Omega,    \\
u=v=0  \quad \text{on }  \partial\Omega,
\end{gathered}
\end{equation}
where $\Omega\subset \mathbb{R}^N, N\geq 3$, is an exterior domain,
$0<\lambda<1$ is a real parameter, $\partial \Omega\neq \emptyset$ and
$1<p<\frac {N+2}{N-2}$. In general, in a unbounded domain $\Omega$, the
inclusion of $H_0^1(\Omega)\hookrightarrow L^p(\Omega), 2\leq p<\frac
{2N}{N-2}$, is not compact,  the (PS) condition in critical point
theory does not satisfy for related functionals. In some special
cases, for instance, if $\Omega= \mathbb{R}^N$, $H_r^1(\Omega)$ is
compactly embedded in $L^p(\Omega), 2\leq p<\frac {2N}{N-2}$. Using the
fact, it was proved in \cite{BL1} that the problem
\begin{equation}\label{1.2}
-\Delta u+u=|u|^{p-1}u     \quad {\rm in } \quad \mathbb{R}^N
\end{equation}
possesses a positive solution and infinitely many solutions
respectively. The general case was considered in \cite{YZ}; i.e.,
problem
\begin{equation}\label{1.3}
  \begin{gathered}
-\Delta u+a(x)u=b(x)|u|^{p-1}u    \quad \text{in }  \mathbb{R}^N,    \\
u=0  \quad \text{on }  \partial\Omega.
\end{gathered}
\end{equation}
Suppose $a(x)\geq 0, b(x)\geq 0$ and $\lim_{|x|\to \infty}a(x)=\bar
a,\lim_{|x|\to \infty}b(x)=\bar b$, let $c_\Omega$ be the mountain pass
level of problem \eqref{1.3} and $c_\infty$ be the mountain pass
level of the limiting problem
\begin{equation}\label{1.4}
\begin{gathered}
-\Delta u+\bar au=\bar b|u|^{p-1}u    \quad \text{in }  \mathbb{R}^N,    \\
u\in H^1(\mathbb{R}^N).
\end{gathered}
\end{equation}
It was showed in \cite{YZ} that the $(PS)_c$ condition holds for the
associated functional of \eqref{1.3} provided that
$c\in (0, c_\infty)$. However, for problems defined in an exterior
domain, it
was proved in \cite{BC} that $c_\Omega=c_\infty$. One then has to look
for solutions with higher energy. Using barycenter function lifting
critical values up, a solution of \eqref{1.2} with the critical
value belonging in $(c_\infty,2c_\infty)$ was found in \cite{BC}.
The uniqueness of positive solution, up to a translation, of problem
\eqref{1.4} and the behavior of the solution at infinity play
crucial roles in insuring that there are no solutions with energy in
between $c_\infty$ and $2c_\infty$.

In this paper, we are interested in finding solutions of problem
\eqref{1.1}.  The limiting problem of \eqref{1.1} is
\begin{equation}\label{1.5}
  \begin{gathered}
-\Delta u+u=|u|^{p-1}u+\lambda v     \quad \text{in }  \mathbb{R}^N,    \\
\\-\Delta v+v=|v|^{p-1}v+\lambda u    \quad \text{in }  \mathbb{R}^N.
\end{gathered}
\end{equation}
In a recent paper \cite{ACR}, Ambrosetti, Cerami and Ruiz showed
that solutions of problem \eqref{1.5} bifurcating from the
semi-trivial solutions if $\lambda$ is sufficiently small. We will
show that ground state solutions of problem \eqref{1.5} are
obstacles preventing the global compactness of the associated
functional of problem \eqref{1.1}, and furthermore,  problem
\eqref{1.1} has no ground state solutions. So we have to find
solutions at higher energy levels. It is not known whether problem
\eqref{1.5} has unique positive solution or not. This brings
difficulties in finding solutions. Fortunately, it was showed in
\cite{ACR} that ground state levels of \eqref{1.5} are isolated if
$\lambda$ is sufficiently small or $\lambda<1$ and sufficiently
close to $1$.

Our main result is the following.

\begin{theorem}\label{thm1.1}
There exist $\delta>0$ and a constant $\bar\rho=\bar\rho(\lambda)$
such that if $\lambda\in(0,\delta)$ and
$$
\mathbb{R}^N\setminus \Omega\subset B_{\bar\rho}(x_0)=\{x\in
\mathbb{R}^N: |x-x_0|\leq \bar\rho\},
$$
problem \eqref{1.1} has at least three pairs of nontrivial
solutions.
\end{theorem}

Theorem \ref{thm1.1} will be proved by finding critical points of the
 corresponding functional of problem \eqref{1.1}
\begin{equation}\label{1.6}
\begin{aligned}
I(u,v)
&=\frac 12 \int_\Omega |\nabla u|^2+u^2\,dx +\frac 12 \int_\Omega
|\nabla v|^2+v^2\,dx \\
&\quad -\frac 1{p+1}\int_\Omega
|u|^{p+1}+|v|^{p+1}\,dx-\lambda\int_\Omega uv\,dx,
\end{aligned}
\end{equation}
where $(u,v)\in E= H_0^1(\Omega)\times H_0^1(\Omega)$. In section 2, we
show that ground state solutions are exponentially decaying at infinity
and that problem \eqref{1.1} has no ground state solution. In final
section, we prove Theorem \ref{thm1.1}.

\section{Preliminaries}

It was proved in \cite{ACR} that problem \eqref{1.5} has a ground
state solution $(u_\lambda,v_\lambda)$ for $0<\lambda<1$,  which is
positive and radially symmetric.

\begin{lemma}\label{thm2.1}
There exist  $\delta=\delta(\lambda)>0$ and $C>0$ such that
\begin{equation}\label{2.1}
|D^\alpha u_\lambda(x)|\leq Ce^{-\delta |x|},\quad
|D^\alpha v_\lambda(x)|\leq Ce^{-\delta |x|} \quad
 \forall x\in\mathbb{R}^N
\end{equation}
for $|\alpha|\leq 2$.
\end{lemma}

\begin{proof}
Let $w_\lambda=u_\lambda+v_\lambda$, then $w_\lambda$ satisfies
\begin{equation}\label{2.2}
-\Delta w_\lambda+w_\lambda=(u_\lambda^p+v_\lambda^p)+\lambda
w_\lambda, \quad {\rm in } \mathbb{R}^N.
\end{equation}
Since $w=w(r)$ is radially symmetric, let
$\phi(r)=r^{\frac{N-1}{2}}w_\lambda$, then $\phi$ satisfies
\begin{equation}\label{2.3}
\phi_{rr}=[q(r)+\frac {b}{r^2}]\phi
\end{equation}
with
$q(r)=\frac{(1-\lambda)w_\lambda-(u_\lambda^p+v_\lambda^p)}{w_\lambda}$
and $b=\frac {(N-1)(N-3)}{4}$. Since $u_\lambda$ and $v_\lambda$ are
radially symmetric,  $u_\lambda(r),v_\lambda(r)\to 0$ as $|x|\to
\infty$. There is $r_0>0$ such that $q(r)\geq \frac {1-\lambda} 2$
if $r\geq r_0$. Set $\psi=\phi^2$, then $\psi$ satisfies
\begin{equation}\label{2.4}
\frac 12 \psi_{rr}=\phi_r^2+(q(r)+\frac b {r^2})\psi,
\end{equation}
this implies that $\psi_{rr}\geq (1-\lambda)\psi$ for $r\geq r_0$.
Let $z=e^{-\sqrt{1-\lambda}r}[\psi_r+\sqrt{1-\lambda}\psi]$, we have
\begin{equation}\label{2.5}
z_r=e^{-\sqrt{1-\lambda}r}[\psi_{rr}-(1-\lambda)\psi]\geq 0
\end{equation}
for $r\geq r_0$. So $z$ is nondecreasing on $(r_0,+\infty)$. If
there exists $r_1>r_0$ such that $z(r_1)>0$, then $z(r)\geq
z(r_1)>0$ for $r\geq r_1$, that is
\begin{equation}\label{2.6}
\psi_{r}+\sqrt{1-\lambda}\psi\geq (z(r_1))e^{\sqrt{1-\lambda}r},
\end{equation}
implying that $\psi_{r}+\sqrt{1-\lambda}\psi$ is not integrable, a
contradiction to the fact that both $\psi$ and $\psi_r$ are
integrable. Hence, there holds
\begin{equation}\label{2.7}
(e^{\sqrt{1-\lambda}r}\psi)_r
=e^{\sqrt{1-\lambda}r}\psi_r+\sqrt{1-\lambda}e^{\sqrt{1-\lambda}r}\psi
=e^{2\sqrt{1-\lambda}r}z\leq 0
\end{equation}
for $r\geq r_0$. This implies
\begin{equation}\label{2.8}
\psi(r)\leq Ce^{-\sqrt{1-\lambda}r};
\end{equation}
i.e.,
\begin{equation}\label{2.9}
\phi(r)\leq Ce^{-\frac {\sqrt{1-\lambda}}2r}.
\end{equation}
By the definition of $\phi,w$ and the fact that
$u_\lambda,v_\lambda>0$ we have
\begin{equation}\label{2.10}
u_\lambda,v_\lambda\leq C r^{-\frac{N-1}2}e^{-\frac
{\sqrt{1-\lambda}}2r}.
\end{equation}
This proves (\ref{2.1}) with $\alpha=0$. Next we estimate the
derivatives of $u_\lambda,v_\lambda$. Since
\begin{equation}\label{2.11}
(r^{N-1}(u_\lambda)_r)_r=-r^{N-1}[-u_\lambda+u_\lambda^p+\lambda
v_\lambda],
\end{equation}
we have
\begin{equation}\label{2.12}
\begin{split}
\int_s^R|(r^{N-1}(u_\lambda)_r)_r|\,dr
&=\int_s^Rr^{N-1}[-u_\lambda+u_\lambda^p+\lambda v_\lambda]\,dr\\
&\leq C\int_s^\infty r^{\frac {N-1}2}e^{-\frac
{\sqrt{1-\lambda}}{2}r}\,dr\\&\leq Ce^{-\frac
{\sqrt{1-\lambda}}{4}s},
\end{split}
\end{equation}
this means that $r^{N-1}u_r$ has a limit as $r\to \infty$ and this
limit can only be $0$ by (\ref{2.12}). Integrating (\ref{2.11}) on
$(r,\infty)$ we get
\begin{equation}\label{2.13}
-r^{N-1}(u_\lambda)_r\leq Ce^{-\frac {\sqrt{1-\lambda}}{4}r}.
\end{equation}
Similarly,  $-r^{N-1}(v_\lambda)_r\leq Ce^{-\frac
{\sqrt{1-\lambda}}{4}r}$. Finally the exponential decay of
$(u_\lambda)_{rr}$ and $(v_\lambda)_{rr}$ follows from equation
\eqref{1.5}. This completes the proof.
\end{proof}

Now we consider the variational problem
\begin{equation}\label{2.14}
m_\lambda=\inf_{(u,v)\in \mathcal{N}} I(u,v),
\end{equation}
where
\begin{equation}\label{2.15}
\mathcal{N}=\{(u,v)\in
E\setminus\{(0,0)\}:\langle I'(u,v),(u,v)\rangle=0 \}
\end{equation}
is the Nehari manifold related to $I$. Minimizers of $m_\lambda$ are
ground state solutions of \eqref{1.1}. By a ground state solution of
\eqref{1.1} we mean a nontrivial solution of \eqref{1.1} with the
least energy among all nontrivial solutions of \eqref{1.1}.
Correspondingly, for the limiting problem \eqref{1.5}, the
associated functional
\begin{equation}\label{2.16}
\begin{aligned}
I_\infty(u,v)
&=\frac 12 \int_{\mathbb{R}^N} |\nabla u|^2+u^2\,dx
+\frac 12 \int_{\mathbb{R}^N} |\nabla v|^2+v^2\,dx\\
&\quad -\frac 1{p+1}\int_{\mathbb{R}^N}
|u|^{p+1}+|v|^{p+1}\,dx-\lambda\int_{\mathbb{R}^N} uv\,dx
\end{aligned}
\end{equation}
is well defined  in $H^1(\mathbb{R}^N)\times H^1(\mathbb{R}^N)$. We
define
\begin{equation}\label{2.17}
m_\infty^\lambda=\inf_{(u,v)\in \mathcal{N}_\infty} I_\infty(u,v),
\end{equation}
where
\begin{equation}\label{2.18}
\mathcal{N}_\infty=\{(u,v)\in H^1(\mathbb{R}^N)
 \times H^1(\mathbb{R}^N)\setminus\{(0,0)\}:\langle I_\infty'(u,v),(u,v)
\rangle=0 \}
\end{equation}
is the Nehari manifold for $I_\infty$.

\begin{lemma}\label{lem2.2}
Problem \eqref{1.1} has no ground state solution.
\end{lemma}

\begin{proof}
First we show that $m_\lambda=m_\infty^\lambda$. The fact
$H_0^1(\Omega)\subset H^1(\mathbb{R}^N)$ implies $m_\lambda\geq
m_\infty^\lambda$. Let $\bar\xi$ be a cutoff function such that
$0\leq \bar\xi(t)\leq 1$, $\bar\xi(t)=0$ for $t\leq 1$,
$\bar\xi(t)=1$ for $t\geq 2$ and $|\bar\xi'(t)|\leq 2$. Set
$\xi(x)=\bar\xi(\frac {|x|}{\rho})$, where $\rho$ is the smallest
positive number such that $\mathbb{R}^N\setminus \Omega\subset
B_\rho(0)$. Consider the sequence $\{(\phi_n,\psi_n)\}\subset E$
defined by
\begin{equation}\label{2.19}
(\phi_n,\psi_n)=(\xi(x)u_\lambda(x-y_n),\xi(x)v_\lambda(x-y_n)),
\end{equation}
where $\{y_n\}\subset \Omega$ is a sequence of points such that
$|y_n|\to \infty$. We may verify that there exists a sequence
$\{t_n\}\in \mathbb{R}^+$ such that
$t_n(\xi(x)u_\lambda(x-y_n),\xi(x)v_\lambda(x-y_n))\in \mathcal{N}$.
In fact, we may choose $t_n$ so that
\begin{equation}\label{2.20}
t_n^{p-1}=\frac{\int_\Omega |\nabla \phi_n|^2+\phi_n^2+|\nabla
\psi_n|^2+\psi_n^2-\lambda \phi_n\psi_n\,dx}{\int_\Omega
|\phi_n|^{p+1}+|\psi_n|^{p+1}\,dx}.
\end{equation}
Hence, for $2\leq q<\frac {2N}{N-2}$,
\begin{gather*}
\|\phi_n(x)-u_\lambda(x-y_n)\|_{L^{q}}^q\leq
2\int_{B_\rho}|u_\lambda(x-y_n)|^{q}\,dx \to 0, \\
\|\psi_n(x)-v_\lambda(x-y_n)\|_{L^{q}}^q\leq
2\int_{B_\rho}|v_\lambda(x-y_n)|^{q}\,dx \to 0, \\
\|\nabla \phi_n(x)-\nabla u_\lambda(x-y_n)\|_{L^{2}}^2\leq
C\int_{B_\rho}|\nabla u_\lambda(x-y_n)|^{2}\,dx \to 0, \\
\|\nabla \psi_n(x)-\nabla v_\lambda(x-y_n)\|_{L^{2}}^2\leq
C\int_{B_\rho}|\nabla v_\lambda(x-y_n)|^{2}\,dx \to 0
\end{gather*}
and
$$
\int_{\mathbb{R}^N}\phi(x)\psi(x)-u_\lambda(x-y_n)v_\lambda(x-y_n)\,dx\to 0
$$
as $n\to \infty$. It follows that $t_n\to 1$ as $n\to \infty$ since
$(u_\lambda, v_\lambda)\in \mathcal{N}$. By the definition of
$m_\lambda$, we have
\begin{equation}\label{2.21}
 m_\lambda\leq I(t_n(\phi_n,\psi_n))=m_\infty^\lambda +o(1)
\end{equation}
as $n\to\infty$, which implies $m_\lambda=m_\infty^\lambda$.

Suppose now that $m_\lambda$ is achieved by  $(\bar u, \bar v)$.
Extending $(\bar u, \bar v)$ to $\mathbb{R}^N$ by setting $(\bar u,
\bar v)=(0,0)$ outside $\Omega$, we see that $(\bar u, \bar v)$ is a
minimizer of $m_\infty$. Since we may assume that $\bar u\geq 0,
\bar v\geq 0$, we obtain a contradiction by  the strong maximum
principle. This completes the proof.
\end{proof}

\section{Proof of Theorem \ref{thm1.1}}

Problem \eqref{1.1} is setting in a unbounded, in general, $(PS)$
condition does not hold for $I$. In spirit of  \cite[Lemma 3.1]{BC}
and \cite[Lemma 4.1]{ACR}, we have the following global compact
result.

\begin{lemma}\label{thm3.1}
Let $\{(u_n,v_n)\}\subset E$ be a sequence such that $I(u_n,v_n)\to
c$ and $I'(u_n,v_n)\to 0$ as $n\to \infty$. Then there are a number
$K\in \mathbb N$, $K$ sequences of points $\{y_n^j\}$ such that
$|y_n^j|\to \infty$ as $n\to \infty$, $1\leq j\leq K$, $K+1$
sequences of functions $(u_n^j,v_n^j)\subset H^1(\mathbb{R}^N)\times
H^1(\mathbb{R}^N),0\leq j\leq K$ such that up to a subsequence,
\begin{itemize}
\item[(i)]
 $u_n(x)=u_n^0(x)+\sum_{j=1}^{K}u_n^j(x-y_n^j),v_n(x)=v_n^0(x)+\sum_{j=1}^{K}v_n^j(x-y_n^j)$.

\item[(ii)] $u_n^0(x)\to u^0(x),v_n^0(x)\to v^0(x)$ as $n\to \infty$
strongly in $H_0^1(\Omega)$.

\item[(iii)] $u_n^j(x)\to u^j(x),v_n^j(x)\to v^j(x)$ as $n\to \infty$
strongly in $H^1(\mathbb{R}^N)$, where $1\leq j\leq K$.

\item[(iv)] $(u^0,v^0)$ is a solution of \eqref{1.1} and $(u^j,v^j)$ is a
solution of
\eqref{1.5} for $1\leq j\leq K$.
Moreover, when $n\to \infty$
\begin{gather}\label{3.1}
\|u_n\|^2\to\|u^0\|^2+\sum_{j=1}^K\|u^j\|^2,\|v_n\|^2\to\|v^0\|^2
+\sum_{j=1}^K\|v^j\|^2, \\
\label{3.2}
I(u_n,v_n)\to I(u_0,v_0)+\sum_{j=1}^K I_\infty(u^j,v^j).
\end{gather}
\end{itemize}
\end{lemma}

\begin{proof}
We sketch the proof for reader's convenience. We may verify
that $(u_n,v_n)$ is bounded.
Suppose that $u_n\rightharpoonup u^0,v_n\rightharpoonup v^0 $ in
$H_0^1(\Omega)$ and $u_n\to u^0,v_n\to v^0$ a.e in $\Omega$. Then,
$(u^0,v^0)$ solves \eqref{1.1}. If $(u_n,v_n)\to (u^0,v^0)$, then we
are done. Otherwise, let
$$
z_n^1(x)=  \begin{cases}
u_n-u^0(x),   &x\in\Omega,    \\
 0,   &x\in  \mathbb{R}^N\setminus \Omega,
\end{cases} \quad
 w_n^1(x)=  \begin{cases}
v_n-v^0(x),   &x\in\Omega,    \\
 0,  &x\in  \mathbb{R}^N\setminus \Omega,
\end{cases}
$$
then
$$
\|u_n\|^2=\|u^0\|^2+\|z_n^1\|^2+o(1), \quad
\|v_n\|^2=\|v^0\|^2+\|w_n^1\|^2+o(1).
$$
By Brezis-Lieb's Lemma \cite{W}, we deduce
$$
\|u_n\|^{p+1}_{L^{p+1}}=\|u^0\|^{p+1}_{L^{p+1}}+\|z_n^1\|^{p+1}_{L^{p+1}}+o(1),
\quad
\|v_n\|^{p+1}_{L^{p+1}}=\|v^0\|^{p+1}_{L^{p+1}}+\|w_n^1\|^{p+1}_{L^{p+1}}+o(1).
$$
Thus,
\begin{gather*}
I(z_n^1,w_n^1)=I(u_n,v_n)-I(u^0,v^0)+o(1), \\
I'(z_n^1,w_n^1)=I'(u_n,v_n)-I'(u^0,v^0)+o(1)=o(1).
\end{gather*}
Suppose now that $(z_n^1,w_n^1)\not\to (0,0)$ in
$H^1(\mathbb{R}^N)\times H^1(\mathbb{R}^N)$, we define
$$
\delta_z=\limsup_{n\to \infty}\sup_{y\in \mathbb{R}^N}
 \int_{B_1(y)}|z_n^1|^{p+1}\,dx,\quad
\delta_w=\limsup_{n\to \infty}\sup_{y\in \mathbb{R}^N}
 \int_{B_1(y)}|w_n^1|^{p+1}\,dx.
$$
We may verify that $\delta_z+\delta_w>0$ since
$(z_n^1,w_n^1)\not\to (0,0)$. We may suppose  $\delta_z>0$,
then there is a sequence
$\{y_n^1\}\subset \mathbb{R}^N$ such that
$\int_{B_1(y_n^1)}|z_n^1|^{p+1}\geq \frac {\delta_z}2$. Let us
consider now the sequence $(z_n^1(x+y_n^1),w_n^1(x+y_n^1))$. We
assume that $(z_n^1(x+y_n^1),w_n^1(x+y_n^1))\rightharpoonup
(u^1,v^1)$, then $(u^1,v^1)$ is a nontrivial solution of
\eqref{1.5}. By the fact that $z_n^1\rightharpoonup 0$ we see that
$|y_n^1|\to \infty$. Set
$$
z_n^2(x)=z_n^1(x)-u^1(x-y_n^1), w_n^2(x)=w_n^1(x)-v^1(x-y_n^1),
$$
and repeat above procedure, it will stop at finite steps. The lemma
follows.
\end{proof}

By  \cite[Lemmas 7.8 and 7.9]{ACR}, there exist
$0<\lambda_1\leq\lambda_2<1$ such that $m_\infty$ is an isolated
critical value\ of $I_\infty$ for $\lambda\in
(0,\lambda_1)\cup(\lambda_2,1)$. Denote
$m_0=\inf\{\alpha>m_\infty^\lambda: \alpha\ {\rm\ is\ a\ critical \
value\ of}\ I_\infty\}$ and $\bar m=\min\{m_0,2m_\lambda\}$, then we
have the following result.

\begin{corollary}\label{coro3.2}
The functional $I$ satisfies the $(PS)_c$ condition for $c\in
(m_\lambda,\bar m)$.
\end{corollary}

\begin{proof}
Let $\{(u_n,v_n)\}\subset E$ be such that $I(u_n,v_n)\to c$ and
$I'(u_n,v_n)\to 0$ with $c\in (m_\lambda,\bar m)$. Since
$\{(u_n,v_n)\}$ is bounded, we may assume that $u_n\rightharpoonup
u$ and $v_n\rightharpoonup v$. By Lemma \ref{thm3.1},
$$
(u_n,v_n)-\sum_{j=1}^K (u^j(x-y_n^j),v^j(x-y_n^j))\to (u,v),
$$
 where $(u,v)$ is a solution of \eqref{1.1} and
$(u^j,v^j)$ is a solution of \eqref{1.5}, $\{y_n^j\}(1\leq j\leq K)$
are $K$ sequences of points in $\mathbb{R}^N$. Moreover,
$$
I(u_n,v_n)=I(u,v)+\sum_{j=1}^KI_\infty(u^j,v^j)+o(1).
$$
To prove that $u_n\to u,v_n\to v$ in $H_0^1(\Omega)$, we need
only to show  $K=0$. Since $c<2m_\lambda$, we have $K<2$. We claim
that $K=0$. Indeed, if $K=1$, we have either $(u,v)\neq (0,0)$ or
$(u,v)= (0,0)$. If  $(u,v)\neq (0,0)$, then $I(u_n,v_n)\geq
2m_\lambda+o(1)$, which contradicts to the fact that $c<2m_\lambda$;
if $(u,v)= (0,0)$, then $I_\infty(u^1,v^1)=c$, which contradicts the
definition of $\bar m$. The assertion follows.
\end{proof}



Now we introduce a function
$\Phi_\rho:\mathbb{R}^N\to H^1(\mathbb{R}^N)\times H^1(\mathbb{R}^N)$
defined by
\begin{equation}\label{4.1}
\Phi_\rho(y)=t_\rho(\xi(\frac {|x|}{\rho})u_\lambda(x-y),
\xi(\frac{|x|}{\rho})v_\lambda(x-y)),
\end{equation}
where $(u_\lambda,v_\lambda)$ is a ground state solution of
equation \eqref{1.5}, $t_\rho$ is chosen such that
$t_\rho(\xi(\frac {|x|}{\rho})u_\lambda(x-y),
\xi(\frac {|x|}{\rho})v_\lambda(x-y))\in \mathcal{N}$.

\begin{lemma}\label{lem4.1}
\begin{itemize}
\item[(i)] $\Phi_\rho(y)$ is continuous in $y$ for every $\rho>0$.

\item[(ii)] $\Phi_\rho(y)\to (u_\lambda(x-y),v_\lambda(x-y))$ strongly in $H^1(\mathbb{R}^N)\times H^1(\mathbb{R}^N)$ uniformly in $y$ as $\rho\to 0$.

\item[(iii)] $I(\Phi_\rho(y))\to m_\lambda$ as $|y|\to \infty$ uniformly for every
$\rho$.
\end{itemize}
\end{lemma}

\begin{proof}
(i) is obvious since $\Phi_\rho(\cdot)$ is the composition of
continuous functions. (iii) follows from the same argument of Lemma
\ref{lem2.2}. It remains to prove (ii).  We claim that
\begin{gather*}
\|\xi(\frac {|x|}{\rho})u_\lambda(x-y)\|_{L^{p+1}}\to
 \|u_\lambda(x)\|_{L^{p+1}},\quad
\|\xi(\frac {|x|}{\rho})v_\lambda(x-y)\|_{L^{p+1}}\to
 \|v_\lambda(x)\|_{L^{p+1}},\\
\|\xi(\frac {|x|}{\rho})u_\lambda(x-y)\|\to \|u_\lambda(x)\|,\quad
 \|\xi(\frac{|x|}{\rho})v_\lambda(x-y)\|\to \|u_\lambda(x)\|,\\
\int_{\mathbb{R}^N}\xi(\frac {|x|}{\rho})u_\lambda(x-y)\xi(\frac
{|x|}{\rho})v_\lambda(x-y)\,dx \to
 \int_{\mathbb{R}^N}u_\lambda(x-y)v_\lambda(x-y)\,dx.
\end{gather*}
Indeed,
\begin{equation}\label{4.2}
\begin{split}
\|\xi(\frac {|x|}{\rho})u_\lambda(x-y)-u_\lambda(x-y)\|_{L^{p+1}}^{p+1}
&\leq 2^{p+1}\int_{B_{2\rho}}|u_\lambda(x-y)|^{p+1}\,dx\\
&\leq 2^{p+1}|\max u_\lambda|^{p+1}\mathop{\rm meas} (B_{2\rho})\to 0.
\end{split}
\end{equation}
Similarly, we have
\begin{equation}\label{4.3}
\|\xi(\frac {|x|}{\rho})v_\lambda(x-y)-v_\lambda(x-y)\|_{L^{p+1}}\to
0
\end{equation}
and
\begin{equation}\label{4.4}
\begin{split}
&\|\xi(\frac{|x|}{\rho})u_\lambda(x-y)-u_\lambda(x-y)\|^2\\
&=\int_{\mathbb{R}^N}|\frac 1{\rho}\nabla \xi(\frac
{|x|}{\rho})u_\lambda(x-y)-\xi(\frac {|x|}{\rho})\nabla
u_\lambda(x-y)-\nabla u_\lambda(x-y) |^2\,dx+k_2
\mathop{\rm meas}(B_{2\rho})\\
&\leq 2\int_{\rho\leq |x|\leq 2\rho}|\nabla
\xi(\frac {|x|}{\rho})u_\lambda(x-y)|^2\,dx\\
&\quad +2\int_{\rho\leq |x|\leq
2\rho}|\xi(\frac {|x|}{\rho})\nabla u_\lambda(x-y)-\nabla
u_\lambda(x-y)|^2\,dx
+k_2 \mathop{\rm meas}(B_{2\rho})\\&\leq k_3 \rho^{N-2}+k_4 \rho^N\to 0
\end{split}
\end{equation}
as well as
\begin{equation}\label{4.5}
\begin{split}
&|\int_{\mathbb{R}^N}\xi(\frac {|x|}{\rho})u_\lambda(x-y)\xi(\frac
{|x|}{\rho})v_\lambda(x-y)-u_\lambda(x-y)v_\lambda(x-y)\,dx|\\
&\leq \int_{\mathbb{R}^N}|\xi(\frac {|x|}{\rho})u_\lambda(x-y)\xi(\frac
{|x|}{\rho})v_\lambda(x-y)-u_\lambda(x-y)v_\lambda(x-y)|\,dx\\
&\leq k_5 \rho^{N}\to 0.
\end{split}
\end{equation}
This proves the claim. The definition of $t_\rho$ and the claim
yield that $t_\rho\to 1$ as $\rho\to 0$. This together with equation
(\ref{4.4}) imply (ii).
\end{proof}

Since $I_\infty^\lambda(u_\lambda(x-y),v_\lambda(x-y))=m_\lambda$,
the following result is a consequence of
 (ii) in Lemma \ref{lem4.1}.

\begin{corollary}\label{coro4.2}
For $0<\lambda<\lambda_1$ or $\lambda_2<\lambda<1$, there exists a
$\bar \rho=\bar\rho(\lambda)$ such that for  $\rho\leq \bar \rho$,
there holds
\begin{equation}\label{4.6}
\sup_{y\in \mathbb{R}^N}I(\Phi_\rho(y))<\bar m.
\end{equation}
\end{corollary}

From now on we will suppose that $\Omega$ is fixed in such a way
that $\rho<\bar \rho$. Now we define a function
$\beta: H^1(\mathbb{R}^N)\to R^N$ as follows
$$
\beta (u)=\int_{\mathbb{R}^N}u(x)\chi(|x|)x\,dx,
$$
where
$$
\chi(t)=  \begin{cases}
1    & \text{if }  0\leq t\leq R,    \\
  R/tt   & \text{if }  t>R
\end{cases}
$$
and $R$ is chosen such that
$\mathbb{R}^N\setminus \Omega \subset B_R$.

Let $\mathcal{B}_0:=\{(u,v)\in \mathcal{N}: \beta (u)=0 \text{ or }
\beta (v)=0\}$
 and let $c_0=\inf_{(u,v)\in \mathcal B_0}I(u,v)$.

\begin{lemma}\label{lem4.3}
There holds $c_0>m_\lambda$, and there is an $R_0>\rho$ such that
\begin{itemize}

\item[(a)] if $|y|\geq R_0$, then
$I(\Phi_\rho(y))\in (m_\lambda,\frac {m_\lambda+c_0} 2)$;

\item[(b)] if $|y|=R_0$, then $\langle \beta \circ
P_1\circ\Phi_\rho(y),y\rangle>0$ or $\langle \beta \circ
P_2\circ\Phi_\rho(y),y\rangle>0$, where $P_i(u,v)$ is the projection
of $(u,v)$ on the $i\rm th$ coordinate.
\end{itemize}
\end{lemma}

\begin{proof}
It is obvious that $c_0\geq m_\lambda$. Now suppose that
$c_0=m_\lambda$, then there exists a sequence $(u_n,v_n)\in
\mathcal{N}$ with $\beta(u_n)=0$ or $\beta(v_n)=0$ such that
$I(u_n,v_n)\to m_\lambda$. We may assume that $\beta(u_n)=0$. By
Lemma \ref{thm3.1},  $ u_n(x)=u_0(x-y_n)+o(1), v_n=v_0(x-y_n)+o(1) $
with $|y_n|\to \infty$. Denote
$(\mathbb{R}^N)_n^+=\{x\in \mathbb{R}^N: \langle x,y_n\rangle >0\},
(\mathbb{R}^N)_n^-=\mathbb{R}^N\setminus (\mathbb{R}^N)_n^+$,
then for $n$ large we have $B_{\hat
r}(y_n):=\{x:|x-y_n|<\hat r\}\subset (\mathbb{R}^N)_n^+$ for some
fixed $\hat r>0$ and
$u_0(x-y_n)\geq\delta_0>0,v_0(x-y_n)\geq\delta_0>0$ for $x\in
B_{\hat r}(y_n)$ and some $\delta_0>0$. Lemma \ref{thm2.1} implies
$$
u_0(x-y_n)\leq \frac{K}{e^{\delta |x-y_n|}|x-y_n|^{\frac {N-1}2}},\quad
v_0(x-y_n)\leq \frac{K}{e^{\delta |x-y_n|}|x-y_n|^{\frac {N-1}2}}
$$
 for $x\in B_{\hat r}(y_n)$. So we have
\begin{equation}\label{4.7}
\begin{split}
&\langle \beta(u_0(x-y_n)),y_n\rangle\\
&=\int_{(\mathbb{R}^N)_n^+}u_0(x-y_n)\chi(|x|)\langle x,y_n\rangle\,dx
+\int_{(\mathbb{R}^N)_n^-}u_0(x-y_n)\chi(|x|)\langle x,y_n\rangle\,dx\\
&\geq \int_{B_{\hat  r}(y_n)}\delta_0\chi(|x|)\langle x,y_n\rangle\,dx
-\int_{(\mathbb{R}^N)_n^-}\frac{KR|y_n|}{e^{\delta |x-y_n|}|x-y_n|^{\frac
 {N-1}2}}\,dx\\
&\geq \alpha-o(\frac 1 {|y_n|})>0,
\end{split}
\end{equation}
where $\alpha>0$ is a constant. Since $\beta$ is continuous, we have
$\beta(u_n)\neq 0$. This contradicts to the fact that
$\beta(u_n)=0$.

(a) can be proved in the same way as the proof of Lemma \ref{lem2.2}
and (b) can be proved as (\ref{4.7}).
\end{proof}

Now let us consider the set $\Sigma$ given by
$$
\Sigma:=\{t_\rho \Phi_\rho (y): |y|\leq R_0\},
$$
where $t_\rho$ is chosen such that $t_\rho \Phi_\rho (y)\in
\mathcal{N}$. We define $$ H=\{h\in C(\mathcal{N},\mathcal{N}):
h(u,v)=(u,v)\ {\rm for}\ \forall (u,v)\in \mathcal{N}\ {\rm with}\
I(u,v) \leq \frac{c_0+m}{2}\}
$$
and
$\Gamma=\{A\subset \mathcal{N}, A=h(\Sigma)\}$.

\begin{lemma}\label{lem4.4}
If $A\in \Gamma$, then $A\cap \mathcal B_0\neq \emptyset$.
\end{lemma}

\begin{proof}
The proof of the lemma is equivalent to prove that for $\forall h\in
H$, there is $\bar y\in \mathbb{R}^N$ with $|\bar y|\leq R_0$ such
that $\beta\circ h\circ P_1\circ \Phi_\rho (y)=0$ or $\beta\circ
h\circ P_2\circ \Phi_\rho (y)=0$. By Lemma \ref{lem4.3}, we have
$\langle \beta\circ P_1\circ \Phi_\rho (y),y\rangle>0$ or $\langle
\beta\circ P_2\circ \Phi_\rho (y),y\rangle>0$ for $|y|=R_0$. Assume
that $\langle \beta\circ P_1\circ \Phi_\rho (y),y\rangle>0$ without
of loss generality and define
\begin{gather*}
f(y)=\beta\circ h\circ P_1\circ \Phi_\rho (y),\\
F(t,y)=tf(y)+(1-t)id.
\end{gather*}
(b) of Lemma \ref{lem4.3} implies $0\not\in F(t,\partial B_{R_0}$),
 hence, $deg(F,B_{R_0},0)=\deg (id,B_{R_0},0)=1$. This yields that
there exists $\bar y\in B_{R_0}$ such that $\beta\circ h\circ
P_1\circ \Phi_\rho (y)=0$.

If $\langle \beta\circ P_2\circ \Phi_\rho (y),y\rangle>0$, we may
show that there exists a $\bar y\in B_{R_0}$ such that $\beta\circ
h\circ P_2\circ \Phi_\rho (y)=0$ in the same way. This proves the
Lemma.
\end{proof}

\begin{proof}[Proof of Theorem \ref{thm1.1}]
For $\lambda\in (0,\delta)$, obviously, problem \eqref{1.1}
has two pair of positive
solutions $(U_{1-\lambda}, U_{1-\lambda})$ and $(\pm U_{1+\lambda},
\mp U_{1+\lambda})$, where $U_{1-\lambda}$ and $U_{1+\lambda}$ are
positive solutions of
\begin{equation}\label{3.10}
  \begin{gathered}
-\Delta u+(1-\lambda)u=|u|^{p-1}u     \quad \text{in }  \Omega,    \\
u=0  \quad \text{on }  \partial\Omega,
\end{gathered}
\end{equation}
and
\begin{equation}\label{3.11}
  \begin{gathered}
-\Delta u+(1+\lambda)u=|u|^{p-1}u     \quad \text{in }  \Omega,    \\
u=0  \quad \text{on }  \partial\Omega,
\end{gathered}
\end{equation}
respectively. It is proved in \cite{BC} that problem (\ref{3.10})
and problem (\ref{3.11}) have nontrivial solutions. Define
\begin{equation}
c_\lambda=\inf_{A\in \Gamma}\sup_{(u,v)\in A}I(u,v),
\end{equation}
then we have $\bar m>c_\lambda\geq c_0>m_\lambda$ since $id\in H$
and $A\cap \mathcal B_0\neq \emptyset$. A standard deformation
argument implies that $c_\lambda$ is a critical value of $I$. Now,
we claim that $c_\lambda<I(U_{1-\lambda},U_{1-\lambda})<I(\pm
U_{1+\lambda},\mp U_{1+\lambda})$ for $\bar\rho$ small sufficiently.
Then the critical points corresponding to $c_\lambda$ are different
from trivial solutions $(U_{1-\lambda},U_{1-\lambda})$ and
$(\pm U_{1+\lambda},\mp U_{1+\lambda})$. In fact, we note that
$U_0(x)=(1-\lambda)^{-\frac1{p-1}}U_{1-\lambda}(\frac x{\sqrt{1-\lambda}})$
is a solution of
\begin{equation}\label{3.12}
 \begin{gathered}
-\Delta u+u=|u|^{p-1}u   \quad \text{in }  \Omega_{\sqrt{1-\lambda}},    \\
u\in H^1_0(\Omega_{\sqrt{1-\lambda}})
\end{gathered}
\end{equation}
and extend $U_{1-\lambda}$ to $\mathbb{R}^N$ by setting
$U_{1-\lambda} = 0$ outside $\Omega$. Denote by $J_\lambda(u)$ the
functional corresponding to problem (\ref{3.10}) and let
$(u_\lambda, v_\lambda)$ be a ground state solution of \eqref{1.5},
since $(U_0,0)\in \mathcal{N}$ is not a ground state solution of
\eqref{1.5}, for $\lambda$ small, we have
\begin{align*}
I_\infty(u_\lambda,v_\lambda)
&\leq I_\infty(U_0,0)
=J_0(U_0)\\
&< 2(1-\lambda)^{\frac{p+1}{p-1}-\frac N2}J_0(U_0)\\
&=2J_\lambda(U_{1-\lambda})\\
&=I(U_{1-\lambda},U_{1-\lambda}).
\end{align*}
By (ii) of Lemma \ref{lem4.1}, $c_\lambda\to I_\infty(u_\lambda,v_\lambda)$
as $\rho\to 0$, for fixed $\lambda_0>0$ small, there exists
$\bar\rho=\bar\rho(\lambda_0)$ such that
$c_{\lambda_0}<I(U_{1-\lambda_0},U_{1-\lambda_0})$. Noticing that
$c_\lambda$ and $I(U_{1-\lambda},U_{1-\lambda})$ are continuous in
$\lambda$,  applying compact argument to $[0, \lambda_0]$, we may
find $\bar \rho_1\leq\bar\rho$ such that for $\lambda\in [0,
\lambda_0]$ we have $c_{\lambda}<I( U_{1-\lambda},U_{1-\lambda})$ if
$0<\rho\leq\bar\rho_1$. On the other hand, by \cite{ACR} we have $I(
U_{1-\lambda},U_{1-\lambda})<I(\pm U_{1+\lambda},\mp
U_{1+\lambda})$, the proof is completed.
\end{proof}

\begin{remark} \rm
For $\lambda$ close to $1$, we may also obtain a critical value
$c_\lambda$ of $I$ as the proof of Theorem \ref{thm1.1}. However,
$c_\lambda$ and $I( U_{1-\lambda},U_{1-\lambda})$ are close to each
other if $\rho\to 0$. Hence, we may not obtain nontrivial solutions
in this way.
\end{remark}

 \subsection*{Acknowledgements}
This work is supported by National Natural Sciences Foundations of
China, No: 10571175 and 10631030.


\begin{thebibliography}{00}
\bibitem{ACR} A. Ambrosetti, G. Cerami, D. Ruiz;
 \emph{Solitons of linearly coupled
system of semilinear non-autonomous equations on $\mathbb{R}^N$}, J.
Funct. Anal., to appear.

\bibitem{BC} V. Benci, G. Cerami;
 \emph{Positive solutions of some nonlinear
elliptic problems in exterior domains}, Arch. Rat. Math. Anal.
\textbf{99}(1987),283-300.

\bibitem{BC1} V. Benci, G. Cerami;
 \emph{The effect of the domain topology on the
number of positive solutions of nonlinear elliptic problems}, Arch.
Rat. Math. Anal. \textbf{114}(1991),79-93.

\bibitem{BL1} H. Berestycki, P. L. Lions;
 \emph{Nonlinear scalar field equations, I and II},
Arch. Rat. Math. Anal. \textbf{82}(1983),313-345 and
347-376.

\bibitem{CP} G. Cerami, D. Passaseo;
 \emph{Existence and multiplicity of positive
solutions for nonlinear elliptic problems in exterior domains with
"rich" topology}, Nonlinear Anal. TMA \textbf{18}(1992),109-119.

\bibitem{EL} M. J. Esteban, P. L. Lions;
 \emph{Existence and Nonexistence Results for
Semilinear Elliptic Problems in Unbounded Domains}, Proc. Royal.
Edinbourgh Soc. \textbf{93} (1982),1-14.

\bibitem{L1} P. L. Lions;
 \emph{The concentration-compactness principle in the
calculus of variations, The local compact case, part I and II},
 Ann Inst. Henri. Poincare. \textbf{1}(1984),109-145 and 223-283.

\bibitem{R} P. H. Rabinowitz;
 \emph{Minimax Theorems and Applications to Partial
Differential Equations}, AMS Memoirs \textbf{65}(1986).

\bibitem{W} M. Willem;
\emph{Minimax Theorems}, Progr. Nonlinear Differential
Equations Appl.,vol 24, Birkh\"auser, Basel(1996).

\bibitem{YZ} J. Yang, X. Zhu;
 \emph{On the existence of nontrivial solutions of
 a quasilinear elliptic boundary value problem for unbounded
domains(I)and (II)}, Acta Math.Sci. \textbf{7}(1987), 341-359 and 47-459.

\end{thebibliography}

\end{document}
