% Please expand the abstract. % Family name YANG, given name Jianfu? \documentclass[twoside]{article} \usepackage{amsfonts} % used for R in Real numbers \pagestyle{myheadings} \markboth{ Nontrivial solutions of semilinear elliptic systems } {Jianfu Yang} \begin{document} \setcounter{page}{343} \title{\vspace{-1in}\parbox{\linewidth}{\footnotesize\noindent USA-Chile Workshop on Nonlinear Analysis, \newline Electron. J. Diff. Eqns., Conf. 06, 2001, pp. 343--357. \newline http://ejde.math.swt.edu or http://ejde.math.unt.edu \newline ftp ejde.math.swt.edu or ejde.math.unt.edu (login: ftp)} \vspace{\bigskipamount} \\ % Nontrivial solutions of semilinear elliptic \\ systems in $\mathbb{R}^N$ % \thanks{ {\em Mathematics Subject Classifications:} 35J50, 35J55. \hfil\break\indent {\em Key words:} indefinite, semilinear, elliptic system. \hfil\break\indent \copyright 2001 Southwest Texas State University. \hfil\break\indent Published January 8, 2001. } } \date{} \author{ Jianfu Yang } \maketitle \begin{abstract} We establish an existence result for strongly indefinite semilinear elliptic systems in $\mathbb{R}^N$. \end{abstract} \newtheorem{theorem}{Theorem}[section] \newtheorem{lemma}[theorem]{Lemma} \newtheorem{prop}{Proposition}[section] \renewcommand{\theequation}{\thesection.\arabic{equation}} \catcode`@=11 \@addtoreset{equation}{section} \catcode`@=12 \section{Introduction} The main objective of this paper is to establish existence results for the semilinear elliptic system \begin{eqnarray} & -\Delta u + u = g(x,v) ,\quad -\Delta v + v = f(x,u) \quad {\rm in } \mathbb{R}^N ,&\\ &u(x) \to 0 \quad {\rm and} \quad v(x) \to 0 \quad {\rm as} \quad |x| \to \infty .& \end{eqnarray} The existence of solutions of (1.1)-(1.2) is usually investigated by finding critical points of a related functional. Typical features of the problem are that firstly, the related functional is strongly indefinite; secondly, the growths of $f$ in $u$ and $g$ in $v$ at infinity may not be `symmetric'; and lastly, Sobolev embeddings in general are not compact, then the Palais - Smale condition may not be satisfied. Existence results were recently obtained in \cite{FF} and \cite{HV} in bounded domains. The arguments lie in the decomposition of Sobolev spaces by eigenfunctions of Laplacian operator and a use of linking theorems. Using spectral family theory of non-compact operator, the author and Figueiredo \cite{FY} find a suitable linking structure for the functional associate to (1.1)-(1.2) and prove that problem (1.1)-(1.2) possesses at least a positive solution if $f$ and $g$ depend on the variable $x$ radially. Furthermore, it is also shown in \cite{FY} that all positive solutions of problem (1.1)-(1.2) are exponentially decaying. In this paper, we establish existence results for general cases. Assume that \begin{enumerate} \item[H1)] $f,g: \mathbb{R}^N\times\mathbb{R} \to \mathbb{R}$ are measurable in the first variable, continuous in the second variable. Both $F(x,t) = \int_0^t f(x,s)ds$ and $G(x,t) =\int_0^t g(x,s)ds$ are increasing and strictly convex in $t$. \item[H2)] $ \lim_{t \to 0} f(x,t)/t = 0, \quad \lim_{t \to 0} g(x,t)/t = 0 \quad{\rm uniformly\,\, in }\quad x\in \mathbb{R}^N$. \item[H3)] There is a constant $c > 0$ such that $|f(x,t)|\leq c(|t|^p + 1)$ and $|g(x,t)| \leq c(|t|^q + 1)$, where $0
2$ such that
$0 <\alpha F(x,t) \leq tf(x,t)$ and $0 < \beta G(x,t)\leq tg(x,t)$,
for $t\neq 0$.
\item[H5)] $f(x,t) \to \bar f(t)$ and $g(x,t) \to \bar g(t)$
uniformly for $t$ bounded as $|x| \to \infty$.\\
$|f(x,t) - \bar f(t)| \leq \epsilon(R) |t|$ and
$|g(x,t) - \bar g(t)| \leq \epsilon(R) |t|$ whenever $|x| \geq R$,
$|t| \leq \delta$,
where $\epsilon(R) \to \infty$ as $R \to \infty$.
\item[H6)]
$F(x,t) \geq \bar F(t)$ and $G(x,t) \geq \bar G(t)$,\\
$\mathop{\rm meas} \{x\in R^N: f(x,t) \not\equiv \bar f(t)\}>0$
or $\mathop{\rm meas} \{x\in R^N: g(x,t) \not\equiv \bar g(t)\} > 0$.
\item[H7)] Both $\bar f(t)/t$ and $\bar g(t)/t$ are
increasing in $t$.
\end{enumerate}
Our main result is as follows.
\begin{theorem} Assume (H1)-(H7). Then problem (1.1)-(1.2) possesses
at least one nontrivial exponentially decaying solution.
\end{theorem}
The restriction of exponents in (H3) is due to the fact that we only
know the decaying law in the case.
We analyze the convergence of Palais-Smale sequence of associate
functional to (1.1)-(1.2) in Section 3. It is shown that the
energy levels of solutions of the related autonomous system
\begin{eqnarray}
&-\Delta u + u = \bar g(v), \quad -\Delta v + v = \bar f(u) \quad
{\rm in} \quad \mathbb{R}^N ,&\\
&u(x) \to 0, \quad v(x) \to 0 \quad {\rm as} \quad |x| \to 0\,.&
\end{eqnarray}
are obstacle levels preventing strong convergence of Palais-Smale
sequences of (1.1)-(1.2). The possible critical values to be found
are between obstacle levels. To retain the compactness, we have to
get control of critical values. It is harder to handle critical
values described by linking structure than that by the Mountain
Pass Theorem. We use dual variational method as \cite{AS1},
\cite{AS2} and \cite{CV}. The method is of the advantage avoiding
the indefinite character of original functional. We start with
problem (1.1)-(1.2) in bounded domains. Although existence
result in the case is known, it has no control of critical values.
We establish in Section 2 an existence result by the Mountain Pass
Theorem and bound uniformly corresponding critical values by the
energy level of ground state of problem (1.3)-(1.4). Then we
construct a Palais - Smale sequence for the functional associated
to problem (1.1)-(1.2). Theorem 1.1 is proved in Section 4.
\section{Existence results in bounded domains}
Let $\Omega$ be a bounded domain. We consider the problem
\begin{eqnarray}
&- \Delta u + u = g(x,v), \quad - \Delta v + v = f(x,u) \quad {\rm
in} \quad \Omega,&\\
&u = 0, \quad v = 0 \quad {\rm on} \quad \partial \Omega\,.&
\end{eqnarray}
The solutions of (2.1)-(2.2) will be found by looking for critical
points of associate functional. The main result in this section is
as follows.
\begin{theorem} Assume $(H1)-(H4)$. Then problem (2.1)-(2.2) possesses
at least a nontrivial solution.
\end{theorem}
To prove Theorem 2.1 we will need the lemmas below. First we define
the dual functional associate to (2.1)-(2.2). It is well known
that the inclusions
\[
i_r: W_o^{1,r}(\Omega) \to L^{p+1}(\Omega), \quad i_s: W_o^{1,s}(\Omega) \to
L^{q+1}(\Omega)
\]
are compact if $2< p+1 < \frac {rN}{N-r}, N > r$ and $2 s$. The operator $-\Delta + id: W_o^{1,r}(\Omega) \to W_o^{-1,r'}(\Omega)$
is an isomorphism, where $r'= \frac r{r-1}$. Hence
\[
{\cal T} = i_2 (-\Delta + id)^{-1} i_2^*: L^{1+1/q}(\Omega) \to
L^{p+1} (\Omega).
\]
is continuous. Denote by $X = L^{p+1}(\Omega)\times
L^{q+1}(\Omega), X^* = L^{1+1/p}(\Omega) \times L^{1 +
1/q}(\Omega)$ and let
\[
A = \pmatrix{ 0 & {\cal T} \cr {\cal T} &0 }, \quad K = A^{-1} =
\pmatrix{ 0 & {\cal T}^{-1}\cr {\cal T}^{-1} & 0}.
\]
For each $x$, the Legendre-Fenchel transformations $F^*(x,\cdot)$
of $F(x, \cdot)$, and $G^*(x,\cdot)$ of $G(x,\cdot)$ are defined by
\begin{equation}
F^*(x,s) = \sup_{t\in R} \{st - F(x,t) \}, \quad G^*(x,s) =
\sup_{t\in R} \{st - G(x,t) \}
\end{equation}
respectively. Equivalently, we have
\begin{equation}
F^*(x,s) = st -F(x,t) \quad {\rm with} \quad s = f(x,t), \quad t =
F^{*'}_ s(x,s)
\end{equation}
and
\begin{equation}
G^*(x,s) = st - G(x,t) \quad {\rm with} \quad s = g(x,t), \quad t
= G^{*'}_ s(x,s).
\end{equation}
In the same way, we define $\bar F^*, \bar G^*$ for $\bar F, \bar
G$. By (H6) and properties of Legendre-Fenchel transformations, we
have
\begin{equation}
F^*(x,s) \leq \bar F^*(s), \quad G^*(x,s) \leq \bar G^*(s).
\end{equation}
We may verify following properties of $F^*, G^*$ in Lemmas 2.2 and
2.3 as \cite{AS1}, \cite{CH} and \cite{MW}.
\begin{lemma} $F^*, G^*
\in C^1$ and
\begin{eqnarray}
F^*(x,s) \geq (1 - \frac 1{\alpha})sF^{*'}(x,s), \quad G^*(x,s)
\geq (1 - \frac 1{\beta})sG^{*'}(x,s), \\
F^*(s,x) \geq C|s|^{1+1/p} - C, \quad G^*(x,s) \geq C|s|^{1+1/q}
-C.
\end{eqnarray}
\end{lemma}
\begin{lemma} There exist $\delta > 0, C_{\delta}$ and
$C'_{\delta}>0$ such that $$ F^*(x,s) \geq \cases {C_{\delta}
|s|^2, &${\rm if} \quad |s| \leq \delta $\cr C'_{\delta} |s|^{1 +
\frac 1p}, &${\rm if} \quad |s| \geq \delta$\cr} , \quad G^*(x,s)
\geq \cases {C_{\delta} |s|^2, &${\rm if} \quad |s| \leq \delta
$\cr C'_{\delta} |s|^{1 + \frac 1q}, &${\rm if} \quad |s| \geq
\delta$\cr} .$$
\end{lemma}
We may verify that the dual functional
\[
\Psi (w) = \Psi_{\Omega}(w) = \int_{\Omega} (F^*(x, w_1) +
G^*(x,w_2))\,dx - \frac 12 \int_{\Omega} \langle w, Kw\rangle
\,dx,
\]
is well defined and $C^1$ on $X^*$. A critical point $w$ of $\Psi$
satisfies
\[
(-\Delta + id)^{-1} w_2 = F^{*'}_s(x,w_1)\quad {\rm and} \quad
(-\Delta + id)^{-1} w_1 = G^{*'}_s(x,w_2).
\]
Let
\[
u = (-\Delta + id)^{-1} w_2, \quad v = (-\Delta + id)^{-1} w_1.
\]
Then $(u,v)$ satisfies (2.1)-(2.2). Furthermore, denoting by
\[
\Phi(z) = \int_{\Omega} (\nabla u \nabla v + u v)\,dx -
\int_{\Omega} F(x,u)\,dx - \int_{\Omega} G(x,v) \,dx
\]
the functional of (2.1) -(2.2) defined on $H_o^1(\Omega)\times
H_o^1(\Omega)$, we deduce by (2.4) and (2.5) that $\Phi(z) =
\Psi(w)$. Such a result is also valid for solutions of
(1.1)-(1.2). Now we use the Mountain Pass Theorem to find critical
points of $\Psi$.
Following arguments of \cite{BF}, we know that (H2)
implies $F^*(x,t)/t^2 \to \infty$ and $G^*(x,t)/t^2 \to \infty$.
Thus $0$ is a local minmum of $\Psi$. Precisely,
\begin{lemma} There exist constants $\alpha, \rho >0$, independent of $\Omega$,
such that
\[
\Psi (w) \geq \alpha >0 \quad {\rm if} \quad \|w\|_{X^*} = \rho.
\]
\end{lemma}
\begin{lemma} There exist $T > 0$ and $w \in E$ such that
$\Psi(tw) \leq 0$ whenever $t \geq T$.
\end{lemma}
\paragraph{Proof.} Taking $w \in X^*, w \not\equiv 0$ such that
\[
\int_{\Omega} \langle w, Kw\rangle \,dx > 0,
\]
whence by (H4), for $t >0$
\[
\Psi(tw) \leq t^{\frac {\alpha}{\alpha -1}}
\int_{\Omega}|w_1|^{\frac {\alpha}{\alpha -1}}\,dx + t^{\frac
{\beta}{\beta -1}} \int_{\Omega} |w_2|^{\frac {\beta}{\beta
-1}}\,dx - \frac 12 t^2 \int_{\Omega}\langle w,Kw\rangle \,dx.
\]
Since $\frac {\alpha}{\alpha -1} , \frac {\beta}{\beta -1} < 2$,
the assertion follows for $t > 0$ large. \hfill$\Box$
Let
\[
\Gamma = \{g\in C([0,1], X^*): g(0) = 0, g(1) = e\},
\]
where $e = Tw$. We define
\begin{equation}
c = c_{\Omega} = {\rm inf}_{g\in\Gamma}\sup_{t\in[0,1]}
\Psi(g(t)).
\end{equation}
The Mountain Pass Theorem will imlpy that $c$ is a critical value
of $\Psi$ if the Palais-Smale ((PS) for short) condition holds. It
is known from Lemma 2.4 that corresponding critical points are
nontrivial. Then the proof of Theorem 2.1 is completed.
Now we verify the (PS) condition. By a (PS) condition
for $\Psi$ we mean that any sequence $\{w_n\} \subset X^*$ such that
$|\Psi(w_n)|$ is uniformly bounded in $n$ and $\Psi'(w_n) \to 0$ as $n \to
\infty$ possesses a convergent subsequence.
\begin{lemma} The (PS) condition holds for $\Psi$.
\end{lemma}
\paragraph{Proof.} Let $\{w_n\}$ be a (PS) sequence of $\Psi$, that
is
\[
|\Psi(w_n)| \leq C \quad \Psi'(w_n) \to 0 \quad {\rm as} \quad
n\to \infty
\]
for some constant $C >0$. This inequality and lemma 2.2 yield
\begin{eqnarray*}
\lefteqn{\int_{\Omega} [F^*(x,w_n^1) + G^*(x,w_n^2)]\,dx }\\
&\leq& \frac12\int_{\Omega} \langle w_n, Kw_n\rangle\,dx +C\\
&\leq& \frac 12\int_{\Omega} (F^{*'}_s (x,w_n^1)w_n^1 +
G^{*'}_s(x,w_n^2)w_n^2)\,dx +o(1)\|w_n\|_{X^*} + C\\
&\leq& \frac12 \frac {\alpha}{\alpha -1}\int_{\Omega} F^*(x,w_n^1)\,dx
+ \frac 12 \frac {\beta}{\beta -1}\int_{\Omega} G^*(x,w_n^2)\,dx +
o(1)\|w_n\|_{X^*}.
\end{eqnarray*}
That is
\[
\int_{\Omega} [F^*(x,w_n^1) + G^*(x,w_n^2)]\,dx \leq C +
o(1)\|w_n\|_{X^*}.
\]
By Lemma 2.3, we obtain
\[
\|w_n^1\|^{1+1/p}_{L^{1+1/p}} + \|w_n^2\|^{1+1/q}_{L^{1+1/q}} \leq
C + o(1) \|w_n\|_{X^*}.
\]
It implies that $\|w_n\|_{X^*}$ is bounded. We may assume $w_n \to
w$ weakly in $X^*$ as $n \to \infty$. Since the operator $(-
\Delta + id)^{-1}$ is compact, it follows
$$\displaylines{
u_n:= (-\Delta + id)^{-1} w_n^2 \to (-\Delta +id)^{-1}w^2 \quad
{\rm in} \quad X^* \quad {\rm as} \quad n \to \infty,
\cr
v_n:= (-\Delta + id)^{-1} w_n^1 \to (-\Delta +id)^{-1}w^1 \quad
{\rm in} \quad X^* \quad {\rm as} \quad n \to \infty.
}$$
As a result,
$w_n = (f(x,u_n), g(x,v_n)) \to w \quad {\rm in} \quad X^* \quad
{\rm as} \quad n\to \infty $
which completes the present proof.
\section{Palais-Smale sequence}
In this section, we prove a global compact result for problem (1.1)-(1.2). Let $E = H^1(\mathbb{R}^N)
\times H^1(\mathbb{R}^N)$. By our assumptions, the functional
\[
{\bf \Phi}(z) = \int_{\mathbb{R}^N}(\nabla u \nabla v + u v)\,dx -
\int_{\mathbb{R}^N}(F(x,u) + G(x,v))\,dx
\]
is $C^1$ on $E$. The functional $\bf \Phi^{\infty}$ is defined
with $\bar F$ and $\bar G$ replacing $F$ and $G$ in $\bf \Phi$
respectively.
\begin{prop} Asumme (H1)-(H6). Let $\{z_n\}$ be a
$(PS) _c$ sequence of $\bf \Phi$, i.e.
\begin{equation}
{\bf \Phi} (z_n) \to c \quad {\rm and} \quad {\bf\Phi}'(z_n) \to 0
\quad {\rm as} \quad n \to 0.
\end{equation}
Then there exists a subsequence (still denoted by $\{z_n\}$) for
which the following holds: there exist an integer $k \geq 0$,
sequences $\{x_n^i\} \subset \mathbb{R}^N, |x^i_n| \to \infty \,\,{\rm
as} \,\,n \to \infty$ for $1 \leq i \leq k$, a solution $z$ of
(1.1)-(1.2) and solutions $z^i(1\leq i \leq k)$ of (1.3)-(1.4)
such that
\begin{eqnarray}
&z_n \to z \quad {\rm weakly}\quad {\rm in} \quad E,&\\
&{\bf \Phi}(z_n) \to {\bf \Phi}(z) + \sum_{i=1}^k {\bf
\Phi}^{\infty}(z^i),&\\
&z_n - (z + \sum_{i=1}^k z^i(x-x^i_n)) \to 0 \quad {\rm in} \quad E&
\end{eqnarray}
as $n \to \infty$, where we agree that in the case $k=0$ the above
holds without $z^i, x^i_n$.
\end{prop}
\paragraph{Proof.} The result can be derived from the arguments for
one equation \cite{BC}. First we remark that the boundedness of
$\{z_n\}$ in $E$ can be deduced as \cite{FY} by (3.1). Therefore
we may assume
\begin{eqnarray*}
&& z_n \to z \quad {\rm weakly}\quad {\rm in} \quad E,\\ && z_n
\to z \quad {\rm strongly}\quad {\rm in}\quad L^{p+1}_{loc}(\Bbb
R^N)\times L^{q+1}_{loc}(\mathbb{R}^N),\\ && z_n \to z \quad a.e.
\quad {\rm in}\quad \mathbb{R}^N \nonumber
\end{eqnarray*}
as $n \to \infty$. Denote $Q(z) = \int_{\mathbb{R}^N} (\nabla u \nabla
v + u v)\,dx$, we have
\begin{equation}
Q(z_n) = Q(z_n - z) + Q(z) + o(1).
\end{equation}
It follows from Brezis \& Lieb's lemma \cite{BrL} that
\begin{equation}
\int_{\mathbb{R}^N} F(x,u_n)\,dx = \int_{\mathbb{R}^N} F(x, u_n - u)\,dx +
\int_{\mathbb{R}^N} F(x,u)\,dx + o(1)
\end{equation}
and
\begin{equation}
\int_{\mathbb{R}^N} G(x, v_n)\,dx = \int_{\mathbb{R}^N} G(x, v_n - v)\,dx
+ \int_{\mathbb{R}^N} G(x, v)\,dx + o(1).
\end{equation}
Hence we obtain
\begin{equation}
{\bf \Phi}(z_n) = {\bf \Phi} (z_n - z) + {\bf \Phi} (z) + o(1),
\quad {\bf \Phi}'(z_n) = {\bf \Phi}'(z_n -z) + {\bf \Phi}'(z) +
o(1)
\end{equation}
as $n \to \infty$. Let $z_n^1 = z_n -z$. We may deduce from (H5)
as \cite{PL} and \cite{YZ} that
\[
\int_{\mathbb{R}^N} u_n^1 [f(x,u_n^1) - \bar f(u_n^1)]\,dx \to 0 \quad
{\rm and} \quad \int_{\mathbb{R}^N} v_n^1 [g(x,v_n^1) - \bar
g(v_n^1)]\,dx \to 0
\]
as well as
\[
\int_{\mathbb{R}^N} [F(x,u_n^1) - \bar F(u_n^1)]\,dx \to 0 \quad {\rm
and} \quad \int_{\mathbb{R}^N} [G(x,v_n^1) - \bar G(v_n^1)]\,dx \to 0
\]
as $n \to \infty$. Therefore
\begin{eqnarray}
&{\bf \Phi}^{\infty}(z_n^1) = {\bf \Phi}(z_n^1) + o(1) =
{\bf\Phi}(z_n) - {\bf \Phi}(z) + o(1)&\\ & {\bf \Phi}^{\infty
'}(z_n^1) = {\bf \Phi}' (z_n^1) + o(1) = {\bf \Phi}'(z_n) - {\bf
\Phi}'(z) + o(1).&
\end{eqnarray}
Suppose $z_n^1 = z_n - z \not\to 0 \quad {\rm strongly} \,\,{\rm
in}\,\, E$(otherwise we shall have finished). We want to show that
there exists $ {x_n^1} \subset \mathbb{R}^N$ such that $|x_n^1| \to
+\infty$ and $z_n^1(x + x_n^1) \to z^1 \not\equiv 0$ weakly in
$E$. We note that
\[
{\bf \Phi}^{\infty}(z_n^1) \geq \alpha > 0
\]
because $\|z_n^1\|_E \not \to 0$. In fact, were it not true, we
would have
\begin{equation}
{\bf \Phi}^{\infty}(z_n^1) \to 0, \quad <{\bf \Phi}^{\infty
'}(z_n^1),\eta> = o(1)\|\eta\|_E \quad {\rm as} \quad n \to
\infty.
\end{equation}
Taking $\eta = (\frac {\beta}{\alpha + \beta}u_n^1, \frac
{\alpha}{\alpha + \beta} v_n^1) =:\eta_n$ in (3.11), it follows
\begin{eqnarray}
o(1)\|\eta_n\|_E &=&\frac {\beta}{\alpha + \beta}\int_{\mathbb{R}^N}
u_n^1\bar f(u_n^1)\,dx + \frac {\alpha}{\alpha + \beta}\int_{\Bbb
R^N} v_n^1\bar g(v_n^1)\,dx \nonumber\\
&&-\int_{\mathbb{R}^N} \bar F(u_n^1)\,dx
- \int_{\mathbb{R}^N} \bar G(v_n^1)\,dx.
\end{eqnarray}
Using hypothesis (H4) we obtain
\[
\int_{\mathbb{R}^N} (\bar F(u_n^1) + \bar G(v_n^1))\,dx =
o(1)\|\eta\|_E.
\]
This and (3.12) yield
\begin{equation}
\int_{\mathbb{R}^N} u_n^1\bar f(u_n^1)\,dx = o(1)\|\eta_n\|_E, \quad
\int_{\mathbb{R}^N} v_n^1\bar g(v_n^1)\,dx = o(1)\|\eta_n\|_E.
\end{equation}
It follows from assumptions (H2)-(H4) that
\begin{equation}
|\bar f(t)|^2 \leq C t\bar f(t) \quad {\rm if}\quad |t| \leq 1,
\quad |\bar f(t)|^{(p+1)'} \leq C t\bar f(t) \quad {\rm if}\quad
|t|> 1.
\end{equation}
Taking $\eta = (\phi, 0)$ in (3.11) and using (3.14) and
H\"older's inequality, we obtain
\begin{eqnarray}\label{eq:3.15}
\lefteqn{ |\int_{\mathbb{R}^N} (\nabla \phi
\nabla v_n^1 + \phi v_n^1)\,dx| }\nonumber\\
&\leq& |\int_{\{|u_n^1|\leq 1\}} + \int_{\{|u_n^1| >
1\}}\phi\bar f(u_n^1)\,dx| \\
&\leq& C(\int_{\mathbb{R}^N}
|f(u_n^1)|^2\,dx)^{\frac 12}\|\phi\|_{L^2} + C(\int_{\Bbb
R^N}|\bar f(u_n^1)|^{(p+1)'}\,dx)^{1/(p+1)'}\|\phi\|_{L^{p+1}}\nonumber\\
&\leq& C\|\phi \|_{H^s}[(\int_{\mathbb{R}^N} u_n^1 \bar
f(u_n^1)\,dx)^{\frac 12} + C(\int_{\mathbb{R}^N}u_n^1\bar
f(u_n^1)\,dx)^{1/(p+1)'}].\nonumber
\end{eqnarray}
which with (3.13) imply that
\begin{equation}
\|v_n^1\|_{H^1} = o(1).
\end{equation}
Similarly, we show that
\begin{equation}
\|u_n^1\|_{H^1} = o(1).
\end{equation}
(3.16) and (3.17) yield $\|z_n^1\|_E \to 0$ , we get a
contradiction.
We decompose $\mathbb{R}^N$ into N-dimensional unit hypercubes
$Q_j$ with vertices having integer coordinates and put
\[
d_n = {\rm max}_j(\|u_n^1\|_{L^{p + 1}(Q_j)} + \|v_n^1\|_{L^{q +
1} (Q_j)}).
\]
We claim that there is a $\beta > 0$ such that
\begin{equation}
d_n \geq \beta
> 0 \quad \forall n \in \Bbb N.
\end{equation}
Suppose, by contradiction, that $d_n \to 0$ as $n \to \infty$.
Since
\begin{equation}
{\bf \Phi}^{\infty '}(z_n^1) \to 0 \quad {\rm as} \quad n \to
\infty,
\end{equation}
noting that $\|z_n^1\|_E$ is bounded, we have by (H2) and (H3)
that
\begin{eqnarray*}
0&\leq&{\bf\Phi}^{\infty}(z_n^1) \leq \int_{\mathbb{R}^N} u_n^1\bar
f(u_n^1)\,dx + \int_{\mathbb{R}^N} v_n^1 \bar g(v_n^1)\,dx + o(1)\\
&\leq& C_{\epsilon} (\|u_n^1\|^{p+1}_{L^{p+1}(R^N)} +
\|v_n^1\|^{q+1}_{L^{q+1} (R^N)}) + \epsilon
(\|u_n^1\|^2_{L^2(R^N)} + \|v_n^1\|^2_{L^2(R^N)})\\
& \leq&
C_{\epsilon}\sum_j (\|u_n^1\|^{p + 1}_{L^{p+1}(Q_j)} +
\|v_n^1\|^{q+1}_{L^{q+1}(Q_j)}) + \epsilon( \|u_n^1\|_{L^2(R^N)}^2
+ \|v_n^1\|^2_{L^2(R^N)})\\
& \leq& C_{\epsilon}d_n^{p-1} \sum_j
\|u_n^1\|^2_{L^{p+1}(Q_j)} + C_{\epsilon}d_n
^{q-1}\sum_j\|v_n^1\|^2_{L^{q+1}(Q_j)} + \epsilon C\\
&\leq& C_{\epsilon}d_n^{p-1} \sum_j \|u_n^1\|^2_{H^1(Q_j)} +
C_{\epsilon}d_n^ {q-1} \sum_j\|v_n^1\|^2_{H^1(Q_j)} + \epsilon C\\
&\leq& C_{\epsilon}d_n^{p-1} \|u_n^1\|^2_{H^1} +
C_{\epsilon}d_n^{q-1} \|v_n^1\|^2_{H^1} + \epsilon C.
\end{eqnarray*}
Let $n \to \infty$ and then $\epsilon \to 0$, we obtain
$\Phi^{\infty}(z_n^1) \to 0 \quad {\rm as} \quad n \to \infty$.
This and (3.19) imply as above that $\|z_n^1\|_E \to 0$ as $n \to
\infty$, a contradiction, hence we have (3.18).
Let $\{x_n^1\}$ be the center of a hypercube $Q_j$ in which
\[
d_n = \|u_n^1\|_{L^{p+1}(Q_j)} + \|v_n^1\|_{L^{q+1}(Q_j)}.
\]
Now we show that
\begin{equation}
|x_n^1| \to \infty \quad {\rm as} \quad n \to \infty.
\end{equation}
If $\{x_n^1\}$ were bounded, by passing to a subsequence if
necessary we should find that $x_n^1$ would be in the same ${Q_j}$
and so they should coincide. Therefore in that $Q_j$, for every $n
>n_o$, $n_o$ fixed and large enough, we should have
\begin{eqnarray*}
{\bf \Phi}^{\infty}|_{E(Q_j)}(\bar z_n^1)
&=& \int_{Q_j}
(\nabla \bar u_n^1 \nabla \bar v_n^1 + \bar u_n^1 \bar v_n^1)\,dx
- \int_{Q_j} (\bar F(\bar u_n^1) + \bar G(\bar v_n^1))\,dx +o(1)\\
&\geq& (\alpha -1)\int_{R^N}\bar F(\bar u_n^1)\,dx + (\beta
-1)\int_{R^N}\bar G(\bar v_n^1)\,dx + o(1)\\
&\geq& C(\|\bar
u_n^1\|_{L^{\alpha}(Q_j)}^{\alpha} + \|\bar v_n^1\|_{L^{\beta}
(Q_j)}^{\beta}) + o(1)\\
&\geq& C(\|\bar u_n^1\|^{\alpha}_{L^{p+1}(Q_j)} + \|\bar v_n^1\|^{\beta}_
{L^{q+1}(Q_j)}) + o(1),
\end{eqnarray*}
and
\[
{\bf \Phi}^{\infty '}(\bar z_n^1) \to 0 \quad {\rm as} \quad n \to
0,
\]
where $$ \bar z_n^1 (x)=\cases {z_n^1(x) &$z\in Q_j$\cr 0 &$x \in
\mathbb{R}^N\backslash Q_j$.}
$$ Hence $\bar z_n^1$ should converge
strongly in $E(Q_j)$ to a nonzero function, contradicting to
$z_n^1 \to 0 \quad {\rm weakly}\quad {\rm in} \quad E,$ so we have
(3.20). Let
\[
z_n^1(\cdot + x_n^1) \to z^1 \quad {\rm weakly}\quad {\rm in}
\quad E.
\]
Denote by $\bar Q$ the unit hypercube centered at the origin, we
have $\|z_n^1\|_{E(\bar Q)} \geq \beta
>0,$ thus $z^1 \not\equiv 0$ and
\begin{equation}
\langle {\bf\Phi}^{\infty '}(z^1), \eta \rangle = 0,\quad \forall \eta \in E.
\end{equation}
Iterating the procedure, we obtain sequences $x_n^l, |x_n^l| \to
\infty$ and $$\displaylines{ z_n^l(x) = z_n^{l-1}(x +x_m) -
z^{l-1}(x), \quad j \geq 2 \cr z_n^l(x+x_n^l) \to z^l(x) \quad
{\rm weakly} \quad {\rm in} \quad E }$$ as $n \to 0$, where each
${z^l}$ satisfying (3.21) and by induction
$$\displaylines{
\|z_n^l\|_E = \|z_n^{l-1}\|^2_E - \|z^{l-1}\|^2_E =\|z_n\|^2_E -
\|z\|^2_E - \sum_{i=1}^{l-1} \|z^i\|^2_E + o(1). \cr
{\bf\Phi}^{\infty}(z_n^l) = {\bf\Phi}^{\infty}(z_n^{l-1}) -
{\bf\Phi}^{\infty}(z^{l-1})+o(1) = {\bf\Phi}(z_n) - {\bf\Phi}(z) -
\sum_{i=1}^{l-1} {\bf\Phi}(z^i) + o(1). }$$ Since $z^l$ is a
solution of (1.3)-(1.4) and $z^l \not \equiv 0$, we may prove as
Lemma 4.1 below that $\|z^l\|_E \geq C
>0$. Thus the iteration will terminate at some index $k \geq 0$.
The assertion follows.
\section{Uniform bounds and proof of Theorem 1.1}
We shall bound critical values defined in (2.9) by the energy of the ground
state of problem (1.3)-(1.4). By a ground state of problem (1.3)-(1.4) we
mean a minimizer of the variational problem
\begin{equation}
\Phi^{\infty} = \inf \{ \Phi^{\infty}(u,v): (u,v)\in E
\mbox{ is a solution of (1.3)-(1.4)}, (u,v)
\not\equiv (0,0)\}.
\end{equation}
It is shown in \cite{FY} that problem (1.3)-(1.4) has a positive
radially decaying solution, so the variational problem (4.1) is
well defined.
\begin{lemma} Variational problem (4.1) is assumed by a
nontrivial solution of (1.3)-(1.4).
\end{lemma}
\paragraph{Proof.} Let $z_n = (u_n,v_n)$ be a minimizing sequence of
$\Phi^{\infty}$. It is obvious that $\{z_n\}$ is a (PS) sequence
of $\Phi^{\infty}$. We deduce by Proposition 3.1 that $$
\Phi^{\infty} = \Phi(z_n) + o(1) = \sum_{j=1}^k\Phi^{\infty}(z_j)
+ o(1), $$ where $z_j$ is a solution of (1.3)-(1.4). By the
definition of $\Phi^{\infty}$, $k = 1$. The proof will be
completed if we show $z_1 \not = 0$. To this end, we bound
solutions of (1.3)-(1.4) in $H^1$ norm below by a positive
constant.
Suppose $z = (u,v)$ is a solution of (1.3)-(1.4), we have
\begin{equation}
\|u\|^2_{H^1} = \int_{\mathbb{R}^N} u \bar g (v) \,dx, \,\,
\|v\|^2_{H^1} = \int_{\mathbb{R}^N} v \bar f(u)\,dx,
\end{equation}
and
\begin{equation}
\int_{\mathbb{R}^N} (\nabla u \nabla v + u v ) \,dx = \int_{\mathbb{R}^N}
v \bar g(v)\,dx = \int_{\mathbb{R}^N} u\bar f(u)\,dx.
\end{equation}
By assumptions (H2), (H3) and (H5), we obtain
\begin{equation}
\bar f(u) \leq C_{\epsilon} |u|^{\frac {N+2}{N-2}} + \epsilon u,
\quad \bar g(v) \leq C_{\epsilon} |v|^{\frac {N+2}{N-2}} +
\epsilon v.
\end{equation}
We deduce by (4.2)-(4.4) and H\"older's inequality that
\[
\|u\|^2_{H^1} \leq C_{\epsilon} \|v\|^{2^*-1}_{L^{2^*}}
\|u\|_{L^{2^*}} +
\epsilon \|u\|_{L^2} \|v\|_{L^2},
\]
where $2^* = \frac {2N}{N - 2}$. Using Young's inequality and Sobolev
embedding, we obtain
\[
\|u\|^2_{H^1} \leq C_{\epsilon} (\|u\|^{2^*}_{H^1} + \|v\|^{2^*}
_{H^1} ) +
\epsilon \|v\|^2_{H^1}.
\]
Similarly,
\[
\|v\|^2_{H^1} \leq C_{\epsilon} (\|u\|_{H^1}^{2^*} + \|v\|_{H^1}^{2^*}) + \epsilon
\|u\|^2_{H^1}.
\]
So for $\epsilon$ small, we have
\[
\|u\|^2_{H^1} + \|v\|^2_{H^1} \leq C (\|u\|^{2^*}_{H^1} +
\|v\|^{2^*}_{H^1} ).
\]
It yields that
$\|u\|_{H^1} \quad {\rm or} \quad \|v\|_{H^1} \geq C > 0$,
uniformly for solutions of (1.3)-(1.4), and where $C > 0$ is independent of
$z = (u,v)$. Consequently,
$z_1 = (u_1, v_1) \not\equiv 0$. \hfill$\Box$\medskip
Let $R_n \to \infty, B_n = B_{R_n}(0)$. Taking $\Omega = B_n$ in problem
(2.1)-(2.2), we infer from Theorem 2.1 that there exists a solution $z_n$
of problem (2.1)- (2.2) defined on $B_n$ for each $n$. Moreover,
\begin{equation}
\Phi(z_n) = \Psi(w_n) = c_n \geq \alpha >0,
\end{equation}
where $z_n = K w_n$ , $\Phi = \Phi_{R^N}$ and $\Psi = \Psi_{R^N}$.
We have extended $z_n$ to $\mathbb{R}^N$ by letting $z_n = 0$ outside
$B_n$.
\begin{prop} $c_n < \Phi^{\infty}$ for $n$ large.
\end{prop}
\paragraph{Proof.} Since each element $w$ in $X^*_n =
L^{1+1/p}(B_n)\times L^{1+1/q} (B_n)$ can be extended to an
element of $X^*$ by letting $w = 0$ outside $B_n$, we shall denote
$\Psi_{B_n}$ as $\Psi$ in brief. By Lemma 4.1, $\Phi^{\infty}$
is assumed. Let $z_o = (u_o, v_o)$ be a minimizer of
$\Phi^{\infty}$. Choosing
\[
w_1^o = \bar f(u_o), \quad w_2^o = \bar g(v_o)
\]
and using (H4)-(H5) and equations (1.3)-(1.4), one has
$\int_{\mathbb{R}^N}