\documentclass[reqno]{amsart}
\usepackage{amsfonts}

\AtBeginDocument{{\noindent\small
{\em Electronic Journal of Differential Equations},
Vol. 2004(2004), No. 138, pp. 1--8.\newline
ISSN: 1072-6691. URL: http://ejde.math.txstate.edu or http://ejde.math.unt.edu
\newline ftp ejde.math.txstate.edu  (login: ftp)}
\thanks{\copyright 2004 Texas State University - San Marcos.}
\vspace{9mm}}


\begin{document}
\title[\hfilneg EJDE-2004/138\hfil Elliptic systems involving critical Sobolev exponents]
{Existence results for elliptic systems involving critical
Sobolev exponents}

\author[Mohammed Bouchekif, Yasmina Nasri\hfil EJDE-2004/138\hfilneg]
{Mohammed Bouchekif, Yasmina Nasri} % in alphabetical order

\address{Mohammed Bouchekif \hfill\break 
Departement of Mathematics,
University of Tlemcen B. P. 119 Tlemcen 13000, Algeria}
\email{m\_bouchekif@mail.univ-tlemcen.dz}

\address{Yasmina Nasri \hfill\break 
Departement of Mathematics, University
of Tlemcen B. P. 119 Tlemcen 13000, Algeria}
\email{y\_nasri@mail.univ-tlemcen.dz}

\date{}
\thanks{Submitted July 6, 2004. Published November 25, 2004.}
\subjclass[2000]{35J20, 35J50, 35J60}
\keywords{Elliptic system; critical Sobolev exponent; variational method;
\hfill\break\indent moutain pass theorem}

\begin{abstract}
 In this paper, we study the existence and nonexistence of
 positive solutions of an elliptic system involving critical 
 Sobolev exponent perturbed by a weakly coupled term.
\end{abstract}

\maketitle

\numberwithin{equation}{section} 
\newtheorem{theorem}{Theorem}[section]
\newtheorem{lemma}[theorem]{Lemma}

\section{Introduction}

We establish conditions for existence and nonexistence of nontrivial
solutions to the system 
\begin{equation}
\begin{gathered} 
-\Delta u=(\alpha +1)u^{\alpha }v^{\beta +1} +\mu (\alpha
'+1)u^{\alpha '}v^{\beta '+1} \quad \text{in }\Omega \\ 
-\Delta v=(\beta
+1)u^{\alpha +1}v^{\beta }+\mu (\beta '+1)u^{\alpha '+1}v^{\beta '} \quad
\text{in } \Omega \\ 
u>0,\quad v>0 \quad \text{in }\Omega \\ 
u=v=0 \quad \text{on }\partial \Omega , \end{gathered}  \label{Smu}
\end{equation}
where $\Omega $ is a bounded regular domain of $\mathbb{R}^{N}$ $(N\geq 3)$
with smooth boundary $\partial \Omega $, $\mu \in \mathbb{R}$, $\alpha $, 
$\beta $, $\alpha '$, $\beta '$ are positive constants such
that $\alpha +\beta =\frac{4}{N-2}$ and 
$0\leq \alpha '+\beta '<\frac{4}{N-2}$.

In the scalar case, the problem 
\begin{equation}  \label{Pmu}
\begin{gathered} -\Delta u=u^{p}+\mu u^{q} \quad \text{in }\Omega \\ 
u>0 \quad \text{in }\Omega \\ u=0\quad \text{on }\partial \Omega , \end{gathered}
\end{equation}
has been considered by several authors. The paper of Brezis-Nirenberg 
\cite{7} has drawn our attention.

In \cite{7}, they have obtained the following results: Suppose that $\Omega $
is a bounded domain in $\mathbb{R}^{N}$, $N\geq 3$, $p=\frac{N+2}{N-2}$, 
$q=1 $ and let $\lambda _{1}>0$ denote the first eigenvalue of the operator 
$-\Delta $ with homogeneous Dirichlet boundary conditions.

\begin{enumerate}
\item If $N\geq 4$, then for any $\mu \in (0,\lambda _{1})$ there exists a
solution of \eqref{Pmu}.

\item If $N=3$, there exists $\mu ^{\ast }\in (0,\lambda _{1})$ such that
for any $\mu \in (\mu ^{\ast },\lambda _{1})$ problem \eqref{Pmu} admits a
solution.

\item If $N=3$ and $\Omega $ is a ball, then 
$\mu ^{\ast }=\frac{\lambda _{1}}{4}$ and for 
$\mu \leq \frac{\lambda _{1}}{4}$ problem \eqref{Pmu} has no
solution.
\end{enumerate}

They have also obtained the following results for $1<q<\frac{N+2}{N-2}$:

\begin{itemize}
\item[(a)] There is no solutions of \eqref{Pmu} when $\mu \leq 0$ and $\Omega $ 
is a starshaped domain.

\item[(b)] When $N\geq 4$, \eqref{Pmu} has at least one solution for every $\mu >0$.

\item[(c)] When $N=3$, We distinguish two cases:\newline
(i) If  $3<q<5$, then for every $\mu >0$ there is a solution of \eqref{Pmu}. \newline
(ii) If $1<q\leq 3$, then for every $\mu $ large enough there is a solution
of \eqref{Pmu}. \newline
Moreover, \eqref{Pmu} has no solution for every small $\mu >0$ when $\Omega $
is strictly starshaped.
\end{itemize}

In the vectorial case, Alves et al. \cite{1} and Bouchekif and Nasri \cite{4}
have extended the results of \cite{7} to elliptic system. A number of works
contributed to study the elliptic system for example: Boccardo and de
Figueiredo \cite{3}, de Th\'{e}lin and V\'{e}lin \cite{11} and Conti et al. 
\cite{8}.

Our aim is to generalize the results of \cite{7} to an elliptic system when
the lower order perturbation of $u^{\alpha +1}v^{\beta +1}$ for each
equation is weakly coupled i. e. 
\begin{equation*}
-\overset{\rightarrow }{\Delta }U=\nabla H+\mu \nabla G,
\end{equation*}
where 
\begin{equation*}
\overset{\rightarrow }{\Delta }= 
\begin{pmatrix}
\Delta \\ 
\Delta
\end{pmatrix}
,\quad H(u,v)=u^{\alpha +1}v^{\beta +1},\quad U= 
\begin{pmatrix}
u \\ 
v\end{pmatrix},
\end{equation*}
$G(u,v)=u^{\alpha '+1}v^{\beta '+1}$ and $\mu $ is a real
parameter.

Our main results are stated as follows :

\begin{theorem} \label{thm1}
If $\alpha +\beta =\frac{4}{N-2}$; $0\leq \alpha '+\beta'<\frac{4}{N-2}$;
$\mu \leq 0$ and $\Omega $ is a starshaped
domain, then \eqref{Smu} has no solution.
\end{theorem}

\begin{theorem} \label{thm2}
We suppose that $N\geq 4$ and $\alpha +\beta =\frac{4}{N-2}$. We have:
\begin{itemize}
\item If $0<\alpha '+\beta '<\frac{4}{N-2}$, then\ for every $
\mu >0$  problem \eqref{Smu} has at least one solution.

\item If $\alpha '+\beta '=0$, then $\ $for every $0<\mu<\lambda _{1}$ 
problem \eqref{Smu} has a solution.
\end{itemize}
\end{theorem}

\begin{theorem} \label{thm3}
Assume that $N=3$ and $\alpha +\beta =4$. We distinguish two cases:
\begin{itemize}
\item If $2<\alpha '+\beta '<4$, then for every $\mu >0$
problem \eqref{Smu} has a solution.

\item If $0<\alpha '+\beta '\leq 2$, then for every $\mu $
large enough there exists a solution to problem \eqref{Smu}.
\end{itemize}
\end{theorem}

The paper is organized as follows. Section 2 contains some preliminaries and
notations. Section 3 contains the proof of nonexistence result. Section 4
deals with the existence theorems proofs.

\section{Preliminaries}

\begin{lemma}[Pohozaev identity \cite{10}] \label{lem1}
Suppose that $(u,v)\in [C^{2}(\Omega )] ^{2}$ is the solution to
the problem
\begin{gather*}
-\Delta u=\frac{\partial F}{\partial u}(u,v)\quad \text{in } \Omega \\
-\Delta v=\frac{\partial F}{\partial v}(u,v)\quad \text{in } \Omega \\
u=v=0\quad \text{on }\partial \Omega ,
\end{gather*}
where $\Omega $ is a bounded domain in $\mathbb{R}^{N}$
$(N\geq 3)$ with smooth boundary $\partial \Omega $,
$F\in C^{1}(\mathbb{R}^{2})$, $F(0,0)=0$, then
we have
\begin{equation} \label{e1}
\int_{\partial \Omega }(| \frac{\partial u}{\partial \nu }| ^{2}
+| \frac{\partial v}{\partial \nu }|^{2})x\nu d\sigma
+(N-2)\big[ \int_{\Omega }(u
\frac{\partial F}{\partial u}+v\frac{\partial F}{\partial v})dx\big]
=2N\int_{\Omega }F(u,v)dx
\end{equation}
where $\nu $ denotes the exterior unit normal.
\end{lemma}

We shall use the following version of the Brezis-Lieb lemma \cite{6}.

\begin{lemma} \label{lem2}
Assume that $F\in C^{1}(\mathbb{R}^{N})$ with $F(0) =0$ and 
$|\frac{\partial F}{\partial u_{i}}| \leq C| u| ^{p-1}$. Let
$(u_{n})\subset L^{p}(\Omega )$ with $1\leq p<\infty $. If
$(u_{n})$ is bounded in $L^{p}(\Omega )$ and $u_{n}\to u$
a.e. on $\Omega $, then
\[
\lim_{n\to \infty}(\int_{\Omega }F(u_{n})-F(u_{n}-u))=\int_{\Omega }F(u).
\]
\end{lemma}
Let us define: 
\begin{gather*}
S_{\alpha +\beta +2}=S_{\alpha +\beta +2}(\Omega ):=\inf_{u\in
H_{0}^{1}(\Omega )\backslash \{0\}}\frac{\int_{\Omega }|\nabla u|^{2}dx}{
(\int_{\Omega }|u|^{\alpha +\beta +2}dx)^{\frac{2}{\alpha +\beta +2}}} \\
S_{\alpha ,\beta }=S_{\alpha ,\beta }(\Omega ):=\inf_{(u,v)\in \left[
H_{0}^{1}(\Omega )\right] ^{2}\backslash \{(0,0)\}}\frac{\int_{\Omega
}(|\nabla u|^{2}+|\nabla v|^{2})dx}{(\int_{\Omega }|u|^{\alpha +1}|v|^{\beta
+1}dx)^{\frac{2}{\alpha +\beta +2}}}.
\end{gather*}

\begin{lemma}[\cite{1}] \label{llem3}
Let $\Omega $ be a domain in $\mathbb{R}^{N}$
(not necessarily bounded) and $\alpha +\beta \leq \frac {4 }{N-2}$, 
then we have
\[
S_{\alpha ,\beta }=\Big[ \big(\frac{\alpha +1}{\beta +1}\big)
^{\frac{\beta +1}{\alpha +\beta +2}}
+\big(\frac{\alpha +1}{\beta +1}\big)^{\frac{-\alpha -1}{\alpha +\beta +2}}\Big]
S_{\alpha +\beta +2}.
\]
Moreover, if $S_{\alpha +\beta +2}$ is attained at $\omega _{0}$, then $
S_{\alpha ,\beta }$ is attained at $(A\omega _{0},B\omega _{0})$
for any real constants $A$ and $B$ such that $\frac{A}{B}=(\frac{
\alpha +1}{\beta +1})^{1/2}$.
\end{lemma}

We adopt the following notation:

\begin{itemize}
\item For $p>1$, $\Vert u\Vert _{p}=[\int_{\Omega }|u|^{p}dx]^{\frac{1}{p}}$;

\item $H_{0}^{1}(\Omega )$ is the Sobolev space endowed with the norm $\Vert
u\Vert _{1,2}=[\int_{\Omega }|\nabla u|^{2}dx]^{1/2}$;

\item $\Vert (u,v)\Vert _{E}^{2}:=\Vert u\Vert _{1,2}^{2}+\Vert v\Vert
_{1,2}^{2};$

\item $E:=[H_{0}^{1}(\Omega )]^{2}$;

\item $E'$ denotes the dual of $E$;

\item $2^{\ast }:=\frac{2N}{N-2}$ is the critical Sobolev exponent;

\item $u^{+}:=\max (u,0)$ and $u^{-}=u^{+}-u$.
\end{itemize}

The functional associated to problem \eqref{Smu} is written as 
\begin{equation}  \label{e2}
J(u,v):=\frac{1}{2}\| (u,v)\| _{E}^{2}-\int_{\Omega }(u^{+})^{\alpha
+1}(v^{+})^{\beta +1}dx -\mu \int_{\Omega }(u^{+})^{\alpha '+1}(
v^{+})^{\beta '+1}dx.  
\end{equation}

\section{Nonexistence result}

Theorem \ref{thm1} is a direct consequence of the Pohozaev identity.

\begin{proof}[Proof of Theorem \protect\ref{thm1}]
Arguing by contradiction. Suppose that problem \eqref{Smu} has a solution 
$(u,v)\neq (0,0)$, applying Lemma \ref{lem1} and putting 
\begin{equation*}
F(u,v)=H(u,v)+\mu G(u,v),
\end{equation*}
the expression \eqref{e1} becomes 
\begin{equation*}
\int_{\partial \Omega }\big(| \frac{\partial u}{\partial \nu }| ^{2}+| \frac{%
\partial v}{\partial \nu }| ^{2}\big)x\nu \,d\sigma =\mu \left[
2N-(N-2)(\alpha '+\beta '+2)\right] \int_{\Omega }| u|
^{\alpha '+1}| v| ^{\beta '+1}dx.
\end{equation*}
Since $2N-(N-2)(\alpha '+\beta '+2)>0$ and the fact that 
$\Omega $ is starshaped with respect to the origin, we get 
\begin{equation*}
0\leq \int_{\partial \Omega }(| \frac{\partial u}{\partial \nu } | ^{2}+| 
\frac{\partial v}{\partial \nu }| ^{2})x\nu\, d\sigma <0.
\end{equation*}
A contradiction. Hence \eqref{Smu} has no a solution for $\mu\leq 0$.
\end{proof}

\section{Existence results}

The proof of Theorems \ref{thm2} and \ref{thm3} are based on the following
Ambrosetti-Rabinowitz result \cite{2}.

\begin{lemma}[Mountain Pass Theorem] \label{lem4}
 Let $J$ be a $C^{1}$ functional on a Banach space $E$. Suppose there exits
a neighborhood $V$ of $0$ in $E$ and a positive
constant $\rho $ such that
\begin{itemize}
\item[(i)]  $J(u,v)\geq \rho $ for every $U$ in the
boundary of $V$.

\item[(ii)]  $J(0,0)<\rho $ and $J(\varphi ,\psi)<0$ for some
$\Psi:=(\varphi ,\psi )\notin V$.
\end{itemize}
We set
\[
c=\inf_{\phi \in \Gamma }\max_{t\in [ 0,1] }J(\phi (t))
\]
with $\Gamma =\{\phi\in C([ 0,1] ,E):\phi (0)=0,\, \phi
(1)=\Psi\}$. Then there exists a sequence $(u_{n},v_{n})$ in $E$
such that $J(u_{n},v_{n})\to c$ and $J'(u_{n},v_{n})\to 0$ in
$E'$.
\end{lemma}

\begin{proof}
Using Holder's inequality and Sobolev injection, we obtain that 
\begin{align*}
J(u,v)&=\frac{1}{2}\| (u,v)\| _{E}^{2}-\int_{\Omega }(u^{+})^{\alpha
+1}(v^{+}) ^{\beta +1}dx-\mu \int_{\Omega }(u^{+})^{\alpha ^{\prime
}+1}(v^{+})^{\beta '+1}dx \\
&\geq \frac{1}{2}\| (u,v)\| _{E}^{2}-A\|(u,v)\| _{E}^{2^{\ast }}-B\| (u,v)
\| _{E}^{\alpha '+\beta '+2}
\end{align*}
where $A$ and $B$ are positive constants.

If $\alpha '+\beta '>0$ then $(i)$ is satisfying for small
norm $\| (u,v)\| _{E}=R$. If $\alpha '+\beta '=0$, we have%
\begin{equation*}
J(u,v)\geq \frac{1}{2}\big(1-\frac{\mu }{\lambda _{1}}\big) \| (u,v)\|
_{E}^{2}-A\| (u,v) \| _{E}^{2^{\ast }}
\end{equation*}
and condition $(i)$ is still satisfied for $\mu <\lambda _{1}$ and 
$R<( \frac{1-\frac{\mu }{\lambda _{1}}}{2A})^{\frac{1}{2^{\ast }-2}}$. 
For any $(\varphi ,\psi )\in E$ with $\varphi \neq 0$ and $\psi \neq 0$, 
we have that 
$\lim_{t\to +\infty }J(t\varphi ,t\psi )=-\infty $. Thus, there are many 
$(\varphi ,\psi )$ satisfying $(ii)$. It will be important to use with a
special $(\varphi ,\psi ):=(t_{0}\varphi_{0},t_{0}\psi _{0})$ for some 
$t_{0}>0$ chosen large enough so that $(\varphi ,\psi )\notin V$, 
$J(\varphi,\psi )<0$ and 
$\sup_{t\geq 0}J(t\varphi ,t\psi )<\frac{2^{\ast } }{N}
(\frac{S_{\alpha ,\beta }}{2^{\ast }})^{N/2}$. Then there exists a sequence 
$(u_{n},v_{n})\in E$ such that $J(u_{n},v_{n})\to c$ and 
$J'(u_{n},v_{n})\to 0$ in $E'$.
\end{proof}

\begin{lemma} \label{lem5}
Suppose $\mu >0$ and let $(u_{n},v_{n})$ be a sequence in $E$ such
that \break $J(u_{n},v_{n})\to c$ and $ J'(u_{n},v_{n})\to 0$ in $ E'$
with
\[
c<\frac{2^{\ast }}{N}(\frac{S_{\alpha ,\beta }}{2^{\ast }})^{N/2}
=\frac{2}{N-2}(\frac{S_{\alpha ,\beta }}{2^{\ast}}) ^{N/2}
\]
Then $(u_{n},v_{n})$ is relatively compact in $E$.
\end{lemma}

\begin{proof}
We show that the sequence $(u_{n},v_{n})$ is bounded in $E$. Since 
$(u_{n},v_{n})$ satisfies: 
\begin{equation}
\frac{1}{2}\Vert (u_{n},v_{n})\Vert _{E}^{2}-\int_{\Omega
}(u_{n}^{+})^{\alpha +1}(v_{n}^{+})^{\beta +1}dx-\mu \int_{\Omega
}(u_{n}^{+})^{\alpha '+1}(v_{n}^{+})^{\beta '+1}dx=c+o(1)
\label{e3}
\end{equation}
and
\begin{equation}
\begin{aligned} &\| (u_{n},v_{n})\| _{E}^{2}-2^{\ast }\int_{\Omega
}(u_{n}^{+})^{\alpha +1}(v_{n}^{+})^{\beta +1}dx -\mu (\alpha '+\beta
'+2)\int_{\Omega }( u_{n}^{+})^{\alpha '+1}(v_{n}^{+})^{\beta '+1}dx\\
&=\langle \varepsilon _{n},(u_{n},v_{n})\rangle \end{aligned}  \label{e4}
\end{equation}
with $\varepsilon _{n}\rightarrow 0$ in $E'$. Combining (4.1) and
(4.2), we obtain 
\begin{equation}
\begin{aligned} 
&(\frac{2^{\ast }}{2}-1)\int_{\Omega }(u_{n}^{+}) ^{\alpha
+1}(v_{n}^{+})^{\beta +1}dx+\mu (\frac{\alpha '+\beta '}{2})\int_{\Omega
}(u_{n}^{+}) ^{\alpha '+1}(v_{n}^{+})^{\beta '+1}dx\\ 
&\leq c+o(1)+\|
\varepsilon _{n}\| _{E'}\| (u_{n},v_{n})\| _{E}. \end{aligned}  \label{e5}
\end{equation}
From this inequality, we obtain 
\begin{gather*}
\int_{\Omega }(u_{n}^{+})^{\alpha +1}(v_{n}^{+})^{\beta +1}dx\leq C\,, \\
\int_{\Omega }(u_{n}^{+})^{\alpha '+1}(v_{n}^{+})^{\beta ^{\prime
}+1}dx\leq C\,.
\end{gather*}
Where $C$ is any generic positive constant. Therefore, the sequence 
$(u_{n},v_{n})$ is bounded in $E$. By the Sobolev embedding Theorem, there
exists a subsequence again denoted by $(u_{n},v_{n})$ such that

\begin{itemize}
\item $(u_{n},v_{n})\to (u,v)$ weakly in $E$

\item $(u_{n},v_{n})\to (u,v)$ strongly in $L^{r}\times L^{q}$ for 
$2\leq r,q<2^{\ast }$

\item $( u_{n},v_{n})\to (u,v)$ a. e. on $\Omega$.
\end{itemize}

Since $w_{n}:=u_{n}^{\alpha }v_{n}^{\beta +1}$ and $t_{n}:=u_{n}^{\alpha
+1}v_{n}^{\beta }$ are bounded sequences in 
$[L^{\frac{2^{\ast }}{2^{\ast }-1}}(\Omega )]^{2}$, these sequences 
converge to $w:=u^{\alpha }v^{\beta +1}$
and to $t:=u^{\alpha +1}v^{\beta }$ respectively. Passing to the limit, we
obtain 
\begin{gather*}
-\Delta u=(\alpha +1)(u^{+})^{\alpha }(v^{+})^{\beta +1}+\mu (\alpha
'+1)(u^{+})^{\alpha '}(v^{+})^{\beta '+1} \\
-\Delta v=(\beta +1)(u^{+})^{\alpha +1}(v^{+})^{\beta }
+\mu (\beta '+1)(u^{+})^{\alpha '+1}(v^{+})^{\beta '}
\end{gather*}
i.e 
\begin{equation*}
\Vert (u,v)\Vert _{E}^{2}=2^{\ast }\int_{\Omega }(u^{+})^{\alpha
+1}(v^{+})^{\beta +1}dx+\mu (\alpha '+\beta '+2)
\int_{\Omega }(u^{+})^{\alpha '+1}(v^{+})^{\beta '+1}dx
\end{equation*}
Moreover, 
\begin{equation*}
J(u,v)=(\frac{2^{\ast }}{2}-1)\int_{\Omega }(u^{+})^{\alpha
+1}(v^{+})^{\beta +1}dx+\mu (\frac{\alpha '+\beta '}{2}
)\int_{\Omega }(u^{+})^{\alpha '+1}(v^{+})^{\beta '+1}dx\geq 0.
\end{equation*}
We put 
\begin{equation*}
u=u_{n}+\varphi _{n},\text{\ }v=v_{n}+\psi _{n}\quad \text{and}\quad
H(u_{n},v_{n})=u_{n}^{\alpha +1}v_{n}^{\beta +1}
\end{equation*}
Applying Lemma \ref{lem2} for $H(u_{n},v_{n})$ and the following two
relations (Brezis-Lieb \cite{6}) 
\begin{gather*}
\Vert u_{n}\Vert ^{2}=\Vert u-\varphi _{n}\Vert ^{2}=\Vert u\Vert ^{2}+\Vert
\varphi _{n}\Vert ^{2}+o(1)\,, \\
\Vert v_{n}\Vert ^{2}=\Vert v-\varphi _{n}\Vert ^{2}=\Vert v\Vert ^{2}+\Vert
\psi _{n}\Vert ^{2}+o(1),
\end{gather*}
we obtain 
\begin{equation}
J(u,v)+\frac{1}{2}\Vert (\varphi _{n},\psi _{n})\Vert _{E}^{2}-\int_{\Omega
}H(\varphi _{n}^{+},\psi _{n}^{+})dx=c+o(1)  \label{e7}
\end{equation}%
and 
\begin{equation}
\begin{aligned} 
\| (\varphi _{n},\psi _{n})\| _{E}^{2}+\|(u,v)\| _{E}^{2}
&=2^{\ast }\big[ \int_{\Omega }(H(u^{+},v^{+})+H(\varphi _{n}^{+},\psi
_{n}^{+}) )dx\big]\\ 
&\quad +\mu (\alpha '+\beta '+2)\int_{\Omega
}(u^{+})^{\alpha '+1}(v^{+})^{\beta'+1}dx+o(1). \end{aligned}  \label{e8}
\end{equation}
From this equality, we deduce 
\begin{equation*}
\Vert (\varphi _{n},\psi _{n})\Vert _{E}^{2}=2^{\ast }\int_{\Omega
}H(\varphi _{n}^{+},\psi _{n}^{+})dx+o(1).
\end{equation*}
We may therefore assume that 
\begin{equation*}
\Vert (\varphi _{n},\psi _{n})\Vert _{E}^{2}\rightarrow k\quad \text{and}
\quad 2^{\ast }\int_{\Omega }H(\varphi _{n}^{+},\psi _{n}^{+})dx\rightarrow
k.
\end{equation*}%
By the Sobolev inequality, 
\begin{equation*}
\Vert (\varphi _{n},\psi _{n})\Vert _{E}^{2}\geq S_{\alpha ,\beta }\Big(%
\int_{\Omega }\left( \varphi _{n}^{+}\right) ^{\alpha +1}(\psi
_{n}^{+})^{\beta +1}dx\Big)^{\frac{2}{2^{\ast }}}.
\end{equation*}
In the limit, $k\geq S_{\alpha ,\beta }(\frac{k}{2^{\ast }})^{2/2^{\ast }}$.
It follows that either $k=0$ or 
$k\geq 2^{\ast }(\frac{S_{\alpha ,\beta }}{2^{\ast }})^{N/2}$.

We show that $(u_{n},v_{n})\to (u,v)$ strongly in $E$ i. e. 
$(\varphi_{n},\psi _{n})\to (0,0)$ strongly in $E$. Suppose that 
$k\geq 2^{\ast }(\frac{S_{\alpha ,\beta }}{2^{\ast }})^{N/2}$. Since 
\begin{equation*}
J(u,v)+\frac{k}{N}=c
\end{equation*}
and $J(u,v)\geq 0$, then $\frac{k}{N}\leq c$ i.e. $c\geq \frac{ 2^{\ast }}{N}%
(\frac{S_{\alpha ,\beta }(\Omega )}{2^{\ast } })^{N/2}$ in contradiction
with the hypothesis. Thus $k=0$ and $(u_{n},v_{n})\to (u,v)$ strongly in $E$.
\end{proof}

\begin{proof}[Proof of Theorem \protect\ref{thm2}]
It suffices to apply the mountain pass theorem with the value 
$c<\frac{2^{\ast }}{N}(\frac{S_{\alpha ,\beta }(\Omega )}{2^{\ast }})^{N/2}$. 
We have
to show that this geometric condition on $c$ is satisfied. Following the
method in \cite{7}. Without loss of generality we assume that $0\in \Omega $, 
we use the test function 
\begin{equation*}
\omega _{\varepsilon }(x)=\frac{\varphi (x)}{(\varepsilon +\left\vert
x\right\vert ^{2})^{\frac{N-2}{2}}}\,,\quad \varepsilon >0
\end{equation*}%
where $\varphi $ is a cut-off positive function such that $\varphi \equiv 1$
in a neighborhood of $0$. Let $A$ and $B$ be positive constants such that 
\begin{equation*}
\frac{A}{B}=(\frac{\alpha +1}{\beta +1})^{1/2}
\end{equation*}
then $(A\omega _{\varepsilon },B\omega _{\varepsilon })$ is a solution of 
\begin{gather*}
-\Delta u=(\alpha +1)u^{\alpha }v^{\beta +1}\quad \text{in }\mathbb{R}^{N} \\
-\Delta v=(\beta +1)u^{\alpha +1}v^{\beta }\quad \text{in }\mathbb{R}^{N} \\
u(x)=0,\quad v(x)=0\quad \mbox{as }|x|\rightarrow +\infty
\end{gather*}
By \cite[lemma 1]{7}, we obtain 
\begin{equation*}
\sup_{t\geq 0}J(tA\omega _{\varepsilon },tB\omega _{\varepsilon })
\leq \frac{2^{\ast }}{N}\big(\frac{S_{\alpha ,\beta }}{2^{\ast }}\big)^{N/2}
+O\big(\varepsilon ^{\frac{N-2}{2}}\big)-\mu K\varepsilon ^{\theta }
\end{equation*}
where $K$ is a positive constant independent of $\varepsilon $, and 
$\theta:=(4-(\alpha '+\beta ')(N-2))/4$.

For $\theta <\frac{N-2}{2}$ if $N>4$ the inequality is satisfying for all 
$0\leq \alpha '+\beta '<\frac{4}{N-2}$. Thus we obtain 
\begin{equation*}
\sup_{t\geq 0} J(tA\omega _{\varepsilon },tB\omega _{\varepsilon })
<\frac{2^{\ast }}{N}\big(\frac{S_{\alpha ,\beta }}{ 2^{\ast }}\big)^{N/2}\quad 
\text{for }\varepsilon >0\text{ small enough}.
\end{equation*}
Then problem \eqref{Smu} has a solution for every $\mu >0$.

For $N=4$, we distinguish two cases. Case 1: We have $\theta <1$ for all $%
\alpha '+\beta '>0$.\newline
Case 2: If $\alpha '+\beta '=0$, we obtain 
\begin{equation*}
\sup_{t\geq 0} J(tA\omega _{\varepsilon },tB\omega _{\varepsilon })
\leq (\frac{S_{\alpha ,\beta }}{4}) ^{2}+O(\varepsilon )-\mu K\varepsilon | \log
\varepsilon|,
\end{equation*}
so for $\varepsilon >0$ small enough, $\sup_{t\geq 0}J(tA\omega
_{\varepsilon },tB\omega _{\varepsilon })<(\frac{ S_{\alpha ,\beta }}{4})^{2}$.

Note that the maximum principle ensures the positivity of solution.
\end{proof}

\begin{proof}[Proof of Theorem \protect\ref{thm3}]
In three dimension the situation is different. We have 
\begin{equation*}
\sup_{t\geq 0} J(tA\omega _{\varepsilon },tB\omega _{\varepsilon })
\leq 2(\frac{S_{\alpha ,\beta }}{6})^{3/2}+O(\varepsilon ^{1/2}) 
-\mu K\varepsilon ^{\theta }.
\end{equation*}
In this case we distinguish two cases.

\begin{itemize}
\item[(i)] $0<\theta <\frac{1}{2}$ if $2<\alpha '+\beta '<4$,

\item[(ii)] $\theta \geq \frac{1}{2}$ if $0<\alpha '+\beta
'\leq 2$.
\end{itemize}

In case (i) we have the same conclusion as in the previous proof for 
$(N\geq 4)$. So for the case $0<\alpha '+\beta '\leq 2$, the
existence of positive solution is assured for $\mu $ large enough. It
follows that $\sup_{t\geq 0}J(tA\omega _{\varepsilon},tB\omega _{\varepsilon
}) <2(\frac{S_{\alpha ,\beta }}{6})^{3/2}$. Thus \eqref{Smu} has a
solution.
\end{proof}

\begin{thebibliography}{99}
\bibitem{1} C. O. Alves, D. C. de Morais Filho and M. A. S. Souto; \emph{On
Systems of Elliptic equations involving subcritical or critical Sobolev
exponents}, Nonlinear Anal. N. 5, Ser. Theory Methods, 42 (2000), 771--787.

\bibitem{2} A. Ambrosetti and P.H. Rabinowitz;\ \emph{Dual variational
methods in critical point theory and applications}, J. Funct. Anal., 14
(1973), 349-381.

\bibitem{3} L. Boccardo, D.G. de Figueiredo: \emph{Some remarks on system of
quasilinear elliptic equations}, Nonlinear diff. eq. appl. 9 (2002), pp.
309-323.

\bibitem{4} M. Bouchekif and Y. Nasri; \emph{On a class of elliptic system
involving critical Sobolev exponent}, Preprint Tlemcen 2004.

\bibitem{5} H. Brezis; \emph{Some Variational Problems with Lack of
Compactness}, Proceedings of Symposia in Pure Mathematics, Vol. 45 (1986),
165-201.

\bibitem{6} H. Brezis and E. Lieb; \emph{A relation between point wise
convergence of functions and convergence of functionals}, Proc. A.M.S. Vol.
48, No. 3 (1993), 486-499.

\bibitem{7} H. Brezis and L. Nirenberg; \emph{Positive Solutions of
Nonlinear Elliptic Equations Involving Critical Exponents}, Comm. Pure Appl.
Math, Vol. 36 (1983), 437-477.

\bibitem{8} M. Conti, L. Merizzi and S. Terracini; \emph{On the existence of
many solutions for a class of superlinear elliptic system}, J. Diff. Eq. 167
(2000), 357--387.

\bibitem{9} S. I. Pohozaev; \emph{Eigenfunctions of the equation $\Delta
u+\lambda f(u)=0$}, Nonlinearity Doklady Akad. Nauk SSRR 165, (1965), 9-36.

\bibitem{10} P. Pucci and J. Serrin; \emph{A general variational identity},
Indiana University Mathematics Journal, (1986), 681-703.

\bibitem{11} F. de Th\'{e}lin and J. V\'{e}lin; \emph{Existence and
nonexistence of nontrivial solutions for some nonlinear elliptic systems},
Revista Mathematica Universidad complutense de Madrid, vol. 6 (1993),
153-193.
\end{thebibliography}

\end{document}