\documentclass[reqno]{amsart} \usepackage{hyperref} \AtBeginDocument{{\noindent\small \emph{Electronic Journal of Differential Equations}, Vol. 2008(2008), No. 153, pp. 1--13.\newline ISSN: 1072-6691. URL: http://ejde.math.txstate.edu or http://ejde.math.unt.edu \newline ftp ejde.math.txstate.edu (login: ftp)} \thanks{\copyright 2008 Texas State University - San Marcos.} \vspace{9mm}} \begin{document} \title[\hfilneg EJDE-2008/153\hfil Critical nonlinear elliptic systems in exterior domains] {Critical Neumann problem for nonlinear elliptic systems in exterior domains} \author[S. Deng, J. Yang\hfil EJDE-2008/153\hfilneg] {Shengbing Deng, Jianfu Yang} % in alphabetical order \address{Shengbing Deng \newline Department of Mathematics, Jiangxi Normal University, Nanchang, Jiangxi 330022, China} \email{shbdeng@yahoo.com.cn} \address{Jianfu Yang \newline Department of Mathematics, Jiangxi Normal University, Nanchang, Jiangxi 330022, China} \email{jfyang\_2000@yahoo.com} \thanks{Submitted June 25, 2008. Published November 7, 2008.} \subjclass[2000]{35J50, 35J60} \keywords{Neumann problem; elliptic systems; exterior domains; \hfill\break\indent critical Sobolev exponent; least energy solutions} \begin{abstract} In this paper, we investigate the Neumann problem for a critical elliptic system in exterior domains. Assuming that the coefficient $Q(x)$ is a positive smooth function and $\lambda$, $\mu\geq0$ are parameters, we examine the common effect of the mean curvature of the boundary $\partial \Omega$ and the shape of the graph of the coefficient $Q(x)$ on the existence of the least energy solutions. \end{abstract} \maketitle \numberwithin{equation}{section} \newtheorem{theorem}{Theorem}[section] \newtheorem{lemma}[theorem]{Lemma} \newtheorem{proposition}[theorem]{Proposition} \newtheorem{corollary}[theorem]{Corollary} \section{Introduction} In this paper, we are concerned with the following Neumann problem for elliptic systems $$\label{e1.1} \begin{gathered} -\Delta u+\lambda u=\frac{2\alpha}{\alpha+\beta}Q(x)|u|^{\alpha-2}u|v|^\beta \quad \text{in } \Omega^{c},\\ -\Delta v+\mu v= \frac{2\beta}{\alpha+\beta}Q(x)|u|^{\alpha}|v|^{\beta-2}v \quad \text{in} \Omega^{c},\\ \frac{\partial u}{\partial \nu}=\frac{\partial v}{\partial \nu}=0 \quad \text{on } \partial \Omega, \\ \quad u,v>0 \quad \text{in } \Omega^{c}, \end{gathered}$$ where $\Omega\subset\mathbb{R}^N$ is a smooth bounded domain and $\Omega^{c}={\mathbb{R}}^N\setminus\Omega$, we assume that $\Omega^{c}$ has no bounded components. $\lambda, \mu\geq$0 are parameters, $\alpha, \beta>1$ and $\alpha+\beta=2^{\ast}$, where $2^{\ast}$ denotes the critical Sobolev exponent, that is, $2^{\ast}=\frac{2N}{N-2}$ for $N \geq 3$. $\nu$ is the unit inner normal at the boundary $\partial \Omega$. The coefficient $Q(x)$ is H\"{o}lder continuous on $\Omega^{c}$ and $Q(x)> 0$ for all x$\ \in \Omega^{c}$. In\cite{BN}, critical semilinear elliptic problems for one equation with Dirichlet boundary conditions was solved by variational methods. Although the $(PS)$ does not hold globally, it was found in \cite{BN} that the condition is valid locally. Critical point theory then can be used locally to find critical points of associated functionals. The critical Neumann problem was considered in \cite{WX} using the same idea as \cite{BN}. Later on, the critical Neumann has been extensively studied. Various existence results concerning the graph of coefficients, topology of domains etc., can be found in \cite{APY,AY,C,CR,CMW,CW,P,PW,WX} and references therein. Particularly, the Neumann problem in exterior domains $$\label{e1.1a} \begin{gathered} -\Delta u+\lambda u=Q(x)|u|^{2^{\ast}-2}u \quad \text{in }\quad\Omega^{c},\\ \quad \frac{\partial u}{\partial \nu}=0 \quad\text{on }\quad \partial \Omega,\\ \quad u>0 \quad\text{in } \quad\Omega^{c}, \end{gathered}$$ was considered in \cite{CR}. Existence results for \eqref{e1.1a} were obtained by showing $$S(\Omega^{c},Q,\lambda)=\inf\big\{\int_{\Omega^{c}}(|\nabla u|^2+\lambda u^2)\,dx, u\in H^1(\Omega^{c}),\int_{\Omega^{c}}Q(x)|u|^{2^{\ast}}\,dx=1\big\},$$ is achieved. The effect of the graph of $Q$ and the geometry of the domain was taken into account on the existence of solutions of \eqref{e1.1a}. For the system \eqref{e1.1}, it was considered in \cite{AFS} the existence of solutions for subcritical nonlinearities. In the critical case, problem \eqref{e1.1} in bounded domains was investigated in \cite{CY}, where the effect of the shape of $Q(x)$ was considered in the existence of least energy solutions. Inspired of \cite{CR} and \cite{CY}, in this paper, we consider the existence of solutions of problem \eqref{e1.1} in exterior domains. The problem is both critical and setting on unbounded domains. The loss of compactness is caused by noncompact groups of translations and dilations. In applying variational methods, it is necessary to figure out energy levels so that the $(PS)$ condition holds. These energy levels are not only affected by noncompact groups of translations and dilations but also the shape of the coefficient $Q$. Solutions of problem \eqref{e1.1} will be found as a minimizer of the variational problem $$\label{e1.2} S_{\lambda,\mu}(\Omega^c,Q) =\inf_{u,v\in H^1(\Omega^c)\backslash\{0\}} \frac{\int_{\Omega^c}(|\nabla u|^2+|\nabla v|^2+\lambda u^2+\mu v^2)\,dx} {\big(\int_{\Omega^c}Q(x)|u|^\alpha|v|^\beta\,dx\big)^{2/(\alpha+\beta)}},$$ which is a weak solution of \eqref{e1.1} up to a multiple of a constant. It was proved in \cite{AFS} that every weak solution of problem \eqref{e1.1} is classical. As we will see, problem \eqref{e1.2} is closely related to the problem $$\label{e1.3} S_{\alpha,\beta}=\inf_{u,v\in D^{1,2}_0(\Omega^{c})\backslash\{0\}} \frac{\int_{\Omega^{c}}(|\nabla u|^2+|\nabla v|^2)\,dx} {\big(\int_{\Omega^{c}}|u|^\alpha|v|^\beta\,dx\big)^{2/(\alpha+\beta)}}.$$ We may verify as \cite{AFS} that $$S_{\alpha,\beta}=\big[\big(\frac{\alpha}{\beta}\big)^{\beta/(\alpha+\beta)} +\big(\frac{\beta}{\alpha}\big)^ {\alpha/(\alpha+\beta)}\big]S:=A_{\alpha,\beta}S,$$ where $S$ is the best Sobolev constant defined by $$\ S:=\inf_{u\in D_0^{1,2}(\Omega)\backslash\{0\}} \frac{\int_{\Omega}|\nabla u|^2\,dx}{\big(\int_{\Omega}|u|^{2^*}\,dx\big)^{2/2^*}},$$ which is achieved if and only if $\Omega=\mathbb{R}^N$ by the function $$U(x)=\big[\frac{N(N-2)}{N(N-2)+|x|^2}\big]^{(N-2)/2}.$$ The function $U$ satisfies $$-\Delta U=U^{2^{\ast}-1},\quad\text{in } \mathbb{R}^{N}$$ and $$\int_{\mathbb{R}^N}|\nabla U|^2\,dx=\int_{\mathbb{R}^N}|U|^{2^{\ast}}\,dx=S^{\frac{N}{2}}.$$ Denote $$Q_m=\max_{\partial{\Omega}}Q(x), \quad Q_M=\max_{\Omega^{c}}Q(x), \quad Q_{\infty}=\lim_{|x|\to \infty}Q(x).$$ Suppose $Q_m, Q_M$ and $Q_\infty$ are positive, we set $$S_{\infty}=\min \Big\{ \frac{S_{\alpha,\beta}}{2^{2/N}Q_m^{\frac{N-2}{N}}},\frac{S_{\alpha,\beta}}{Q_M^{\frac{N-2}{N}}}, \frac{S_{\alpha,\beta}}{Q_{\infty}^{\frac{N-2}{N}}}\Big\}.$$ Our main result is as follows. \begin{theorem}\label{thm1.1} If $S_{\lambda,\mu}(\Omega^{c},Q) 2^{\frac{2}{N-2}}Q_m$, we assume \begin{itemize} \item[(Q2)] $Q_ M=Q(y)$ for some $y\in \Omega^{c}$ and for $x$ near $y$, there holds $$\label{e4.1} |Q(y)-Q(x)|=o(|x-y|^{N-2}).$$ \end{itemize} If there is $x\in\Omega^c$ such that $Q(x)\geq Q_\infty$, then $$S_{\infty}=\min \Big\{ \frac{S_{\alpha,\beta}}{2^{2/N}Q_m^{\frac{N-2}{N}}}, \frac{S_{\alpha,\beta}}{Q_M^{\frac{N-2}{N}}}\Big\}.$$ If $Q(x) 0$, $C >0$ such that for $R\to \infty$, \label{e5.1} 0\frac{N^2}{2} for every $\bar y \in \partial B_1(0)\cap B_\delta(\bar z)$. \end{itemize} Under conditions (Q1)-(Q3), $S_\infty$ is well defined. \begin{theorem}\label{thm1.2} The condition $S_{\lambda,\mu}(\Omega^{c},Q) 2^{\frac{2}{N-2}}Q_m$ and {\rm (Q2)} holds; \item[(iii)] $Q_{\infty}>2^{2/(N-2)}Q_m$, $Q(x)h(x')\}$ and $h(x')$ be a $C^1$ function defined on $\{x'\in\mathbb{R}^{N-1}:|x'|<1\}$ with $h$, $\nabla h$ vanishing at 0. For every $u$, $v \in H^1(B_1(0))$ with $\mathop{\rm supp} u, \mathop{\rm supp} v \subset \tilde B$, we have \begin{itemize} \item[(A)] If $h\equiv0$, then $\int_{\tilde{B}}(|\nabla u|^2+|\nabla v|^2)\,dx\geq 2^{-2/N} S_{\alpha,\beta}\Big(\int_{\tilde{B}} |u|^{\alpha}|v|^{\beta}\Big)^{2/2^{\ast}}.$ \item[(B)] For every $\varepsilon>0$ there exists a $\delta>0$ depending only on $\varepsilon$ such that if $|\nabla h|\leq \delta$, then \begin{equation*} \int_{\tilde{B}}(|\nabla u|^2+|\nabla v|^2)\,dx \geq \Big(\frac{S_{\alpha,\beta}}{2^{2/N}}-\varepsilon\Big) \Big(\int_{\tilde{B}}|u|^{\alpha}|v|^{\beta}\Big)^{2/2^{\ast}}. \end{equation*} \end{itemize} \end{lemma} To show the compactness of a $(PS)$ sequence, we need a concentration - compactness lemma. In \cite{PL}, it gave a remarkably characterization of non-compactness of the injection of $W^{1,q}(\Omega)$ into $L^{q^*}(\Omega)$ for $1\leq q0\}$ is at most countable. We can therefore write $F=\{x_j\}_{j\in J}$, $\mu_j:=\mu(x_j), \,j\in J$ so that $\mu\geq\sum_{j\in J}\mu_j\delta_{x_j} + \mu_\infty\delta_\infty.$ If $x_j\in \Omega^c$, for any $\xi\in C_0^\infty(\Omega^{c})\cap L^\infty(\Omega^{c})$, we have \label{e2.6a} \begin{aligned} \int_{\Omega^{c}}|\xi|^{2^{\ast}}\,d \nu &= \lim_{n\to \infty}\int_{\Omega^{c}}|\xi|^{2^{\ast}}|u_n|^{\alpha} |v_n|^{\beta}\,dx \\ &\leq \lim_{n\to \infty}S_{\alpha,\beta}^{-2^{\ast}/2} \Big(\int_{\Omega^{c}}|\nabla(\xi u_n)|^{2}+|\nabla(\xi v_n)|^{2}\,dx\Big)^{2^{\ast}/2}. \end{aligned} Since $u_n\to 0$, $v_n\to 0$ in $L^2_{\rm loc}(\Omega^c)$, we deduce $$\label{e2.6b} \int_{\Omega^{c}}|\xi|^{2^{\ast}}\,d \nu \leq S_{\alpha,\beta}^{-2^{\ast}/2}\Big(\int_{\Omega^{c}}|\xi|^2\,d\mu\Big)^{2^{\ast}/2}.$$ By approximation, for any Borel set $E\in \Omega^{c}$ we have $$\label{e2.6c} \nu(E) \leq S_{\alpha,\beta}^{-2^{\ast}/2} \mu(E)^{2^{\ast}/2}$$ as well as particularly, (\ref{e2.4}) and (\ref{e2.5}) hold. Because (\ref{e2.6c}) implies $\nu\ll \mu$, we have for $E\in \Omega^{c}$ being Borel set that $$\label{e2.6d} \nu(E)= \int_E D_\mu\nu d\mu,$$ where $$\label{e2.6e} D_\mu\nu(x) = \lim_{r\to 0}\frac{\nu(B_r(x))}{\mu(B_r(x))},$$ this limit exists for $\mu$-a.e. $x\in \mathbb{R}^N$. From (\ref{e2.6c}), we have $$\label{e2.6f} D_\mu\nu = 0,\quad \mu-\text{a.e. }x\in \Omega^c\setminus F.$$ Define $\nu_j = D_\mu\nu(x_j)\mu_j$, we see from (\ref{e2.6c})-(\ref{e2.6f}) that (\ref{e2.2}) holds for the case $u=v=0$. If $x_j\in \partial\Omega^c$, by Lemma \ref{lem2.2}, \begin{align*} &\int_{\Omega^{c}\cap B_\varepsilon(x_j)\cap \{x_N>h(x')\}}(|\nabla u|^2+|\nabla v|^2)\,dx\\ &\geq \Big(\frac{S_{\alpha,\beta}}{2^{2/N}}-\varepsilon\Big) \Big(\int_{\Omega^{c}\cap B_\varepsilon(x_j)\cap \{x_N>h(x')\}}|u|^{\alpha}|v|^{\beta}\Big)^{2/2^{\ast}} \end{align*} implying similarly that $$\frac{S_{\alpha,\beta}}{2^{2/N}}\nu_{j}^{2/2^{\ast}} \leq \mu_{j}.$$ Next, in the general case, let $\hat{u}_n= u_n-u$ and $\hat{v}_n = v_n-v$. We may apply above results to $\hat{u}_n$ and $\hat{v}_n$. Moreover, in terms of Lemma \ref{lem2.1}, \begin{gather*} |\nabla \hat u_n|^2 + |\nabla \hat v_n|^2\rightharpoonup \mu + |\nabla u|^2 + |\nabla v|^2, \\ |\nabla \hat u_n|^\alpha|\nabla \hat v_n|^\beta \rightharpoonup \nu + |\nabla u|^\alpha|\nabla v|^\beta \end{gather*} in the sense of measures. The proof is complete. \end{proof} \begin{proof}[Proof of Theorem \ref{thm1.1}] Let $\{u_n,v_n\}$ be a minimizing sequence for $S_{\lambda,\mu}(\Omega^{c},Q)$; that is, $$\int_{\Omega^{c}}(|\nabla u_n|^2+|\nabla v_n|^2+\lambda u_n^2+\mu v_n^2)\,dx \to S_{\lambda,\mu}(\Omega^{c},Q),\quad \int_{\Omega^{c}}Q(x)|u_n|^\alpha|v_n|^\beta\,dx=1.$$ We may assume that $u_n\rightharpoonup u$, $v_n\rightharpoonup v$ in $H^1(\Omega^{c})$. By Lemma \ref{lem2.3}, \begin{gather*} |\nabla u_n|^2+|\nabla v_n|^2 \rightharpoonup \mu \geq |\nabla u|^2+|\nabla v|^2+\sum_{j \in J}\mu_{j}\delta_{x_{j}}+\mu_{\infty}\delta_{\infty},\\ Q(x)|u_n|^\alpha|v_n|^\beta \rightharpoonup \nu=Q(x)|u|^{\alpha}|v|^{\beta}+\sum_{j \in J}\nu_{j}Q(x_j)\delta_{x_{j}}+\nu_{\infty}Q_\infty\delta_{\infty} \end{gather*} and $$\label{eS1} 1=\int_{\Omega^{c}}Q(x)|u|^\alpha|v|^\beta\,dx+\sum_{j\in J} \nu_j Q(x_j)+\nu_{\infty}Q_{\infty}.$$ Therefore, using (\ref{e2.4})-(\ref{e2.6}) we obtain \label{eS} \begin{aligned} &S_{\lambda,\mu}(\Omega^{c},Q)\\ & \geq \int_{\Omega^{c}}(|\nabla u|^2+|\nabla v|^2+\lambda u^2+\mu v^2)\,dx+\sum_{j\in J} \mu_j +\mu_{\infty}\\ &\geq S_{\lambda,\mu}(\Omega^{c},Q) \Big(\int_{\Omega^{c}}Q(x)|u|^\alpha|v|^\beta\,dx\Big)^{2/(\alpha+\beta)}+\sum_{x_j\in \Omega^{c}} S_{\alpha,\beta} \nu_j^{2/(\alpha+\beta)}\\ &\quad + \sum_{x_j\in \partial \Omega} \frac{S_{\alpha,\beta}}{2^{2/N}} \nu_j^{2/(\alpha+\beta)}+S_{\alpha,\beta} \nu_{\infty}^{2/(\alpha+\beta)}\\ &= S_{\lambda,\mu}(\Omega^{c},Q) \Big(\int_{\Omega^{c}}Q(x)|u|^\alpha|v|^\beta\,dx\Big)^{{(N-2)/N}}\\ &\quad +\sum_{x_j\in \Omega^{c}} \frac{S_{\alpha,\beta}}{Q(x_j)^{(N-2)/N}}\left(\nu_j Q(x_j)\right)^{(N-2)/N}\\ &\quad + \sum_{x_j\in \partial \Omega} \frac{S_{\alpha,\beta}}{2^{2/N}Q(x_j)^{(N-2)/N}} (\nu_j Q(x_j))^{(N-2)/N}+\frac{S_{\alpha,\beta}}{Q_{\infty}^{(N-2)/2}} (\nu_{\infty}Q_{\infty})^{(N-2)/N}\\ &\geq S_{\lambda,\mu}(\Omega^{c},Q) \Big(\int_{\Omega^{c}}Q(x)|u|^\alpha|v|^\beta\,dx\Big)^{(N-2)/N} +\sum_{x_j\in \Omega^{c}} \frac{S_{\alpha,\beta}}{Q_M^{(N-2/N)}} (\nu_j Q(x_j))^{(N-2)/N}\\ &\quad +\sum_{x_j\in \partial \Omega} \frac{S_{\alpha,\beta}}{2^{2/N}Q_m^{(N-2)/N}} (\nu_j Q(x_j))^{(N-2)/N}+\frac{S_{\alpha,\beta}}{Q_{\infty}^{(N-2)/2}} (\nu_{\infty}Q_{\infty})^{(N-2)/N}. \end{aligned} Since S_{\lambda,\mu}(\Omega^{c},Q) S_{\lambda,\mu}(\Omega^{c},Q) \Big(\int_{\Omega^{c}}Q(x)|u|^\alpha|v|^\beta\,dx+\nu_j Q(x_j)+ \nu_j Q(x_j)+ \nu_{\infty}Q_{\infty}\Big)^{(N-2)/N}\\ &=S_{\lambda,\mu}(\Omega^{c},Q), \end{align*} which is a contradiction. Hence,\int_{\Omega^{c}}Q(x)|u|^\alpha|v|^\beta\,dx=1$, and then $$\int_{\Omega}(|\nabla u|^2+|\nabla v|^2+\lambda u^2+\mu v^2)\,dx \leq S_{\lambda,\mu}(\Omega^{c},Q).$$ The assertion follows. \end{proof} To verify condition (\ref{e1.3a}), we need some preliminaries. Set $$S(\Omega^{c},\lambda)=\inf_{u\in H^1(\Omega^{c})\backslash\{0\}} \frac{\int_{\Omega^{c}}(|\nabla u|^2+\lambda u^2)\,dx}{(\int_{\Omega^{c}}|u|^{2^{\ast}}\,dx)^{2/2^{\ast}}},$$ The following result was proved in \cite{PW}. \begin{lemma}\label{lem2.4} Assume$\Omega$is a smooth bounded domain in$\mathbb{R}^{N}$such that$\Omega^{c}$has no bounded components. Then we have \begin{itemize} \item[(i)]$S(\Omega^{c},\lambda)$is nondecreasing in$\lambda$; \item[(ii)]$00$, and denote$U_{\varepsilon}=U_{\varepsilon,0}$. \begin{corollary}\label{coro2.1} Assume$\Omega$is a smooth bounded domain in$\mathbb{R}^{N}$such that$\Omega^{c}$has no bounded components. Then there hold \begin{itemize} \item[(i)]$S_{\lambda,\mu}(\Omega^{c},1)$is nondecreasing in$\lambda,\mu$; \item[(ii)]$00$,$S_{\lambda,\mu}(\Omega^{c},1)>0$follows from (i). Now we show that$S_{\lambda,\mu}(\Omega^{c},1)\leq (1/2)^{2/N}S_{\alpha,\beta}$for all$\lambda, \mu\geq0$. Let$y$be a point on$\partial\Omega$. Let$\Phi$be the diffeomorphism from a small ball$B_{\delta}(0)$centered at the origin to a neighborhood$\omega$of$y$so that$\Phi: B_{\delta}^{+}(0)\to \bar{\omega}\cap\Omega^{c}$and$\Phi:B_{\delta}(0)\cap\{y_N=0\}\to \bar{\omega}\cap\partial\Omega^{c}$. We denote$\Psi:=\Phi^{-1}$. Take a radial cut-off function$\eta$such that$\eta(r)\equiv1$for$r\leq\frac{\delta}{2}$,$\eta(r)=0$for$r\geq\delta$and$0\leq\eta\leq1$. Define \begin{equation*} \tilde U_{\varepsilon}(x)= \begin{cases} (\eta U_{\varepsilon})(\Psi(x)) & \text{if } x\in\bar{\omega}\cap\Omega^{c};\\ 0 & \text{if } x\in\Omega^{c}\backslash\bar{\omega}. \end{cases} \end{equation*} It has shown in \cite{PW} that $$\label{e2.14} \frac{\int_{\Omega^{c}}[|\nabla \tilde U_{\varepsilon}(x)|^2+\lambda \tilde U_{\varepsilon}(x)^2]\,dx}{\{\int_{\Omega^{c}}|\tilde U_{\varepsilon}(x)|^{2^{\ast}}\,dx\}^{2/2^{\ast}}} =\frac{S}{2^{2/N}}+A_NH(y)\beta_1(\varepsilon) +O(\beta_2(\varepsilon)),$$ where$A_{N}>0$is a constant and$H(y)$denotes the mean curvature of$\partial\Omega$at$y$, when viewed from inside, and $$\label{e2.15} \beta_1(\tau)= \begin{cases} \tau\log \frac({1}{\tau}) &\text{if } N=3;\\ \tau \quad &\text{if } N\geq 4, \end{cases} \quad \beta_2(\tau)= \begin{cases} \tau & \text{if } N=3;\\ \tau^2 \log (\frac{1}{\tau}) & \text{if } N=4;\\ \tau^2 & \text{if } N\geq 5. \end{cases}$$ Choosing$s$and$t$such that $$\label{e2.16} \frac{2\alpha}{\alpha+\beta}s^{-(\alpha-2)}t^{-\beta}=1 \quad \text{and}\quad \frac{2\beta}{\alpha+\beta}s^{-\alpha}t^{-(\beta-2)}=1;$$ that is,$\frac{s^2}{t^2}=\frac{\beta}{\alpha}, we have $$\label{e2.17} \frac{s^2+t^2}{(s^\alpha t^\beta)^{2/2^{\ast}}}=A_{\alpha,\beta}.$$ By (\ref{e2.14}), \label{e2.18} \begin{aligned} S_{\lambda,\mu}(\Omega^c,1) &\leq \frac{J_{\lambda,\mu}(s\tilde U_{\epsilon}(x),t\tilde U_{\epsilon}(x))}{\big(\int_{\Omega^{c}}|s\tilde U_{\epsilon}(x)|^{\alpha} |t\tilde U_{\epsilon}(x)|^{\beta} \,dx\big)^{2/2^{\ast}}} \\ &\leq \frac{s^2+t^2}{(s^\alpha t^\beta)^{2/2^{\ast}}} \frac{\int_{\Omega^{c}}[|\nabla \tilde U_{\epsilon}(x)|^2+\max\{\lambda,\mu\}\tilde U_{\epsilon}(x)^2]\,dx} {\big(\int_{\Omega^{c}}\tilde U_{\epsilon}(x)^{2^{\ast}}\big)^{2/2^{\ast}}} \\ &\leq A_{\alpha,\beta}\frac{\int_{\Omega^{c}}[|\nabla \tilde U_{\epsilon}(x)|^2+(\lambda+\mu)\tilde U_{\epsilon}(x)^2]\,dx} {\big(\int_{\Omega^{c}}\tilde U_{\epsilon}(x)^{2^{\ast}}\big)^{2/2^{\ast}}} \\ &= \frac{S_{\alpha,\beta}}{2^{2/N}}+B_{N}H(y)\beta_1(\varepsilon) +O(\beta_2(\varepsilon)), \end{aligned} whereB_{N}>0$is a constant. Let$\epsilon\to 0$in \eqref{e2.18}, we obtain (ii). Equation \eqref{e2.18} also implies (iii) since we may assume$H(y)<0$. \end{proof} \begin{lemma}\label{lem2.6} For every$\lambda,\mu \geq 0$we have $$\frac{S_{\lambda,\mu}(\Omega^{c},1)}{Q_M^{\frac{N-2}{N}}} \leq S_{\lambda,\mu}(\Omega^{c},Q)\leq S_{\infty}.$$ If$Q(x)0$there exists$R>0$such that$\Omega \subset B_R(0)$, and$Q(x)\geq Q_{\infty}-\varepsilon$for$x\in \mathbb{R}^{N}\setminus B_R(0)$. Hence, $S_{\lambda,\mu}(\Omega^{c},Q)\leq\frac{\int_{\Omega^{c}}(|\nabla u|^2+|\nabla v|^2+\lambda u^2+\mu v^2)\,dx} {(Q_{\infty}-\varepsilon)^{\frac{N-2}{N}} \big(\int_{\mathbb{R}^{N}\setminus B_R(0)}|u|^{\alpha}|v|^{\beta}\,dx\big)^{2/2^{\ast}}}$ for all$u$,$v\in H^1(\Omega^{c})$. Taking infimum over$u,v\in H_0^1(\mathbb{R}^N\setminus B_R(0))$and noting that$S_{\alpha,\beta}$is independent of$\Omega^c$, since$\epsilon>0$is arbitrary, we obtain from the above inequality that $$S_{\lambda,\mu}(\Omega^{c},Q)\leq \frac{S_{\alpha,\beta}}{Q_{\infty}^{\frac{N-2}{N}}}.$$ Next, if$Q(y)=Q_M$for some$y\in \Omega^{c}$, using$(sU_{\varepsilon,y}, tU_{\varepsilon,y})$as test function in the expression of$S_{\lambda,\mu}(\Omega^{c},Q)$, where$s$and$t$satisfy (\ref{e2.14}), we obtain $$S_{\lambda,\mu}(\Omega^{c},Q)\leq \frac{S_{\alpha,\beta}}{Q_M^{\frac{N-2}{N}}}.$$ Finally, if$y\in\partial\Omega$is such that$Q_m=Q(y)$, we deduce as \eqref{e2.18} that $$S_{\lambda,\mu}(\Omega^{c},Q)\leq \frac{S_{\alpha,\beta}}{2^{2/N}Q_m^{\frac{N-2}{N}}}.$$ The proof is complete. \end{proof} \section{Case$Q_M \leq 2^{2/(N-2)} Q_m $} We will prove Theorem \ref{thm1.2}. \begin{proposition}\label{prop3.1} Assume$Q_M \leq 2^{2/(N-2)} Q_m $and {\rm (Q1)}. Then there holds $$\label{e3.2} S_{\lambda,\mu}(\Omega^{c},Q) < \frac{S_{\alpha,\beta}}{2^{2/N}Q_m^{(N-2)/N}}.$$ \end{proposition} \begin{proof} If$N\geq 5$, let$s$,$t>0be chosen as (\ref{e2.16}). Then $$\label{e3.3} \frac{J_{\lambda,\mu}(sU_{\varepsilon,y},tU_{\varepsilon,y})} {\big(\int_{\Omega^{c}}Q(x)|sU_{\varepsilon,y}|^{\alpha} |tU_{\varepsilon,y}|^{\beta} \,dx\big)^{2/2^{\ast}}} \leq A_{\alpha,\beta}\frac{\int_{\Omega^{c}}[|\nabla U_{\varepsilon,y}|^2+(\lambda+\mu)U_{\varepsilon,y}^2]\,dx} {\big(\int_{\Omega^{c}}Q(x)U_{\varepsilon,y}^{2^{\ast}}\big)^{2/2^{\ast}}}.$$ By the assumption (Q1), $$\label{e3.4} \int_{\Omega^{c}}Q(x)U_{\varepsilon,y}^{2^{\ast}}\,dx =Q_m\int_{\Omega^{c}}U_{\varepsilon,y}^{2^{\ast}}\,dx+o(\varepsilon).$$ Corollary \ref{coro2.1}, (\ref{e3.3}) and (\ref{e3.4}) yield \begin{align*} S_{\lambda,\mu}(\Omega^{c},Q) &\leq\frac{J_{\lambda,\mu}(sU_{\varepsilon,y},tU_{\varepsilon,y})} {\big(\int_{\Omega^{c}}Q(x)|sU_{\varepsilon,y}|^{\alpha} |tU_{\varepsilon,y}|^{\beta} \,dx\big)^{2/2^{\ast}}} \\ &<\frac{SA_{\alpha,\beta}}{2^{2/N}Q_m^{(N-2)/N}}= \frac{S_{\alpha,\beta}}{2^{2/N}Q_m^{(N-2)/N}}. \end{align*} IfN=3, 4$, we replace$U_{\varepsilon,y}$by$U_{\varepsilon,y}\phi_R$, where$\phi_R\in C^1(\mathbb{R}^N)$,$\phi_R(x)=1$for$x\in B_R(0)$,$\phi_R(x)=0$for$x\in \mathbb{R}^N\backslash B_{R+1}(0)$, and$0\leq \phi_R(x)\leq$1 on$\mathbb{R}^N$. Then, (\ref{e3.2}) can be proved in the same way. \end{proof} \section{Case$Q_M > 2^{2/(N-2)} Q_m $} In this section, we show (ii) of Theorem \ref{thm1.2}. \begin{proposition}\label{prop4.1} Suppose$Q_M > 2^{2/(N-2)} Q_m $and {\rm (Q2)}, then there exists$\Lambda>0$such that $$S_{\lambda,\mu}(\Omega^{c},Q) < \frac{S_{\alpha,\beta}}{Q_M^{\frac{N-2}{N}}},$$ for all$0\leq \lambda,\ \mu < \Lambda$. \end{proposition} \begin{proof} First we consider the case$N \geq5$. For any$\delta > 0, using (\ref{e4.1}) we have $$\int_{\Omega^{c}}Q(x)|sU_{\varepsilon,y}|^\alpha|tU_{\varepsilon,y}|^\beta\,dx =\int_{\Omega^{c}}s^{\alpha}t^{\beta}Q_MU_{\varepsilon,y}^{2^{\ast}}\,dx +\int_{\Omega^{c}}s^{\alpha}t^{\beta}(Q(x)-Q_M)U_{\varepsilon,y}^{2^{\ast}}\,dx$$ Since $$\int_{\Omega^{c}}s^{\alpha}t^{\beta}Q_MU_{\varepsilon,y}^{2^{\ast}}\,dx =Q_M\int_{\mathbb{R}^{N}}s^{\alpha}t^{\beta}U_{\varepsilon,y}^{2^{\ast}}\,dx -Q_M\int_{\Omega}s^{\alpha}t^{\beta}U_{\varepsilon,y}^{2^{\ast}}\,dx$$ and \begin{align*} &\int_{\Omega^{c}}s^{\alpha}t^{\beta}(Q(x)-Q_M) U_{\varepsilon,y}^{2^{\ast}}\,dx \\ &= \int_{\Omega^{c}\cap B_{\delta}(y)}s^{\alpha}t^{\beta}(Q(x)-Q_M)U_{\varepsilon,y}^{2^{\ast}}\,dx +\int_{\Omega^{c}\backslash B_{\delta}(y)}s^{\alpha}t^{\beta}(Q(x)-Q_M)U_{\varepsilon,y}^{2^{\ast}}\,dx\\ &= \int_{\Omega^{c}\cap B_{\delta}(y)}s^{\alpha}t^{\beta}o(|x-y|^{N-2})U_{\varepsilon,y}^{2^{\ast}}\,dx+o(\varepsilon^N),\\ \end{align*} we have $$\int_{\Omega^{c}}Q(x)|sU_{\varepsilon,y}|^\alpha|tU_{\varepsilon,y}|^\beta\,dx\\ =s^\alpha t^\beta Q_MK_2+o(\varepsilon^{N-2}),$$ whereK_2=\int_{\mathbb{R}^{N}}U^{2^{\ast}}\,dx$. Since$y \in \Omega^{c}$, there exists a constant$C_1>0$such that $$\int_{\Omega^{c}}|\nabla U_{\varepsilon,y}|^2\,dx= \int_{\mathbb{R}^{N}}|\nabla U_{\varepsilon,y}|^2\,dx-\int_{\Omega}|\nabla U_{\varepsilon,y}|^2\,dx \leq K_1-C_1\varepsilon^{N-2},$$ where$K_1=\int_{\mathbb{R}^{N}}|\nabla U|^2\,dx$, and$\frac{K_1}{(K_2)^{2/2^{\ast}}}=S. Hence, \begin{align*}% \label{e4.4} &\frac{\int_{\Omega^{c}}[|\nabla(sU_{\varepsilon,y})|^2+ |\nabla(tU_{\varepsilon,y})|^2+\lambda(sU_{\varepsilon,y})^2+ \mu(tU_{\varepsilon,y})^2]\,dx} {\big(\int_{\Omega^{c}}s^{\alpha}t^{\beta}Q(x)U_{\varepsilon,y}^{2^{\ast}}\big)^{2/2^{\ast}}}\\ &\leq \frac{s^2+t^2}{(s^\alpha t^\beta)^{2/2^{\ast}}} \frac{\int_{\Omega^{c}}[|\nabla U_{\varepsilon,y}|^2+\max\{\lambda,\mu\}U_{\varepsilon,y}^2]\,dx}{(\int_{\Omega^{c}}Q(x)U_{\varepsilon,y}^{2^{\ast}})^{2/2^{\ast}}}\\ &\leq \frac{s^2+t^2}{(s^\alpha t^\beta)^{2/2^{\ast}}}\frac{K_1-C_1\varepsilon^{N-2}+K_3\max\{\lambda,\mu\}\varepsilon^2} {(Q_MK_2+o(\varepsilon^{N-2}))^{2/2^{\ast}}}\\ &= A_{\alpha,\beta}\big(K_1-C_1\varepsilon^{N-2}+K_3\max\{\lambda,\mu\} \varepsilon^2\big) \big\{(Q_MK_2)^{-(N-2)/N}\\ &\quad -\frac{N-2}{N}(Q_MK_2)^{-(2N+2)/N}o(\varepsilon^{N-2})\big\}\\ &< \frac{A_{\alpha,\beta}S}{Q_M^{(N-2)/N}} =\frac{S_{\alpha,\beta}}{Q_M^{(N-2)/N}} \end{align*} for\varepsilon>0$,$\lambda$and$\mu\geq 0$sufficiently small, where$K_3$is a constant independent of$\varepsilon$. If$N=3, 4$, we replace$U_{\varepsilon,y}$by$U_{\varepsilon,y}\phi_R$, where$\phi_R$is a$C^1$function such that$\phi_R=1$if$x\in B_R(y)$,$\phi_R=0$if$x\in\mathbb{R}^{N}\subset B_{R+1}(y)$with$R>0$large, and we may proceed in the same way. \end{proof} \section{Case$S_{\infty}=\frac{S_{\alpha,\beta}}{Q_{\infty}^{(N-2)/N}}$} In this section, we show (iii) of Theorem \ref{thm1.2}. \begin{proposition}\label{prop5.1} Suppose \eqref{e5.1} holds, then there exists$\Lambda>0$such that for$0\leq\lambda$,$\mu<\Lambda$there holds $$S_{\lambda,\mu}(\Omega^c, Q)<\frac{S_{\alpha,\beta}}{Q_{\infty}^{(N-2)/N}}.$$ \end{proposition} \begin{proof} Let$R>0$be such that$\Omega\subset B_{R/2}(0)$and$K_\delta$denote the cone$K_\delta=\{\tau \bar y; \bar y\in\partial B_1(0)\cap B_\delta(\bar z), \tau>0\}$, then for$s$and$tsatisfying (\ref{e2.16}) we have \label{e5.2} \begin{aligned} &\int_{\Omega^{c}}Q(x)|sU_{\varepsilon,R \bar z}|^{\alpha}|tU_{\varepsilon,R\bar z}|^{\beta}\,dx \\ &= s^{\alpha}t^{\beta}\Big\{\int_{B_{R/2}(0)\setminus\Omega}Q(x)U_{\varepsilon,R \bar z}^{2^{\ast}}\,dx+\int_{\mathbb{R}^N\setminus B_{R/2}(0)}(Q(x)-Q_{\infty})U_{\varepsilon,R \bar z}^{2^{\ast}}\,dx \\ &\quad +\int_{\mathbb{R}^N\setminus B_{R/2}(0)}Q_{\infty}U_{\varepsilon,R \bar z}^{2^{\ast}}\,dx \Big\} \\ &\geq s^{\alpha}t^{\beta}\Big\{\int_{\mathbb{R}^N\setminus(B_{R/2}(0)\cup K_{\delta})}(Q(x)-Q_{\infty})U_{\varepsilon,R\bar z}^{2^{\ast}}\,dx\\ &\quad + \int_{K_{\delta}\setminus B_{R/2}(0)}(Q(x)-Q_{\infty})U_{\varepsilon,R \bar z}^{2^{\ast}}\,dx +\int_{\mathbb{R}^N}Q_{\infty}U_{\varepsilon,R\bar z}^{2^{\ast}}\,dx\\ &\quad -\int_{B_{R/2}(0)}Q_{\infty}U_{\varepsilon,R \bar z}^{2^{\ast}}\,dx \Big\} \\ &= s^{\alpha}t^{\beta}(I_{1}+I_{2}+Q_{\infty}K_{2}+I_{3}). \end{aligned} Since|x-R\bar z|>\delta R$if$x\in B_{2R}(0)\setminus(B_{R/2}(0))\cup K_\delta)$, and$|x-R\bar z|\geq |x|/2$if$x\in \mathbb{R}^N\setminus B_{2R}(0), \label{e5.3} \begin{aligned} I_1 &\geq -Q_{\infty}\int_{\mathbb{R}^N\setminus(B_{R/2}(0)\cup K_{\delta})}\frac{\varepsilon^NC_N^{2^{\ast}}}{(N(N-2)\varepsilon^2+|x-R\bar z|^2)^N}\,dx \\ &\geq -Q_{\infty}\varepsilon^NC_N^{2^{\ast}} \int_{R/2}^{2R}\frac{r^{N-1}}{(\delta R)^{2N}}\,dr-Q_{\infty}\varepsilon^NC_N^{2^{\ast}}\int_{2R}^{\infty} \frac{r^{N-1}}{(\frac{r}{2})^{2N}}\,dr \\ &= -C\frac{\varepsilon^N}{R^N}. \end{aligned} Next, by assumption \eqref{e5.1}, $$\label{e5.4} I_2=\int_{K_{\delta}\setminus B_{R/2}(0)}(Q(x)-Q_{\infty})U_{\varepsilon,R\bar z}^{2^{\ast}}\,dx \geq -\frac{C2^p}{R^p} \int_{\mathbb{R}^N}U_{\varepsilon,R\bar z}^{2^{\ast}}\,dx=-\frac{CK_2}{R^p}.$$ Finally, by the fact that|x-R\bar z|\geq \frac{R}{2}$for$x\in B_{R/2}(0), $$\label{e5.5} I_3=\int_{B_{R/2}(0)}Q_{\infty}U_{\varepsilon,R\bar z}^{2^{\ast}}\,dx\geq -\frac{Q_{\infty}C_N^{2^{\ast}}2^{2N}}{R^{2N}} \int_0^{\frac{R}{2}}r^{N-1}\,dr=-\frac{CK_2}{R^N}$$ Consequently, $$\label{e5.6} \int_{\Omega^{c}}Q(x)|sU_{\varepsilon,R\bar z}|^{\alpha}|tU_{\varepsilon,R\bar z}|^{\beta}\,dx \geq s^{\alpha}t^{\beta} \Big(K_2Q_{\infty}-C\frac{\varepsilon^N}{R^p}-\frac{CK_2}{R^p}\Big).$$ On the other hand, \label{e5.7} \begin{aligned} \int_{\Omega^{c}}|\nabla U_{\varepsilon,R\bar z}|^2\,dx &= \int_{\mathbb{R}^N}|\nabla U_{\varepsilon,R\bar z}|^2\,dx -\int_{\Omega}|\nabla U_{\varepsilon,R\bar z}|^2\,dx \\ &\leq K_1-C_N^2(N-2)\varepsilon^{N-2}\int_{\Omega}\frac{|x-R\bar z|^2}{(\varepsilon^2N(N-2)+|x-R\bar z|^2)^N}\,dx \\ &\leq K_1-\frac{C\varepsilon^{N-2}}{R^{2N-2}}. \end{aligned} Therefore, $$\label{e5.8} \frac{J_{\lambda,\mu}(sU_{\varepsilon,R\bar z},tU_{\varepsilon,R\bar z})}{(\int_{\Omega^{c}}Q(x)|sU_{\varepsilon,R\bar z}|^{\alpha} |tU_{\varepsilon,R\bar z}|^{\beta} \,dx)^{2/2^{\ast}}} \\ \leq A_{\alpha,\beta}\frac {K_1-C\frac{\varepsilon^{N-2}}{R^{2N-2}}+\max\{\lambda,\mu\}C\varepsilon^2} {(K_2Q_{\infty}-C\frac{\varepsilon^N}{R^N}-\frac{CK_2}{R^p})^{(N-2)/N}}.$$ Hence, there exist constantsA>0$,$B>0$,$C>0$and$D>0such that \begin{align*} &\frac{J_{\lambda,\mu}(sU_{\varepsilon,R\bar z},tU_{\varepsilon,R\bar z})}{(\int_{\Omega^{c}}Q(x)|sU_{\varepsilon,R\bar z}|^{\alpha} |tU_{\varepsilon,R\bar z}|^{\beta} \,dx)^{2/2^{\ast}}}\\ &\leq \frac{SA_{\alpha,\beta}}{Q_{\infty}^{(N-2)/N}} -\frac{A\varepsilon^{N-2}}{R^{2N-2}}+\max\{\lambda,\mu\}B\varepsilon^2 +\frac{C\varepsilon^N}{R^N}\frac{D}{R^p}. \end{align*} If\lambda=\mu=0$, we choose$\varepsilon=\varepsilon(R)$such that $$\frac{A\varepsilon^{N-2}}{2R^{2N-2}}=\frac{C\varepsilon^N}{R^N}, \quad\text{i.e., } \frac{1}{\varepsilon^2}=\frac{2CR^{N-2}}{A}.$$ Then, we choose$R>0$so that $$\frac{A\varepsilon^{N-2}}{2R^{2N-2}}=\frac{A}{2R^{2N-2}} \Big(\frac{A}{2CR^{N-2}}\Big)^{(N-2)/2}>\frac{D}{R^p};$$ that is, $$\frac{A^{N/2}}{2^{N/2}R^{N^2/2}}>\frac{D}{R^p}\,,$$ which is possible if$p>\frac{N^2}{2}$. Hence for this choice of$R$we have $$\frac{J_{\lambda,\mu}(sU_{\varepsilon,R\bar z},tU_{\varepsilon,R\bar z})}{(\int_{\Omega^{c}}Q(x)|sU_{\varepsilon,R\bar z}|^{\alpha} |tU_{\varepsilon,R\bar z}|^{\beta} \,dx)^{2/2^{\ast}}}<\frac{S_{\alpha,\beta}}{Q_{\infty}^{(N-2)/N}}.$$ If$\lambda,\mu > 0$. we can similarly choose$R\$ such that the above inequality holds. The proof is complete. \end{proof} \subsection*{Acknowledgements} This work is supported by grants 10571175 and 10631030 from the by National Natural Sciences Foundations of China. \begin{thebibliography}{99} \bibitem{APY} Adimurthi, F. Pacella, S. L. Yadava; Interaction between the geometry of the boundary and positive solutions of a semilinear Neumnann problem with critical nonlinearity. \emph{J.Funct.Anal.} 113 (1993), 318-350. \bibitem{AY} Adimurthi, S. L. Yadava; Positive solution for Neumann problem with critical nonlinearity on boundary. \emph{Comm. Partial Differential Equations} 16 (11) (1991), 1733-1760. \bibitem{AFS} C. O. Alves, D. C. de Morais Filho, M. A. S. Souto; On systems of elliptic equations involving subcritical or critical Sobolev exponents. \emph{Nonlinear Anal.} 42 (2000), 771-787. \bibitem{BN} H. Brezis, L. Nirenberg; Positive solution of nonlinear elliptic equation involving critical Sobolev exponents. \emph{ Comm Pure Appl Math.} 36 (1983), 437-477. \bibitem{BL} H. Brezis, E. Lieb; Relation between pointwise convergence of functions and convergence of functionals. \emph{Proc. Amer. Math. Soc.} 88 (1983), 486-490. \bibitem{C} J. Chabrowski; On multiple solutions of the Neumann problem involving the critical Sobolev exponent. \emph{Colloq. Math.} 101 (2004), 203-220. \bibitem{CR} J. Chabrowski, B. Ruf; On the critical Neumann problem with weight in exterior domains. \emph{Nonlinear Anal.} 54 (2003), 143-163. \bibitem{CY} J. Chabrowski, J. Yang; On the Neumann problem for an elliptic system of equation involving the sobolev expoent. \emph{Colloq. Math.} 90(2001), 1-35. \bibitem{CMW} J. Chabrowski, M. Willem; Least energy solutions of critical Neumann problem with a weight. \emph{Calc. Var. Partial Differential Equations} 15 (2002), 421-431. \bibitem{CW} J. Chabrowski, Z.-Q. Wang; Exterior nonlinear Neumann problem. \emph{NoDEA Nonlinear Differential Equations Appl.} 13 (2007), 683-697. \bibitem{E} L. C. Evans; Weak Convergence Methods for Nonlinear Partial Differential Equations, Reg. Conf. Ser. Math., vol. 74, Amer. Math. Soc., Providence, RI, 1988. \bibitem{PL} P. L. Lions; The concentration compactness principle in the calculus of variations. The limit case (Part 1 and Part 2), {\it Rev. Mat. Iberoamericana} 1 (1985) 145-201, 45-121. \bibitem{P} X.-B. Pan; Condensation of least-energy solutions: the effect of boundary conditions. \emph{Nonlinear Anal.} 24, No. 2 (1995), 195-222. \bibitem{PW}X.-B. Pan, X.-F. Wang; Semilinear Neumann problem in exterior domains. \emph{Nonlinear Anal.} 31, No.7 (1998), 791-821. \bibitem{R} P. Rabinowitz; Minmax Methods in Critical Points Theory with Applications to DifferentialEquations,CBMS Reg. Conf. Ser. Math., vol. 65, Amer. Math. Soc., Providence, RI, 1986. \bibitem{WX} X.-J. Wang; Neumann problems of semilinear elliptic equations involving critical Sobolev exponents. \emph{J. Differential Equations} 93 (1991), 283-310. \bibitem{MW} M. Willem; Minimax Theorems, Birkhauser, Boston, Basel, Berlin, 1996. \end{thebibliography} \end{document}