\documentclass[reqno]{amsart} \usepackage{hyperref} \AtBeginDocument{{\noindent\small \emph{Electronic Journal of Differential Equations}, Vol. 2013 (2013), No. 109, pp. 1--16.\newline ISSN: 1072-6691. URL: http://ejde.math.txstate.edu or http://ejde.math.unt.edu \newline ftp ejde.math.txstate.edu} \thanks{\copyright 2013 Texas State University - San Marcos.} \vspace{9mm}} \begin{document} \title[\hfilneg EJDE-2013/109\hfil Existence of solutions] {Existence of solutions for critical elliptic systems with boundary singularities} \author[J. Yang, L. Wu \hfil EJDE-2013/109\hfilneg] {Jianfu Yang, Linli Wu} % in alphabetical order \address{Jianfu Yang \newline Department of Mathematics, Jiangxi Normal University\\ Nanchang, Jiangxi 330022, China} \email{jfyang\_2000@yahoo.com} \address{Linli Wu \newline Department of Mathematics, Jiangxi Normal University\\ Nanchang, Jiangxi 330022, China} \email{llwujxsd@sina.com} \thanks{Submitted October 16, 2012. Published April 29, 2013.} \subjclass[2000]{35J25, 25J50, 35J57} \keywords{Existence; compactness; critical Hardy-Sobolev exponent; \hfill\break\indent nonlinear system} \begin{abstract} This article concerns the existence of positive solutions to the nonlinear elliptic system involving critical Hardy-Sobolev exponent \begin{gather*} -\Delta u= \frac{2\lambda\alpha}{\alpha+\beta} \frac{u^{\alpha-1} v^\beta}{|\pi(x)|^s}- u^p, \quad \text{in } \Omega,\\ -\Delta v= \frac{2\lambda\beta}{\alpha+\beta} \frac{u^\alpha v^{\beta-1}}{|\pi(x)|^s}- v^p, \quad \text{in } \Omega,\\ u>0,\quad v>0, \quad \text{in } \Omega,\\ u=v=0, \quad \text{on } \partial\Omega, \end{gather*} where $N\geq 4$ and $\Omega$ is a $C^1$ bounded domain in $\mathbb{R}^N$, $01$, $\lambda>0$ and $1\leq p<\frac{N}{N-2}$. Let $\mathcal{P}$ be a linear subspace of $\mathbb{R}^N$ such that $k = \dim_{\mathbb{R}}\mathcal{P}\geq 2$, and $\pi$ be the orthogonal projection on $\mathcal{P}$ with respect to the Euclidean structure. We consider mainly the case when $\mathcal{P}^\bot\cap \Omega =\emptyset$ and $\mathcal{P}^\bot\cap\partial\Omega \neq \emptyset$. We show that there exists $\lambda^*>0$ such that the system above possesses at least one positive solution for $0<\lambda<\lambda^*$ provided that at each point $x\in \mathcal{P}^\bot\cap\partial\Omega$ the principal curvatures of $\partial\Omega$ at $x$ are non-positive, but not all vanish. \end{abstract} \maketitle \numberwithin{equation}{section} \newtheorem{theorem}{Theorem}[section] \newtheorem{lemma}[theorem]{Lemma} \newtheorem{proposition}[theorem]{Proposition} \newtheorem{remark}[theorem]{Remark} \allowdisplaybreaks \section{Introduction} Let $\mathcal{P}$ be a linear subspace of $\mathbb{R}^N$ such that $k = \dim_{\mathbb{R}}\mathcal{P}\geq 2$, and $\pi$ be the orthogonal projection on $\mathcal{P}$ with respect to the Euclidean structure. In this paper, we are concerned with the existence of positive solutions of the following nonlinear elliptic system involving critical Hardy-Sobolev exponent $$\label{eq:1.1} \begin{gathered} -\Delta u= \frac{2\lambda\alpha}{\alpha+\beta} \frac{u^{\alpha-1}v^\beta}{|\pi(x)|^s}- u^p, \quad \text{in } \Omega,\\ -\Delta v= \frac{2\lambda\beta}{\alpha+\beta} \frac{u^\alpha v^{\beta-1}}{|\pi(x)|^s}- v^p, \quad \text{in } \Omega,\\ u>0,v>0, \quad \text{in } \Omega,\\ u=v=0, \quad \text{on } \partial\Omega, \end{gathered}$$ where $N\geq4$ and $\Omega$ is a $C^1$ bounded domain in $\mathbb{R}^N$. We assume in this paper that $01$, $\lambda>0$ and $10\quad \text{in } \Omega;\quad u = 0, \quad \text{on } \partial\Omega, \] where$\lambda>0$,$10\quad \text{in } \Omega;\quad u = 0, \quad \text{on } \partial\Omega, using the blow-up method, Hsai et al \cite{HLW} prove that problem \eqref{eq:1.3} possesses at least one positive solution. In \cite{M}, the Hardy-Sobolev inequality $$\label{eq:1.4} \mu_{2^*(s),\mathcal{P}}\Big(\int_{\mathbb{R}^N} \frac{|u|^{2^\ast(s)}}{|\pi(x)|^s}\,dx\Big)^{\frac{2^\ast(s)}2} \leq\int_{\mathbb{R}^N}|\nabla u|^2\,dx,\ \forall u\in D^{1,2}(\mathbb{R}^N)$$ was established for all $u\in D^{1,2}(\mathbb{R}^N)$. Then, it was shown in \cite{FMS} that $\mu_{2^*(s),\mathcal{P}}(\Omega) \geq \mu_{2^*(s),\mathcal{P}}(\mathbb{R}^N)>0$ for all smooth domain $\Omega\subset \mathbb{R}^N$. The attainability of $\mu_{2^*(s),\mathcal{P}}(\Omega)$ depends on the position between $\mathcal{P}$ and $\Omega$, this was discussed in \cite{GR2}. In this article, we study the existence of positive solutions of \eqref{eq:1.1}. In \cite{HY}, positive solutions of problem \eqref{eq:1.1} were found in non-contractible domains if $\lambda=0, k = N$ and $s=0$. In \cite{TZ}, the existence of sign-changing solutions was obtained for \eqref{eq:1.1} with $k = N$ and $s=0$. For further results for the system we refer the references in \cite{HY} and \cite{TZ}. Equation \eqref{eq:1.1} involves the Hardy type potential, that is $s\neq 0$ and possibly, $k\leq N$, and the lower order terms are negative, which will push the energy up. We will prove that \eqref{eq:1.1} possesses at least one positive solution by the blow up argument. The limiting problem after blowing up is as follows. $$\label{eq:1.4b} \begin{gathered} -\Delta u= \frac{2\alpha}{\alpha+\beta}\frac{u^{\alpha-1}v^\beta}{|\pi(x)|^s}, \quad \text{in } \mathbb{R}^N_+,\\ -\Delta v= \frac{2\beta}{\alpha+\beta}\frac{u^\alpha v^{\beta-1}}{|\pi(x)|^s}, \quad \text{in } \mathbb{R}^N_+,\\ u>0,\quad v>0, \quad \text{in } \mathbb{R}^N_+,\\ u=v=0, \quad \text{on } \partial\mathbb{R}^N_+. \end{gathered}$$ Denote $$\label{eq:1.5} \mu_{\alpha,\beta,\mathcal{P}}(\Omega)=\inf_{(u,v)\in (H^1_0(\Omega))^2\backslash\{0\}} \frac{\int_{\Omega}(|\nabla u|^2+|\nabla v|^2)dx} {\big(\int_{\Omega}\frac{u^\alpha v^\beta}{|\pi(x)|^s}dx\big)^{\frac{2}{2^*(s)}}}$$ for a domain $\Omega\subset \mathbb{R}^N$. The solution of \eqref{eq:1.4} will be obtained by showing that $\mu_{\alpha,\beta,\mathcal{P}}(\mathbb{R}^N_+)$ is achieved. The minimizer of $\mu_{\alpha,\beta,\mathcal{P}}(\mathbb{R}^N_+)$ is the least energy solution of \eqref{eq:1.4} up to a multiplicative constant. It was observed in \cite{AFS} that $\mu_{\alpha,\beta,\mathcal{P}}(\Omega)$ and $\mu_{\alpha+\beta,\mathcal{P}}(\Omega)$ are closely related. Precisely, we have $$\label{eq:1.5a} \mu_{\alpha,\beta,\mathcal{P}}(\Omega) =\big[(\frac{\alpha}{\beta})^{\frac{\beta}{\alpha+\beta}} +(\frac{\alpha}{\beta})^{\frac{-\alpha}{\alpha+\beta}}\big] \mu_{\alpha+\beta,\mathcal{P}}(\Omega)$$ for $\alpha +\beta\leq 2^*$. Moreover, if $w_0$ realizes $\mu_{\alpha+\beta,s}(\Omega)$, then $u_0=A w_0$ and $v_0=B w_0$ realizes $\mu_{\alpha,\beta,\mathcal{P}}(\Omega)$ for any real constants $A$ and $B$ such that $\frac{A}{B}=\sqrt{\frac{\alpha}{\beta}}$. In the case $\Omega = \mathbb{R}^N_+$, it was proved in \cite{GR2} that $\mu_{2^*(s),\mathcal{P}}(\mathbb{R}^N_+)$ is achieved by a function $u\in H_0^1(\mathbb{R}^N_+)$ provided that $\mathcal{P}^\bot\subset\partial\mathbb{R}^N_+$. This implies that $\mu_{\alpha,\beta,\mathcal{P}}(\mathbb{R}^N_+)$ is achieved if $\alpha+\beta=2^*(s)$ and $\mathcal{P}^\bot\subset\partial\mathbb{R}^N_+$. Hence, there exists a least energy entire solution of \eqref{eq:1.4} in this case. To deal with \eqref{eq:1.1}, we consider a related subcritical problem, and obtain a sequence of solutions of the subcritical problems. Then, we analyse the blow up behavior of the approximating sequence. Since the coefficient of lower order terms are negative, the energy of the corresponding functional becomes larger, it makes difficult to find the upper compact bound. Our main result is as follows. \begin{theorem}\label{thm:1.1} Let $\Omega$ be a smooth bounded domain of $\mathbb{R}^N$, $N\geq 3$, and let $\mathcal{P}$ be a linear subspace of $\mathbb{R}^N$ such that $k = \dim_{\mathbb{R}}\mathcal{P}\geq 2$. Suppose $s\in (0,2)$, then we have \begin{itemize} \item[(i)] If $\mathcal{P}^\bot\cap\Omega \neq \emptyset$, problem \eqref{eq:1.1} possesses at least one positive solution provided that $s = 1$. \item[(ii)] If $\mathcal{P}^\bot\cap\bar \Omega =\emptyset$, problem \eqref{eq:1.1} possesses at least one positive solution. \item[(iii)] If $\mathcal{P}^\bot\cap \Omega =\emptyset$ and $\mathcal{P}^\bot\cap\partial\Omega \neq \emptyset$, there exists $\lambda^*>0$ such that for $0<\lambda <\lambda^*$ problem \eqref{eq:1.1} possesses at least one positive solution provided that at each point $x\in \mathcal{P}^\bot\cap\partial\Omega$ the principle curvatures of $\partial\Omega$ at $x$ are non-positive, but not all vanish. \end{itemize} \end{theorem} In section 2, we find a suitable upper bound for the mountain pass level and prove (i) and (ii) of Theorem \ref{thm:1.1}, then using this bound and the blow-up argument, we prove $(iii)$ of Theorem \ref{thm:1.1} in section 3. \section {Preliminaries} We recall that $$\label{eq:2.1} \mu_{2^*(s), \mathcal{P}}(\Omega)=\inf_{u\in H^1_0(\Omega)\backslash\{0\}} \frac{\int_{\Omega}|\nabla u|^2\,dx}{\big(\int_{\Omega} \frac{u^{2^*(s)}}{|\pi(x)|^s}\,dx\big)^{\frac{2}{2^*(s)}}},$$ where $2^*(s) = \frac{2(N-2)}{N-2}$, $s\in (0,2)$ and $\pi$ is the orthogonal projection on $\mathcal{P}$ with respect to the Euclidean structure. The attainability of $\mu_{2^*(s), \mathcal{P}}(\Omega)$ depends on the position between $\Omega$ and $\mathcal{P}$. Actually, it was proved in \cite{GR2} that if $\mathcal{P}^\bot\cap\Omega\neq \emptyset$, $\mu_{2^*(s), \mathcal{P}}(\Omega) = \mu_{2^*(s), \mathcal{P}}(\mathbb{R}^N)$. Therefore, $\mu_{2^*(s), \mathcal{P}}(\Omega)$ is not achieved. If $\mathcal{P}^\bot\cap\bar\Omega=\emptyset$, the problem becomes subcritical without singularities, thus $\mu_{2^*(s), \mathcal{P}}(\Omega)$ is attained. Finally, if $\mathcal{P}^\bot\cap\Omega=\emptyset$ and $\mathcal{P}^\bot\cap\partial\Omega\neq \emptyset$, $\mu_{2^*(s), \mathcal{P}}(\Omega)$ is achieved provided that the principle curvatures of $\partial\Omega$ at $x\in\mathcal{P}^\bot\cap\partial\Omega$ are non-positive, and do not all vanish. Furthermore, the following lemma was also shown in \cite{GR2}. \begin{lemma}\label{lem:2.1} There exists a minimizer $u\in C^1(\mathbb{\bar R}^N_+)\cap H^1_0(\mathbb{R}^N_+)$ of $\mu_{2^*(s), \mathcal{P}}(\mathbb{R}^N_+)$ such that $$\label{eq:2.2} \begin{gathered} -\Delta u= \frac{u^{2^*(s)-1}}{|\pi(x)|^s}\quad\text{in } \mathbb{R}^N_+, \\ u>0\quad\text{in }\mathbb{R}^N_+,\quad u=0\quad \text{on } \partial\mathbb{R}^N_+ \end{gathered}$$ in $\mathcal{D}'(\mathbb{R}^N_+)$ satisfying $\int_{\mathbb{R}^N_+}|\nabla u|^2\,dx = \mu_{2^*(s), \mathcal{P}}(\mathbb{R}^N_+)^{\frac{2^*(s)}{2^*(s)-2}}$, provided that $2\leq k\leq N-1$ and $\mathcal{P}^\bot\subset\partial\mathbb{R}^N_+$. \end{lemma} Let $u\in C^1(\mathbb{\bar R}^N_+)\cap H^1_0(\mathbb{R}^N_+)$ be the minimizer of $\mu_{2^*(s), \mathcal{P}}(\mathbb{R}^N_+)$. We have the following estimates. \begin{lemma}\label{lem:2.2} There exists $C>0$ such that $$\label{eq:2.3} |u(x)|\leq C(1+|x|)^{1-N},\quad |\nabla u(x)|\leq C(1+|x|)^{-N}$$ for $x\in \mathbb{R}^N_+$. \end{lemma} \begin{proof} Let $u^*(x) = |x|^{-(N-2)}u(\frac x{|x|^2}),\quad x\in \mathbb{R}^N_+,$ be the Kelvin transformation of $u$. Since $u\in D^{1,2}_0(\mathbb{R}^N_+)$, we may verify that the $u^*$ also satisfies equation \eqref{eq:2.2}, and both $\int_{\mathbb{R}^N_+}|\nabla u^*|^{2}\,dx$ and $\int_{\mathbb{R}^N_+}\frac{|u^*|^{2^*(s)}}{|\pi(x)|^s}\,dx$ are finite. Next, by a regularity result in \cite{GR2}, $u^*\in C^1(\mathbb{\bar R}^N_+)$. It implies in a standard way that \eqref{eq:2.3} holds. The proof is complete. \end{proof} By \eqref{eq:1.5a}, we see that $\mu_{\alpha,\beta,\mathcal{P}}(\Omega)$ and $\mu_{2^*(s), \mathcal{P}}(\Omega)$ are closely related if $\alpha + \beta = 2^*(s)$, which and Lemma \ref{lem:2.2} allow us to state the following result. \begin{proposition}\label{prop:2.1} Suppose $\alpha + \beta = 2^*(s)$. Then \begin{itemize} \item[(i)] $\mu_{\alpha,\beta,\mathcal{P}}(\Omega) = \mu_{\alpha,\beta,\mathcal{P}}(\mathbb{R}^N)$ if $\mathcal{P}^\bot\cap\Omega\neq \emptyset$, and $\mu_{\alpha,\beta,\mathcal{P}}(\Omega)$ is not achieved. \item[(ii)] If $\mathcal{P}^\bot\cap\bar\Omega=\emptyset$, $\mu_{\alpha,\beta, \mathcal{P}}(\Omega)$ is attained. \item[(iii)] If $\mathcal{P}^\bot\cap\Omega=\emptyset$ and $\mathcal{P}^\bot\cap\partial\Omega\neq \emptyset$, $\mu_{\alpha,\beta, \mathcal{P}}(\Omega)$ is achieved provided that the principle curvatures of $\partial\Omega$ at $x\in\mathcal{P}^\bot\cap\partial\Omega$ are non-positive, and do not all vanish. \end{itemize} Moreover, all components of the minimizer of $\mu_{\alpha,\beta,\mathcal{P}}(\mathbb{R}_+^N)$ satisfy the decaying law in \eqref{eq:2.3}. \end{proposition} \begin{proof}[Proof of (i) and (ii) of Theorem \ref{thm:1.1}] In the case (i), problem \eqref{eq:1.1} is a critical problem with singularities in $\Omega$. The existence of positive solution of the problem can be proved by the mountain pass theorem as \cite{AWZ}, \cite{BT}. In the case $(ii)$, problem \eqref{eq:1.1} is a subcritical problem without singularities, the result is readily obtained. \end{proof} In the rest of the paper, we only consider the case (iii); that is, we assume that $\mathcal{P}^\bot\cap\Omega=\emptyset$ and $\mathcal{P}^\bot\cap\partial\Omega\neq \emptyset$. In the following, we establish the upper bound for the mountain pass level. We recall that by \cite{GR}, $\mu_{2^*(s),\mathcal{P}}(\mathbb{R}^N_+)$ is achieved by a function $u\in H_0^1(\mathbb{R}^N_+)$ if $\mathcal{P}^\bot\subset\partial\mathbb{R}^N_+$. This implies that $\mu_{\alpha,\beta,\mathcal{P}}(\mathbb{R}^N_+)$ is achieved if $\alpha+\beta=2^*(s)$. Hence, there exists a least energy entire solution of system \eqref{eq:1.4}. The energy functional for \eqref{eq:1.1} is $$I_\lambda(u,v)=\int_{\Omega}(\frac{1}{2}|\nabla u|^2+\frac{1}{2}|\nabla v|^2 -\frac{2\lambda}{2^*(s)}\frac{u^{\alpha}v^{\beta}}{|\pi(x)|^s} +\frac{1}{p+1}u^{p+1}+\frac{1}{p+1}v^{p+1})\,dx\,,$$ which is well defined on $H^1_0(\Omega)$. It is well known that to find positive solutions of problem \eqref{eq:1.1} is equivalent to find nonzero critical points of functional $I_{\lambda}$ in $H_0^1(\Omega)\times H_0^1(\Omega)$. Now, we bound the mountain pass level for the functional $I_{\lambda}$. \begin{lemma}\label{lem:2.3} Suppose that $\Omega$ is a $C^1$ bounded domain in $\mathbb{R}^N$ with $\mathcal{P}^\bot\cap\Omega=\emptyset$ and $\mathcal{P}^\bot\cap\partial\Omega\neq \emptyset$. There exist $\lambda^*>0$ and nonnegative functions $u_0$ and $v_0$ in $H_0^1(\Omega)\setminus\{0\}$ such that for $0<\lambda<\lambda^*$ and $1\leq p<\frac{N}{N-2}$, we have $I_\lambda(u_0,v_0)<0$ and $$\max_{0\leq t\leq 1}I_\lambda(tu_0,tv_0) < (2\lambda)^{\frac{-2}{2^*(s)-2}}(\frac{1}{2}-\frac{1}{2^*(s)}) \mu_{\alpha,\beta,\mathcal{P}}(\mathbb{R}^N_+)^{\frac{2^*(s)}{2^*(s)-2}}.$$ provided that the principle curvatures of $\partial\Omega$ at $x\in\mathcal{P}^\bot\cap\partial\Omega$ are non-positive, and do not all vanish. \end{lemma} \begin{proof} Let $(u,v)$ be the minimizer of $\mu_{\alpha,\beta,\mathcal{P}}(\mathbb{R}^N_+)$, such that $$\int_{\mathbb{R}^N_+}|\nabla u|^2\,dx+\int_{\mathbb{R}^N_+}|\nabla v|^2\,dx =\mu_{\alpha,\beta,\mathcal{P}}(\mathbb{R}^N_+), \quad\quad \int_{\mathbb{R}^N_+}\frac{u^\alpha v^\beta}{|\pi(x)|^s}\,dx=1.$$ Then, there exist $A$, $B\in\mathbb{R}$ such that $u = A w$, $v = B w$ with $\frac{A}{B}=\sqrt{\frac{\alpha}{\beta}}$, where $w$ is a minimizer of $\mu_{2^*(s), \mathcal{P}}(\mathbb{R}^N_+)$. Since $$|w(x)|\leq C(1+|x|)^{1-N}, \quad\quad |\nabla w(x)|\leq C(1+|x|)^{-N},$$ we obtain \begin{gather}\label{eq:2.4} |u(x)|\leq C(1+|x|)^{1-N},\quad |\nabla u(x)|\leq C(1+|x|)^{-N},\\ \label{eq:2.5} |v(x)|\leq C(1+|x|)^{1-N},\quad |\nabla v(x)|\leq C(1+|x|)^{-N}. \end{gather} Moreover, $(u,v)$ satisfies $$\label{eq:2.6} -\Delta u= \frac{\alpha}{\alpha+\beta}\mu_{\alpha,\beta,\mathcal{P}} (\mathbb{R}^N_+)\frac{u^{\alpha-1}v^\beta}{|\pi(x)|^s}, \quad -\Delta v= \frac{\beta}{\alpha+\beta}\mu_{\alpha,\beta,\mathcal{P}} (\mathbb{R}^N_+)\frac{u^\alpha v^{\beta-1}}{|\pi(x)|^s}, \quad \text{in } \mathbb{R}^N_+.$$ Let $x_0\in \mathcal{P}^\bot\cap\partial\Omega$. Since $\mathcal{P}^\bot\cap\Omega = \emptyset$, we have $\mathcal{P}^\bot\subset T_{x_0}\partial\Omega$, where $T_{x_0}\partial\Omega$ is the tangent space of the smooth manifold $\partial\Omega$ at $x_0$. Thus, $(T_{x_0}\partial\Omega)^\bot\subset \mathcal{P}$. Denote $k = \dim_{\mathbb{R}}\mathcal{P}$, we choose a direct orthonormal basis $(e_1,\dots,e_N)$ of $\mathbb{R}^N$ such that $e_1 = n_{x_0}$ is the outward normal of $\partial\Omega$ at $x_0$, $\operatorname{span}\{e_1,\dots,e_k\} = \mathcal{P}$ and $\operatorname{span}\{e_{k+1},\dots,e_N\} = \mathcal{P}^\bot$. For any $x\in \mathbb{R}^N$, we denote $x = (x_1, y,z)$, where $x_1\in \mathbb{R}$, $y\in \operatorname{span}\{e_2,\dots,e_k\}$ and $z\in \mathcal{P}^\bot$. Since $\partial\Omega$ is smooth, there exist open sets $U, V$ of $\mathbb{R}^N$ such that $0\in U$ and $x_0\in V$, and there exist $\varphi\in C^\infty(U,V)$ and $\varphi_{0}\in C^\infty(U')$ with $U'=\{(y,z): \text{there exists$x_1\in \mathbb{R}$such that } (x_1,y,z)\in U\}$ such that \begin{itemize} \item[(i)] $\varphi: U\to V$ is a $C^\infty$ diffeomorphism, $\varphi(0) = x_0$; \item[(ii)] $\varphi (U\cap\{x_1>0\}) = \varphi(U)\cap\Omega$ and $\varphi(U\cap\{x_1=0\})= \varphi(U)\cap\partial\Omega$; \item[(iii)] $\varphi_0(0) = 0$ and $\nabla\varphi_0(0) = 0$; \item[(iv)] $\varphi(x_1, y, z) = (x_1-\varphi_0(y,z),y,z) + x_0$ for all $(x_1,y,z)\in U$. \end{itemize} Denote $\psi = \varphi^{-1}$. We choose a small positive number $r_0$ so that there exist neighborhoods $V$ and $\tilde{V}$ of $x_0$, such that $\psi(V)=B_{r_0}(0)$, $\psi(V\cap\Omega)=B^+_{r_0}(0)$, $\psi(\tilde{V})=B_{\frac{r_0}{2}}(0)$, $\psi(\tilde{V}\cap\Omega)=B^+_{\frac{r_0}{2}}(0)$. For $\varepsilon>0$, we define $$\tilde u_\varepsilon(x)=\varepsilon^{-\frac{N-2}{2}}\eta(x)u \big(\frac{\psi(x)}{\varepsilon}\big):= \eta(x)u_\varepsilon, \quad\ \tilde v_\varepsilon(x)=\varepsilon^{-\frac{N-2}{2}}\eta(x)v \big(\frac{\psi(x)}{\varepsilon}\big):= \eta(x)v_\varepsilon,$$ where $\eta\in C_0^\infty(V)$ is a positive cut-off function with $\eta\equiv1$ in $\tilde{V}$. In what follows, we estimate each term in $I_\lambda(t \tilde u_\varepsilon,t \tilde v_\varepsilon)$. We have $\int_{\Omega}|\nabla \tilde u_\varepsilon|^2 \,dx = \int_{\Omega}(|\nabla\eta|^2u^2_\varepsilon+\eta^2|\nabla u_\varepsilon|^2 +2\nabla\eta\nabla u_\varepsilon\eta u_\varepsilon)\,dx.$ Since $\int_{\Omega}\eta u_\varepsilon\nabla\eta\nabla u_\varepsilon\,dx = -\int_{\Omega}|\nabla\eta|^2 u^2_\varepsilon\,dx -\int_{\Omega}\nabla\eta\eta\nabla u_\varepsilon u_\varepsilon\,dx -\int_{\Omega}\eta(\Delta\eta)|u_\varepsilon|^2\,dx,$ we obtain $$\int_{\Omega}|\nabla \tilde u_\varepsilon|^2\,dx = \int_{\Omega\cap U}\eta^2|\nabla u_\varepsilon|^2\,dx -\int_{\Omega\cap U}\eta(\Delta\eta)|u_\varepsilon|^2\,dx.$$ By the change of the variable $X =\frac{\psi(x)}{\varepsilon}\in B_{r_0/\varepsilon}^+(0)$ and \eqref{eq:2.4}, we obtain \begin{align*} \big|\int_{\Omega\cap U}\eta(\Delta\eta)u_\varepsilon^2\,dx\big| &\leq C\varepsilon^2 \int_{B^+_{r_0/\varepsilon}(0) \setminus B^+_{\frac {r_0}{2\varepsilon}}(0)} \eta(\varphi(\varepsilon X))\big|\Delta \eta(\varphi(\varepsilon X)) \big| u^2(X)\,dX\\ &=O(\varepsilon^2) \end{align*} and since $\nabla_x u_\varepsilon(x) = \varepsilon^{-\frac N 2}\nabla_X u(\frac{\psi(x)}{\varepsilon})\nabla_x\psi(x)$, we deduce for $X'=(X_2,\dots, X_N)$ and $\nabla' = (\partial_{X_2},\dots,\partial_{X_N})$ that \begin{align*} & \int_{\Omega\cap U}\eta^2|\nabla u_\varepsilon|^2\,dx\\ &\leq \int_{\mathbb{R}^N_+}|\nabla u|^2\,dX -2\int_{B_{r_0/\varepsilon}^+}\eta^2(\varphi(\varepsilon X)) \partial_1 u(X)\nabla'u(X)(\nabla'\varphi_0)(\varepsilon X')\,dX \\ &\quad + \int_{B_{r_0/\varepsilon}^+}\eta^2(\varphi(\varepsilon X)) |\nabla' u(X)|^2|(\nabla'\varphi_0)(\varepsilon X')|^2\,dX = I_1 +I_2+I_3. \end{align*} Using that $$|\nabla'\varphi_0(X')|=O(|X'|),\quad \varphi_0(X') =\sum_{i=2}^{N}\alpha_iX_i^2+o(1)(|X'|^2)$$ and \eqref{eq:2.4}, we have $I_3\leq C\int_{\mathbb{R}^N}(1+|X|)^{-2N}|\varepsilon X|^2\,dX = O(\varepsilon^2).$ Integrating by parts, we infer that \begin{align*} I_2 &= \frac{4}{\varepsilon}\int_{B_{r_0/\varepsilon}^+} \eta(\varphi(\varepsilon X))\nabla'\eta(\varphi(\varepsilon X)) \partial_1 u(X)\nabla'u(X)\varphi_0(\varepsilon X')\,dX \\ &\quad+ \frac{2}{\varepsilon}\int_{B_{r_0/\varepsilon}^+} \eta^2(\varphi(\varepsilon X)) \nabla'\partial_1 u(X)\nabla'u(X)\varphi_0(\varepsilon X')\,dX \\ &\quad+ \frac{2}{\varepsilon}\int_{B_{r_0/\varepsilon}^+} \eta^2(\varphi(\varepsilon X)) \partial_1 u(X)\sum_{i=2}^{N}\partial_{ii}u(X)\varphi_0(\varepsilon X')\,dX = I_{21}+I_{22}+I_{23}. \end{align*} By \eqref{eq:2.4}, $|I_{21}|\leq C\varepsilon^2\int_{B_{r_0/\varepsilon}^+(0) \setminus B_{\frac{r_0}{2\varepsilon}}(0)}(1+|X|)^{-2N}|X|^2\,dX \leq C\varepsilon^N.$ In the same way, $I_{22}=O(\varepsilon^N)$. By equation \eqref{eq:2.6}, $$\sum_{i=2}^{N}\partial_{ii}u(X)= \Delta u-\partial_{11}u(X) = -\frac{\alpha\lambda}{\alpha+\beta}\mu_{\alpha,\beta,s}(\mathbb{R}^N_+) \frac{u^{\alpha-1}v^\beta}{|\pi(X)|^s}-\partial_{11}u(X).$$ Therefore, \begin{align} I_{23} &=-\frac{2}{\varepsilon}\int_{B_{r_0/\varepsilon}^+} \eta^2(\varphi(\varepsilon X)) \partial_1 u(X)\frac{\alpha\lambda}{\alpha+\beta}\mu_{\alpha,\beta,s} (\mathbb{R}^N_+)\frac{u^{\alpha-1}v^\beta}{|\pi(X)|^s}\varphi_0(\varepsilon X')\,dX \\ &\quad - \frac{2}{\varepsilon}\int_{B_{r_0/\varepsilon}^+} \eta^2(\varphi(\varepsilon X)) \partial_1 u(X)\partial_{11}u(X)\varphi_0(\varepsilon X')\,dX:=F_1+F_2. \end{align} Since $u = Aw$, $F_1 = -\frac{C_0}{\varepsilon}\int_{B_{r_0/\varepsilon}^+} \eta^2(\varphi(\varepsilon X)) \frac{\partial_1w(X)^{2^*(s)}}{|\pi(X)|^s}\varphi_0(\varepsilon X')\,dX,$ where $C_0 =\frac{2\alpha\lambda}{(2^*(s))^2}\mu_{\alpha,\beta,s} (\mathbb{R}^N_+)A^\alpha B^\beta$. Integrating by parts, we obtain \begin{align*} F_1 &= C_0\int_{B_{r_0/\varepsilon}^+}\frac{2\eta(\varphi(\varepsilon X)) \partial_1 \eta(\varphi(\varepsilon X))\varphi_0 (\varepsilon X')}{|\pi(X)|^s}w^{2^*(s)}\,dX \\ &\quad- \frac{C_0s}{\varepsilon}\int_{B_{r_0/\varepsilon}^+} \frac{\eta^2(\varphi(\varepsilon X))\varphi_0 (\varepsilon X')X_1}{|\pi(X)|^{s+2}}w^{2^*(s)}\,dX\\ &=F_{11}+ F_{12}. \end{align*} We may verify as above that $F_{11}=O(\varepsilon^{\frac{N^2-N-Ns+2}{N-2}})$. Now, we estimate $F_2$. Integrating by parts, we deduce \begin{align*} F_2 &= \frac{1}{\varepsilon}\int_{B_{r_0/\varepsilon}^+} \partial_1[\eta^2(\varphi(\varepsilon X))\varphi_0(\varepsilon X')] (\partial_1 u)^2\,dX \\ &\quad+ \frac{1}{\varepsilon}\int_{B_{r_0/\varepsilon}^+\cap\{X_1 = 0\}} \eta^2(\varphi(\varepsilon X))\varphi_0(\varepsilon X') (\partial_1 u)^2\nu^N\,dS_X \\ &= \frac{1}{\varepsilon}\int_{B_{r_0/\varepsilon}^+} 2\eta(\varphi(\varepsilon X))\partial_1[\eta(\varphi(\varepsilon X))] \varphi_0(\varepsilon X') (\partial_1 u)^2\,dX \\ &\quad + \frac{1}{\varepsilon}\int_{B_{r_0/\varepsilon}^+ \cap\partial\mathbb{R}^N_+}\eta^2(\varphi(\varepsilon X)) \varphi_0(\varepsilon X')(\partial_1 u)^2\,dS_X\\ &=F_{21}+F_{22}. \end{align*} It can be shown that $F_{21} = O(\varepsilon^{N-1})$. Hence, $I_2 = F_{12}+ F_{22}+ O(\varepsilon^{N-1}).$ Since $\eta(\varphi(\varepsilon X))\equiv1$ in $B_{\frac{r_0}{2\varepsilon}}^+$, we have \begin{align*} F_{12} &= - \frac{C_0s}{\varepsilon}\int_{B_{r_0/\varepsilon}^+ \setminus B_{\frac{r_0}{2\varepsilon}}^+}\frac{\eta^2 (\varphi(\varepsilon X))\varphi_0(\varepsilon X')X_1}{|\pi(X)|^{s+2}}w^{2^*(s)}\,dX \\ &\quad - \frac{C_0s}{\varepsilon}\int_{B_{\frac{r_0}{2\varepsilon}}^+} \frac{\varphi_0(\varepsilon X')X_1}{|\pi(X)|^{s+2}}w^{2^*(s)}\,dX = J_{1}+J_{2}. \end{align*} We have \begin{align*} J_{1} &\leq C\varepsilon\int_{B_{r_0/\varepsilon}^+\setminus B_{\frac{r_0}{2\varepsilon}}^+}\frac{|\pi(X)|^3(1+|X|)^{(1-N)2^*(s)}}{|\pi(X)| ^{s+2}}\,dX\\ &\leq C\varepsilon\Big(\int_{(B_{r_0/\varepsilon}^+ \setminus B_{\frac{r_0}{2\varepsilon}}^+)\cap \mathbb{R}^{N-k}} \frac 1{|x|^{\frac {2^*(s)(N-1)}2}}\,dx\Big)\Big(\int_{(B_{r_0/\varepsilon}^+ \setminus B_{\frac{r_0}{2\varepsilon}}^+)\cap \mathbb{R}^{k}} \frac {|x|^{1-s}}{|x|^{\frac {2^*(s)(N-1)}2}}\,dx\Big)\\ &\leq C\varepsilon^{\frac{N(N-s)}{N-2}}. \end{align*} In the same way, \begin{align*} J_{2} &=- \frac{C_0s}{\varepsilon}\int_{\mathbb{R}^N_+}\frac{\varphi_0(\varepsilon X')X_1}{|\pi(X)|^{s+2}}w^{2^*(s)}\,dX - \frac{C_0s}{\varepsilon}\int_{\mathbb{R}^N_+\setminus B_{r_0/\varepsilon}^+} \frac{\varphi_0(\varepsilon X')X_1}{|\pi(X)|^{s+2}}w(X)^{2^*(s)}\,dX \\ &=- \frac{C_0s}{\varepsilon}\int_{\mathbb{R}^N_+}\frac{\varphi_0(\varepsilon X') X_1}{|\pi(X)|^{s+2}}w^{2^*(s)}\,dX + O(\varepsilon^{\frac{N(N-s)}{N-2}}) \\ &= - \varepsilon C_0 s\sum_{i=2}^{N}\alpha_i\int_{\mathbb{R}^N_+} \frac{X_i^2X_1w(y)^{2^*(s)}}{|\pi(X)|^{s+2}}dX(1+o(1))+ O(\varepsilon^{\frac{N(N-s)}{N-2}}) \\ &= -\frac{s \varepsilon c_1}{N-1}\int_{\mathbb{R}^N_+}\frac{|X'|^2 X_1w(X)^{2^*(s)}}{|\pi(X)|^{s+2}}dX\sum_{i=2}^{N}\alpha_i(1+o(1)) + O(\varepsilon^{\frac{N(N-s)}{N-2}}) \\ &= -C_0K_1H(0)(1+o(1))\varepsilon+ O(\varepsilon^{\frac{N(N-s)}{N-2}}), \end{align*} where $$H(0)=\frac{1}{N-1}\sum_{i=2}^{N}\alpha_i,\quad K_1= s\int_{\mathbb{R}^N_+}\frac{|X'|^2X_1w^{2^*(s)}}{|\pi(X)|^{s+2}}\,dX.$$ Similarly, \begin{align*} F_{22} &= \frac{1}{\varepsilon}\int_{(B_{r_0/\varepsilon}^+ \setminus B_{\frac{r_0}{2\varepsilon}}^+)\cap\{X_1=0\}} \eta^2(\varphi(\varepsilon X))\varphi_0(\varepsilon X')(\partial_1 u(X))^2\,dS_X \\ &\quad + \frac{1}{\varepsilon}\int_{B_{\frac{r_0}{2\varepsilon}}^+ \cap\{X_1=0\}}\varphi_0(\varepsilon X')(\partial_1 u(X))^2\,dS_X =L_1 + L_2. \end{align*} Also \begin{align*} L_1 &\leq \frac{C}{\varepsilon}\int_{\{\frac{r_0}{2}<|\varepsilon X'| \leq r_0\}}|(\partial_1 u)(0,X')|^2|\varphi_0(\varepsilon X')|\,dX' \\ &\leq C\varepsilon\int_{\{\frac{r_0}{2}<|\varepsilon X'|\leq r_0\}}|X'|^{-2N+2} \,dX'=O(\varepsilon^N). \end{align*} Using that $$\int_{\mathbb{R}^{N-1}\setminus (B_{\frac{r_0}{2\varepsilon}}^+ \cap\{X_1 = 0\})}\varphi_0(\varepsilon X') (\partial_N u(X))^2\,dS_X=O(\varepsilon^N),$$ one finds \begin{align*} L_2 &= \frac{1}{\varepsilon}\int_{\mathbb{R}^{N-1}}\varphi_0(\varepsilon X') (\partial_1 u(X))^2\,dSX+O(\varepsilon^{N-1}) \\ &= \varepsilon\sum_{i=2}^{N}\alpha_i\int_{\mathbb{R}^{N-1}} [(\partial_1 u)(0,X')]^2X_i^2\,dX'(1+o(1))+O(\varepsilon^{N-1}) \\ &= K_2H(0)(1+o(1))\varepsilon+O(\varepsilon^{N-1}), \end{align*} where $K_2= \int_{\mathbb{R}^{N-1}}|(\partial_N u)(0,X')|^2|X'|^2\,dX'$. Consequently, $$\int_{\Omega}|\nabla \tilde u_\varepsilon|^2\,dx = \int_{\mathbb{R}^N_+}|\nabla u|^2\,dX-(C_0K_1-K_2)H(0)(1+o(1)) \varepsilon+O(\varepsilon^2),$$ and similarly, $$\int_{\Omega}|\nabla \tilde v_\varepsilon|^2\,dx = \int_{\mathbb{R}^N_+}|\nabla v|^2\,dX-(C_1K_1-K_2)H(0)(1+o(1)) \varepsilon+O(\varepsilon^2).$$ where $C_1 =\frac{2\beta\lambda}{(2^*(s))^2}\mu_{\alpha,\beta,s} (\mathbb{R}^N_+)A^\alpha B^\beta$. Next, let $X=\frac{\psi(x)}{\varepsilon}$. We estimate $$\int_{\Omega}\frac{\tilde u_\varepsilon^{\alpha} \tilde v_\varepsilon^{\beta}}{|\pi(x)|^s}\,dx \geq \int_{\Omega\cap\tilde{V}}\frac{\tilde u_\varepsilon^{\alpha} \tilde v_\varepsilon^{\beta}}{|\pi(x)|^s}\,dx =\int_{\Omega\cap\tilde{V}}\frac{u_\varepsilon^{\alpha} v_\varepsilon^{\beta}}{|\pi(x)|^s}\,dx= \int_{B_{\frac{r_0}{2\varepsilon}}^+} \frac{u^\alpha(X)v^\beta(X)}{|\frac{\pi(\varphi(\varepsilon X))}{\varepsilon}|^s} \,dX$$ since $\eta\equiv1$ in $\Omega\cap\tilde{V}$. We recall that $x_0\in \mathcal{P}^\bot\cap\partial\Omega$, then we may write $\pi(\varphi(\varepsilon X)) = (\varepsilon x_1 +\varphi_0(\varepsilon y, \varepsilon z), \varepsilon y, 0)$ and $|\pi(\varphi(\varepsilon X))|^2 = \varepsilon^2|\pi(X)|^2 \Big(1 + \frac{2X_1\varphi_0(\varepsilon X')}{ \varepsilon|\pi(X)|^2} + \frac{\varphi_0^2(\varepsilon X')}{\varepsilon^2|\pi(X)|^2}\Big).$ Therefore, \begin{align*} \frac{1}{|\frac{\varphi(\varepsilon X)}{\varepsilon}|^s} &= \frac{1}{|\pi(X)|^s}\Big(1-\frac{s X_1\varphi_0(\varepsilon X')}{ \varepsilon|\pi(X)|^2}-\frac{s\varphi_0^2(\varepsilon X')}{2\varepsilon^2|\pi(X)|^2}\Big)\\ &\quad + \frac{1}{|\pi(X)|^s}O\Big(\frac{2X_1\varphi_0(\varepsilon X')}{ \varepsilon|\pi(X)|^2} + \frac{\varphi_0^2(\varepsilon X')}{\varepsilon^2|\pi(X)|^2}\Big).\\ \end{align*} This and $\int_{\mathbb{R}^N_+\setminus B_{\frac{r_0}{2\varepsilon}}^+} \frac{u^\alpha v^\beta}{|\pi(X)|^s}\,dX=O(\varepsilon^{\frac{N(N-s)}{N-2}})$ enable us to show that \begin{align*} \int_{\Omega\cap\tilde{U}}\frac{\tilde u_\varepsilon^{\alpha} \tilde v_\varepsilon^{\beta}}{|\pi(x)|^s}\,dx &= \int_{B_{\frac{r_0}{2\varepsilon}}^+}\frac{u^\alpha v^\beta}{|\pi(X)|^s}\,dX -\frac{s}{\varepsilon}\int_{B_{\frac{r_0}{2\varepsilon}}^+} \frac{X_1\varphi(\varepsilon X')u^\alpha(X)v^\beta(X)}{|\pi(X)|^{s+2}}\,dX +O(\varepsilon^2)\\ &= \int_{\mathbb{R}^N_+}\frac{u^\alpha v^\beta}{|\pi(X)|^s}\,dX -\frac{s}{\varepsilon}\int_{B_{\frac{r_0}{2\varepsilon}}^+} \frac{X_1\varphi(\varepsilon X')u^\alpha v^\beta}{|\pi(X)|^{s+2}}\,dX +O(\varepsilon^2). \end{align*} Moreover, \begin{align*} & -\frac{s}{\varepsilon}\int_{B_{\frac{r_0}{2\varepsilon}}^+} \frac{X_1\varphi(\varepsilon y')u^\alpha v^\beta}{|\pi(X)|^{s+2}}\,dX\\ &= -\frac{s}{\varepsilon}A^\alpha B^\beta\int_{B_{\frac{r_0}{2\varepsilon}}^+} \frac{X_1\varphi(\varepsilon X')w^{2^*(s)}}{|\pi(X)|^{s+2}}\,dX\\ &=-s\varepsilon \sum_{i=2}^{N}\alpha_iA^\alpha B^\beta \int_{\mathbb{R}^N_+}\frac{X_1 X_i^2w^{2^*(s)}}{|\pi(X)|^{s+2}}\,dX (1+o(1))+O(\varepsilon^{\frac{N(N-s)}{N-2}}) \\ &= -\frac{s\varepsilon}{N-1}A^\alpha B^\beta\int_{\mathbb{R}^N_+} \frac{X_1|X'|^2w^{2^*(s)}}{|\pi(X)|^{s+2}}\,dX \sum_{i=2}^{N}\alpha_i(1+o(1))+O(\varepsilon^{\frac{N(N-s)}{N-2}}). \end{align*} Hence, $$\int_{\Omega\cap\tilde{U}}\frac{\tilde u_\varepsilon^{\alpha} \tilde v_\varepsilon^{\beta}}{|\pi(x)|^s}\,dx = \int_{\mathbb{R}^N_+}\frac{u^\alpha v^\beta}{|\pi(X)|^s}\,dX -K_3H(0)(1+o(1))\varepsilon+O(\varepsilon^2),$$ where $K_3= sA^\alpha B^\beta\int_{\mathbb{R}^N_+} \frac{X_1|X'|^2w^{2^*(s)}}{|\pi(X)|^{s+2}}\,dX = A^\alpha B^\beta K_1$. Finally, let $X=\frac{\psi(x)}{\varepsilon}\in B_{r_0/\varepsilon}^+(0)$. We estimate \begin{align*} \int_{\Omega}\tilde{u}_\varepsilon^{p+1}\,dx &= \varepsilon^{\frac{(2-N)(p+1)}{2}} \int_{\Omega\cap U}\eta^2(x)[u(\frac{\psi(x)}{\varepsilon})]^{p+1}\,dx \\ &= \varepsilon^{\frac{(2-N)(p+1)}{2}+N}\int_{B_{r_0/\varepsilon}^+}u^{p+1}\,dX \\ &= \varepsilon^{\frac{N+2}{2}-\frac{(N-2)p}{2}}\int_{\mathbb{R}^N_+}u^{p+1}\,dX +O(\varepsilon^{\frac{N(p+1)}{2}}). \end{align*} Similarly, $$\int_{\Omega}\tilde{v}_\varepsilon^{p+1}\,dx = \varepsilon^{\frac{N+2}{2}-\frac{(N-2)p}{2}}\int_{\mathbb{R}^N_+}v^{p+1}\,dX +O(\varepsilon^{\frac{N(p+1)}{2}}).$$ Since $q<\frac{N}{N-2}$, $\frac{N+2}{2}-\frac{(N-2)p}{2}>1$. For $t\geq0$, we have \begin{align*} &I_\lambda(t\tilde{u}_\varepsilon,t\tilde{v}_\varepsilon)\\ &= \frac{t^2}{2}\Big(\int_{\mathbb{R}^N_+}|\nabla u|^2\,dX+\int_{\mathbb{R}^N_+} |\nabla v|^2\,dX\Big) - \frac{2t^{2^*(s)}\lambda}{2^*(s)}\int_{\mathbb{R}^N_+} \frac{u^\alpha v^\beta}{|\pi(X)|^s}\,dX \\ &\quad + \frac{H(0)}{2}[(2K_2-C_0K_1-C_1K_1)t^2 +\frac{4}{2^*(s)}(\lambda K_3+o(1))t^{2^*(s)}]\varepsilon+O(\varepsilon^2) \\ &= f_1(t)+ \frac{H(0)}{2}\varepsilon f_2(t)+O(\varepsilon^2), \end{align*} where $f_1(t)= \frac{t^2}{2}\mu_{\alpha,\beta,s}(\mathbb{R}^N_+) - \frac{2\lambda t^{2^*(s)}}{2^*(s)}.$ It can be verified that $\max_{0\leq t\leq1}f_1(t)= f_1(t_0)=(2\lambda)^{\frac{-2}{2^*(s)-2}} (\frac{1}{2}-\frac{1}{2^*(s)}) \mu_{\alpha,\beta,s}(\mathbb{R}^N_+)^{\frac{2^*(s)}{2^*(s)-2}},$ with $t_0=(\frac{1}{2\lambda}\mu_{\alpha,\beta,s} (\mathbb{R}^N_+))^{\frac{1}{2^*(s)-2}}$. Since $K_1>0$, \begin{align*} f_2(t_0) &= (2K_2-C_0K_1-C_1K_1)t_0^2 + \frac{4\lambda}{2^*(s)}K_3t_0^{2^*(s)}\\ &= (2K_2- \frac{2\lambda}{2^*(s)}A^\alpha B^\beta K_1)t_0^2 + \frac{4\lambda}{2^*(s)}A^\alpha B^\beta K_1t_0^{2^*(s)} \\ &= 2K_2t_0^2 + \frac{2\lambda}{2^*(s)}A^\alpha B^\beta K_1 (\frac{\mu_{\alpha,\beta,s}(\mathbb{R}^N_+)}\lambda - 1)t_0^{2}. \end{align*} Hence, $f_2(t_0)>0$ if $\lambda>0$ and small. Since $H(0)<0$, by choosing $T$ large enough, we have $I_\lambda(T\tilde{u}_\varepsilon,T\tilde{v}_\varepsilon)<0$ for $t\geq T$ and $\varepsilon\geq0$ small. Let $u_0=T\tilde{u}_\varepsilon$, $v_0=T\tilde{v}_\varepsilon$. We obtain $$\max_{0\leq t\leq1}I_\lambda(tu_0,tv_0)<(2\lambda) ^{\frac{-2}{2^*(s)-2}}(\frac{1}{2}-\frac{1}{2^*(s)}) \mu_{\alpha,\beta,s}(\mathbb{R}^N_+)^{\frac{2^*(s)}{2^*(s)-2}}$$ and $$I_\lambda(u_0,v_0)<0.$$ This completes the proof of Lemma \ref{lem:2.1}. \end{proof} \section{Existence of positive solution in $\Omega$} Now we will use the blow up argument to prove (iii) of Theorem \ref{thm:1.1}. For any $\varepsilon>0$, by the mountain pass theorem, we have a positive solution pair $(u_\varepsilon,v_\varepsilon)$ of the subcritical system $$\label{eq:3.1} \begin{gathered} -\Delta u_\varepsilon= \frac{2\alpha\lambda}{\alpha+\beta-\varepsilon} \frac{u_\varepsilon^{\alpha-1}v_\varepsilon^{\beta-\varepsilon}}{|\pi(x)|^s} - u_\varepsilon^{p-\varepsilon}, \quad \text{in } \Omega,\\ -\Delta v_\varepsilon= \frac{2\beta\lambda}{\alpha+\beta -\varepsilon}\frac{u_\varepsilon^\alpha v_\varepsilon^{\beta-1-\varepsilon}} {|\pi(x)|^s}- v_\varepsilon^{p-\varepsilon}, \quad \text{in } \Omega,\\ u_\varepsilon>0,v_\varepsilon>0, \quad \text{in } \Omega,\\ u_\varepsilon=v_\varepsilon=0, \quad \text{on } \partial\Omega. \end{gathered}$$ Using Lemma \ref{lem:2.3}, we see that the mountain pass level $c_\varepsilon$ of \eqref{eq:3.1} satisfies $$\label{eq:3.2} c_\varepsilon= I_\lambda^\varepsilon(u_\varepsilon,v_\varepsilon) <(2\lambda)^{\frac{-2}{2^*(s)-2}}(\frac{1}{2}-\frac{1}{2^*(s)}) \mu_{\alpha,\beta,s}(\mathbb{R}^N_+)^{\frac{2^*(s)}{2^*(s)-2}}$$ if $0<\lambda<\lambda^*$, where \begin{align*} I_\varepsilon(u_\varepsilon,v_\varepsilon) &=\int_{\Omega}(\frac{1}{2}|\nabla u_\varepsilon|^2 +\frac{1}{2}|\nabla v_\varepsilon|^2-\frac{2\lambda}{2^*(s) -\varepsilon}\frac{u_\varepsilon^{\alpha}v_\varepsilon ^{\beta-\varepsilon}}{|\pi(x)|^s})\,dx\\ &\quad+\int_{\Omega}(\frac{1}{p+1-\varepsilon}u_\varepsilon ^{p+1-\varepsilon}+\frac{1}{p+1-\varepsilon}v_\varepsilon^{p+1-\varepsilon})\,dx. \end{align*} It can be easily shown that both $\|u_\varepsilon\|_{H_0^1(\Omega)}$ and $\|v_\varepsilon\|_{H_0^1(\Omega)}$ are uniformly bounded for $\varepsilon>0$ small. Thus, there is a subsequence $\{(u_j,v_j)\}$ of $\{(u_\varepsilon, v_\varepsilon)\}$ such that $$\label{eq:3.3} \begin{gathered} u_j\rightharpoonup u,\quad v_j\rightharpoonup v, \quad \text{in } H_0^1(\Omega),\\ u_j\to u, \quad v_j\to v, \quad \text{in } L^{p+1}(\Omega),\\ u_j\rightharpoonup u, \quad v_j\rightharpoonup v, \quad \text{in } L^{2^*(s)}(\Omega,|\pi(x)|^{-s}dx),\\ \end{gathered}$$ with $u,v\geq0$ and $(u,v)$ is a solution of system \eqref{eq:1.1}. If $(u,v)$ is a nontrivial solution, by the strong maximum principle, $u,v>0$, then we are done. Now, we prove $(u,v)$ is nontrivial. This will be shown by the blowing up argument. Suppose on the contrary that $u=v=0$ in $\Omega$. By the regularity result, see for instance \cite[Proposition 3.2]{GR2}, $u_\varepsilon, v_\epsilon\in C^1(\bar \Omega)$. Let $x_j, y_j\in \Omega$ be such that $$M_j=u_j(x_j)=\max_{\bar{\Omega}}u_j(x),\quad N_j=v_j(y_j) =\max_{\bar{\Omega}}v_j(x).$$ Then, we have either $m_j\to \infty$ or $n_j\to \infty$ as $j\to\infty$. Indeed, on the contrary we would have $m_j\leq C$ and $n_j\leq C$ for a positive constant $C$. By the Sobolev embedding, $$\int_{\Omega}\frac{u_j^{\alpha}v_j^{\beta-\varepsilon_j}}{|\pi(x)|^s}\,dx \leq C\int_{\Omega}\frac{u_j^{\alpha}}{|\pi(x)|^s}\,dx\to 0$$ as $j\to\infty$. This implies $$\int_{\Omega}(|\nabla u_j|^2+ |\nabla v_j|^2)\,dx = 2\int_{\Omega}\frac{u_j^{\alpha}v_j^{\beta-\varepsilon_j}}{|\pi(x)|^s}\,dx -\lambda\int_{\Omega}u_j^{p+1-\varepsilon_j}dx-\lambda\int_{\Omega}v_j ^{p+1-\varepsilon_j}dx\to 0;$$ that is, $u_j\to 0$, $v_j\to 0$ strongly in $H_0^1(\Omega)$. It yields $$0= \lim_{j\to \infty}\frac{1}{2}\int_{\Omega}(|\nabla u_j|^2 + |\nabla v_j|^2)\,dx=c>0$$ a contradiction. Suppose $N_j\leq M_j\to\infty$. Denote $$\tilde u_j(x)=M_j^{-1}u_j(k_jx+x_j),\quad \tilde v_j(x)=M_j^{-1}v_j(k_jx+x_j),\quad \text{for } x\in\Omega_j,$$ where $k_j = M_j^{-\frac{2^*(s)-2-\varepsilon_j}{2-s}}$ and $\Omega_j=\{x\in\mathbb{R}^N\mid x_j+k_jx\in\Omega\}$. Obviously, $(\tilde u_j, \tilde v_j)$ satisfies $$\label{eq:3.4} \begin{gathered} -\Delta \tilde u_j = \frac{2\alpha\lambda}{\alpha+\beta-\varepsilon_j} \big(\frac{k_j}{|\pi(x_j)|}\big)^s\frac{\tilde u_j^{\alpha-1} \tilde v_j^{\beta-\varepsilon_j}}{|\pi(\frac{x_j}{|\pi(x_j)|} +\frac{k_j}{|\pi(x_j)|}x)|^s}- k_j^2M_j^{p-1-\varepsilon_j} \tilde u_j^{p-\varepsilon_j}, \quad\text{in } \Omega_{j},\\ -\Delta \tilde v_j= \frac{2(\beta-\varepsilon_j)\lambda} {\alpha+\beta-\varepsilon_j}\big(\frac{k_j}{|\pi(x_j)|}\big)^s \frac{\tilde u_j^\alpha \tilde v_j^{\beta-1-\varepsilon_j}} {|\pi(\frac{x_j}{|\pi(x_j)|}+\frac{k_j}{|\pi(x_j)|}x)|^s} - k_j^2M_j^{p-1-\varepsilon_j}\tilde v_j^{p-\varepsilon_j}, \quad \text{in } \Omega_{j},\\ 0\leq\tilde u_j,\tilde v_j \leq1, \quad\text{in } \Omega_j,\\ \tilde u_j=\tilde v_j =0, \quad \text{on } \partial\Omega_{j}. \end{gathered}$$ Since $M_j\to\infty$, $k_j\to 0$ as $j\to \infty$. Furthermore, we have $$k_j^2M_j^{p-1-\varepsilon_j}= k_j ^{2-\frac{(2-s)(p-\varepsilon_j-1)}{2^*(s)-2-\varepsilon_j}} \to 0 \quad \text{as } j\to \infty$$ as the facts that $k_j\to 0$ and $2-\frac{(2-s)(p-\varepsilon_j-1)}{2^*(s)-2-\varepsilon_j}>0$; i.e, $p<\frac{N+2}{N-2}$. We will show that $M_j=O(1)N_j$. First, we claim that $|\pi(x_j)|=O(k_j)$ as $j\to\infty$. Suppose on the contrary that $\limsup_{j\to\infty}\frac{|\pi(x_j)|}{k_j}=\infty$. Because $(\tilde u_j, \tilde v_j )$ is uniformly bounded in $C^1_{\rm loc}$, we may assume that $\tilde u_j\to u, \tilde v_j\to v$ in $C^0_{\rm loc}$. Suppose now $x_j\to x_0\in\bar{\Omega}$. There are two cases: \begin{itemize} \item[(i)] $x_0\in\Omega$ or $x_0\in\partial\Omega$ and $\frac{\operatorname{dist}(x_j,\partial\Omega)}{k_j}\to\infty$; and \item[(ii)] $x_0\in\partial\Omega$ and $\frac{\operatorname{dist}(x_j,\partial\Omega)}{k_j}\to \sigma\geq0$. \end{itemize} In the case (i), we have $\Omega_{j}\to\mathbb{R}^N$ as $j\to\infty$ and $(u,v)$ satisfies \begin{gather*} \Delta u= 0,\quad \Delta v= 0\quad\text{in }\mathbb{R}^N, \\ 0\leq u,v\leq 1,\quad u(0) =1. \end{gather*} Furthermore, $$\int_{\Omega_j}\tilde u_j^{\frac{2N}{N-2}}\,dy =k_j^{\frac{N\varepsilon_j}{2^*(s)-2-\varepsilon_j}} \int_{\Omega}u_j^{\frac{2N}{N-2}}\,dx\leq C, \quad \text{and}\quad \int_{\Omega_j}\tilde v_j^{\frac{2N}{N-2}}\,dy\leq C,$$ which yields $$\int_{\mathbb{R}^N}u^{\frac{2N}{N-2}}\,dy<\infty, \quad \int_{\mathbb{R}^N}v^{\frac{2N}{N-2}}\,dy<\infty.$$ However, by the Liouville theorem, $u\equiv v\equiv 1$ for $x\in\mathbb{R}^N$. This is a contradiction. In case (ii), after an orthogonal transformation, we have $\Omega_{j}\to\mathbb{R}^N_+=\{x=(x_1,\dots,x_N)\mid x_1>0\}$ as $j\to\infty$ and $\tilde u_j$ , $\tilde v_j$ converge to some $u$, $v$ uniformly in every compact subset of $\mathbb{R}^N_+$. Now, $u(0)=1$ and $0\leq v(0)\leq1$. Hence, $(u,v)$ satisfies \begin{gather*} \Delta u=0,\quad \Delta v= 0\quad\text{in }\mathbb{R}^N_{+}, \\ 0\leq u,v\leq1\quad\text{in }\mathbb{R}^N_{+}, \\ u=v=0\quad\text{on } \partial\mathbb{R}^N_{+}. \end{gather*} By the boundary condition and the maximum principle, $u\equiv v\equiv 0$ for $x\in\mathbb{R}^N_{+}$ which violate to $u(0)=1$. Consequently, $\limsup_{j\to\infty}\frac{|\pi(x_j)|}{k_j}<\infty$. Since $k_j\to 0$, we have $\pi(x_j)\to 0$ as $j\to\infty$. Next, we show that $\liminf_{j\to \infty}\frac{|\pi(x_j)|}{k_j}>0$. Were it not the case, we would have, up to a subsequence, that $\lim_{j\to\infty}\frac{|\pi(x_j)|}{k_j}=0$. Then $(\tilde u_j, \tilde v_j)$ satisfies $$\label{eq:3.5} \begin{gathered} -\Delta \tilde u_j = \frac{2\alpha\lambda}{\alpha+\beta -\varepsilon_j}\frac{\tilde u_j^{\alpha-1}\tilde v_j^{\beta-\varepsilon_j}} {|\frac{\pi(x_j)}{k_j}+\pi(x)|^s}- k_j^2M_j^{p-1-\varepsilon_j} \tilde{u_j}^{p-\varepsilon_j}, \quad \text{in } \Omega_{j},\\ -\Delta \tilde v_j= \frac{2(\beta-\varepsilon_j)\lambda} {\alpha+\beta-\varepsilon_j}\frac{\tilde u_j^\alpha \tilde v_j^{\beta-1-\varepsilon_j}}{|\frac{\pi(x_j)}{k_j}+\pi(x)|^s} - k_j^2M_j^{p-1-\varepsilon_j}\tilde{v_j}^{p-\varepsilon_j}, \quad \text{in } \Omega_{j},\\ 0\leq\tilde u_j,\tilde v_j \leq1, \quad \text{in } \Omega_j,\\ \tilde u_j=\tilde v_j =0, \quad \text{on } \partial\Omega_{j}, \end{gathered}$$ Up to a rotation, we have $\Omega_{j}\to\mathbb{R}^N_+$ and $\tilde u_j$, $\tilde v_j$ converge to some $u$, $v$ uniformly in compact subsets of $\mathbb{R}^N_+$ respectively, where $(u,v)$ satisfies \begin{gather*} -\Delta u= \frac{2\alpha\lambda}{\alpha+\beta} \frac{u^{\alpha-1}v^\beta}{|\pi(x)|^s},\quad -\Delta v = \frac{2\beta\lambda}{\alpha+\beta}\frac{u^{\alpha}v^{\beta-1}}{|\pi(x)|^s} \quad\text{in }\mathbb{R}^N_+, \\ 0\leq u,v\leq 1\quad\text{in }\mathbb{R}^N_+,\quad u=v=0\quad \text{on } \partial\mathbb{R}^N_+. \end{gather*} The boundary condition violates to $u(0)=1$. Hence, $\liminf_{j\to \infty}\frac{|\pi(x_j)|}{k_j}>0$. Now, we complete the proof of Theorem \ref{eq:1.1} by showing that problem \eqref{eq:1.1} has a nontrivial solution. First, we remark that $\operatorname{dist}(x_j,\partial\Omega) = O(k_j)$. Indeed, since $\mathcal{P}^\bot\cap\Omega = \emptyset$, we have $x_j-\pi(x_j)\in \mathcal{P}^\bot\subset \mathbb{R}^N\setminus\Omega$. Because $x_j\in\Omega$, there exists $t_j\in(0,1)$ such that $t_jx_j+(1-t_j)(x_j-\pi(x_j))\in\partial\Omega$. Therefore, $d(x_j,\partial\Omega)\leq |x_j-(t_jx_j+(1-t_j)(x_j-\pi(x_j)))| =(1-t_j)|\pi(x_j)|\leq |\pi(x_j)| = O(k_j).$ Hence, we may assume $\frac{\operatorname{dist}(x_j,\partial\Omega)}{k_j}\to \sigma\geq0$. By an affine transformation, we find $(\tilde u_j,\tilde v_j)$ converges to $(u,v)$ uniformly in any compact subset of ${\mathbb{R}^N_+}$ and $(u,v)$ satisfies $$\label{eq:3.6} \begin{gathered} -\Delta u= \frac{2\alpha\lambda}{\alpha+\beta} \frac{u^{\alpha-1}v^\beta}{|\pi(x)|^s},\quad -\Delta v = \frac{2\beta\lambda}{\alpha+\beta}\frac{u^{\alpha}v^{\beta-1}}{|\pi(x)|^s} \quad\text{in }\mathbb{R}^N_+, \\ u,v>0\quad\text{in }\mathbb{R}^N_+;\quad u=v=0\quad \text{on } \partial\mathbb{R}^N_+ \end{gathered}$$ with $u(0,\dots,\sigma)=1$. By the definition of $\mu_{\alpha,\beta,s}(\Omega)$, we have $$\mu_{\alpha,\beta,s}(\Omega_j) \leq \frac{\int_{\Omega}(|\nabla \tilde u_j|^2 +|\nabla \tilde v_j|^2)\,dx}{\big(\int_{\Omega} \frac{\tilde u_j^\alpha \tilde v_j^{\beta-\varepsilon}}{|x|^s}\,dx\big) ^{\frac{2}{2^*(s)}}},$$ and then $$\mu_{\alpha,\beta,s}(\mathbb{R}^N_+) \leq \frac{\int_{\mathbb{R}^N_+}(|\nabla u|^2+|\nabla v|^2)\,dy} {\big(\int_{\mathbb{R}^N_+}\frac{u^\alpha v^\beta}{|\pi(x)|^s}\,dx\big) ^{\frac{2}{2^*(s)}}}= 2\lambda \Big(\int_{\mathbb{R}^N_+} \frac{u^\alpha v^\beta}{|\pi(x)|^s}\,dx\Big)^{\frac{2^*(s)-2}{2^*(s)}};$$ that is, $$\label{eq:3.7} \int_{\mathbb{R}^N_+}(|\nabla u|^2+|\nabla v|^2)\,dx =2\lambda \int_{\mathbb{R}^N_+}\frac{u^\alpha v^\beta}{|\pi(x)|^s}\,dx \geq (2\lambda)^{\frac{-2}{2^*(s)-2}}\mu_{\alpha,\beta,s}(\mathbb{R}^N_+) ^{\frac{2^*(s)}{2^*(s)-2}}.$$ Furthermore, noting that \label{eq:3.8} \begin{aligned} \lim_{j\to \infty}\int_{\Omega}(|\nabla u_j|^2+|\nabla v_j|^2)\,dx &= \lim_{j\to \infty}k_j^{-\frac{(N-2)\varepsilon_j}{2^*(s)-2-\varepsilon_j}} \int_{\Omega_j}(|\nabla \tilde u_j|^2+|\nabla \tilde v_j|^2)\,dx \\ &\geq \lim_{j\to \infty}\int_{\Omega_j}(|\nabla \tilde u_j|^2 +|\nabla \tilde v_j|^2)\,dx\\ &\geq \int_{\mathbb{R}^N_+}(|\nabla u|^2+|\nabla v|^2)\,dx, \end{aligned} we derive from \eqref{eq:3.2}, \eqref{eq:3.7}, \eqref{eq:3.8} that \begin{align*} c &= (\frac{1}{2}-\frac{1}{2^*(s)})\lim_{j\to \infty} \int_{\Omega}(|\nabla u_j|^2+|\nabla v_j|^2)\,dx\\ &\geq (\frac{1}{2}-\frac{1}{2^*(s)})(2\lambda)^{\frac{-2}{2^*(s)-2}} \mu_{\alpha,\beta,s}(\mathbb{R}^N_+)^{\frac{2^*(s)}{2^*(s)-2}},\\ \end{align*} which yields a contradiction to \eqref{eq:3.2}. Thus, $(u,v)$ is a nontrivial solution of \eqref{eq:1.1} if $N_j\leq M_j$. Now we show $M_j=O(N_j)$. Indeed, since $u$ is nontrivial, so is $v$. Otherwise, we would have \begin{gather*} \Delta u= 0\quad\text{in }\mathbb{R}^N_+,\\ 0\leq u\leq1,u(0,\dots,\sigma)=1\quad\text{in }\mathbb{R}^N_+,\\ u= 0\quad\text{on } \partial\mathbb{R}^N_+. \end{gather*} By the strong maximum principle, $u$ would be a constant because it attains its maximum value inside $\mathbb{R}^N_+$. This yields a contradiction between $u(0,\dots,\sigma)=1$ and the boundary condition. Therefore, there exists $y_0\in \mathbb{R}^N_+$ such that $v(y_0)\neq 0$. Hence, $$\tilde v_j(y_0)= m_j^{-1}v_j(x_j+k_jy_0)\to v(y_0)>0$$ implying $$1\geq\frac{n_j}{m_j}\geq\frac{v_j(x_j+k_jy_0)}{m_j}\geq v(y_0)-\varepsilon>0$$ for $\varepsilon>0$ small and $j$ large, namely, $N_j=O(1)M_j$ as $j\to\infty$. Replacing $M_j$ by $N_j$ in above blow up process, we may also derive a contradiction if we assume $u=v=0$. Consequently, \eqref{eq:1.1} has a positive nontrivial solution. The proof of Theorem \ref{thm:1.1} is complete. \subsection*{Acknowledgments} This work is supported by grant 11271170 from the NNSF of China, by GAN PO 555 program of Jiangxi, and by grant 2912BAB201008 from the NNSF of Jiangxi. \begin{thebibliography}{00} \bibitem{AWZ} A. Al-aati, C. Wang, J. Zhao; Positive solutions to a semilinear elliptic equation with a Sobolev-Hardy term, {\it Nonlinear Analysis}, 74(2011), 4847-4861. \bibitem{AFS} C. O. Alves, D. C. de Morais Filho, M. A. S. Souto; On systems of elliptic equations involving subcritical and critical Sobolev exponents, {\it Nonlinear Anal. TMA}, 42 (2000), 771-787. \bibitem{BN} H. Br\'{e}zis, L. Nirenberg; Positive solutions of nonlinear elliptic equations involving critical Sobolev exponents, {\it Comm. Pure Appl. Math.,} 36 (1983), 437-477. \bibitem {BT} M.Badiale, G. Tarantello; A Sobolev-Hardy inequality with applications to a nonlinear elliptic equation arising in astrophysics, {\it Arch. Ration. Mech. Anal.} 163(2002), 259-293. \bibitem{CKN} L. Caffarelli, R. Kohn, L. Nirenberg; First order interpolation inequality with weights, {\it Compositio Math.,} 53 (1984), 259-275. \bibitem{E} H. Egnell; Positive solutions of semilinear equations in cones, {\it Trans. Amer. Math. Soc.,} 11 (1992), 191-201. \bibitem{FMS} I. Fabbri, G. Mancini, K. Sandeep; Classification of solutions of a critical Hardy Sobolev operator, {\it Jour. Diff. Equa.}, 224 (2006), 258-276. \bibitem{GRa} M. Ghergu, V. Radulescu; Singular elliptic problems with lack of compactness, {\it Annali di Matematica Pura ed Applicata,} 185 (2006), 63-79. \bibitem{GK} N. Ghoussoub, X. S. Kang; Hardy-Sobolev critical elliptic equations with boundary singularities, {\it Ann. Inst. H. Poincar\'{e} Anal. Non Lin\'{e}aire,} 21 (2004), 769-793. \bibitem{GR} N. Ghoussoub, F. Robert; The effect of curvature on the best constant in the Hardy-Sobolev inequalities, {\it Geom. Funct. Anal.,} 16 (2006), 1201-1245. \bibitem{GR1} N. Ghoussoub, F. Robert; Concentration estimates for Emden-Fowler equations with boundary singularities and critical growth, {\it IMRP Int. Math. Res. Pap.,} 21867 (2006), 1-85. \bibitem{GR2} N. Ghoussoub, F. Robert; Elliptic equations with critical growth and a large set of boundary singularities, {\it Trans. Amer. Math. Soc.} 361 (2009), 4843-4870. \bibitem{HLW} C. H. Hsia, C. S. Lin, H. Wadade; Revisiting an idea of Br\'{e}zis and Nirenberg, {\it J. Funct. Anal.,} 259 (2010), 1816-1849. \bibitem{HY} Haiyang He, Jianfu Yang; Positive solutions for critical elliptic systems in non-contractible domain, {\it Nonlinear Anal. TMA} 70 (2009), 952-973. \bibitem{L} P. L. Lions; The concentration-compactness principle in the calculus of variations. The locally compact case, {\it Ann. Inst. H. Poincar\'{e} Anal. Non Lin\'{e}aire,} 1 (1984), 109-145 and 223-283. \bibitem{LW} C.S. Lin, H. Wadade; Minimizing problems for the Hardy-Sobolev type inequality with the sigularity on the boundary, preprint 2011. \bibitem{M} V. G. Maz'ya; Sobolev spaces, Springer Series in Soviet Mathematics, Springer-Verlag, Berlin, 1985. \bibitem{RSW} V. Radulescu, D. Smets, M. Willem; Hardy-Sobolev inequalities with remainder terms, {\it Topol. Meth. Nonlin. Anal.,} 20 (2002), 145-149. \bibitem{T} G. Talenti; Best constant in Sobolev inequality, {\it Ann. Mat. Pura Appl.,} 110 (1976), 353-372. \bibitem{TZ} Zhongwei Tang; Sign-changing solutions of critical growth nonlinear elliptic systems, {\it Nonlinear Anal. TMA,} 64 (2006), 2480-2491. \end{thebibliography} \end{document}