\documentclass[reqno]{amsart} \usepackage{amssymb,hyperref} \AtBeginDocument{{\noindent\small {\em Electronic Journal of Differential Equations}, Vol. 2005(2005), No. 107, pp. 1--12.\newline ISSN: 1072-6691. URL: http://ejde.math.txstate.edu or http://ejde.math.unt.edu \newline ftp ejde.math.txstate.edu (login: ftp)} \thanks{\copyright 2005 Texas State University - San Marcos.} \vspace{9mm}} \begin{document} \title[\hfilneg EJDE-2005/107\hfil Existence result] {Existence result for a semilinear parametric problem with Grushin type operator} \author[N. M. Chuong, T. D. Ke\hfil EJDE-2005/107\hfilneg] {Nguyen Minh Chuong, Tran Dinh Ke} % in alphabetical order \address{Nguyen Minh Chuong \hfill\break Institute of Mathematics, Vietnamese Academy of Science and Technology, 18 Hoang Quoc Viet, 10307 Hanoi, Vietnam} \email{nmchuong@math.ac.vn} \address{Tran Dinh Ke \hfill\break Department of Mathematics, Hanoi University of Education, Hanoi, Vietnam} \email{ketd@hn.vnn.vn} \date{} \thanks{Submitted October 10, 2004. Published October 7, 2005.} \thanks{Supported by the National Fundamental Research Program, Vietnam Academy \hfill\break\indent of Science and Technology.} \subjclass[2000]{34B27, 35J60, 35B05} \keywords{Semilinear parametric problem; Grushin type operator; \hfill\break\indent potential function, (sub)critical exponent} \begin{abstract} Using a varational method, we prove an existence result depending on a parameter, for a semilinear system in potential form with Grushin type operator. \end{abstract} \maketitle \numberwithin{equation}{section} \newtheorem{theorem}{Theorem}[section] \newtheorem{lemma}[theorem]{Lemma} \newtheorem{proposition}[theorem]{Proposition} \newtheorem{corollary}[theorem]{Corollary} \renewcommand{\le}{\leqslant} \renewcommand{\ge}{\geqslant} \section{Introduction} Let $\Omega$ be a bounded domain in $\mathbb R^{N_1} \times \mathbb R^{N_2}$, with smooth boundary $\partial \Omega$ and $\{0\}\in \Omega$. We shall be concerned with the existence of solutions of the Dirichlet problem \begin{equation} \begin{gathered} L_{\alpha, \beta} U = \lambda \nabla F \quad \text{in } \Omega, \\ U = 0 \quad \text{on } \partial \Omega, \end{gathered} \label{e1} \end{equation} where \begin{gather*} U=(u,v), \quad L_{\alpha, \beta} = \begin{pmatrix} -G_{\alpha} & 0 \\ 0 & -G_{\beta} \\ \end{pmatrix} ,\quad G_s = \Delta_x + |x|^{2s}\Delta_y \quad \text{for } s\ge 0, \\ \Delta_x = \sum_{i=1}^{N_1}\frac{\partial^2}{\partial x_i^2}, \quad \Delta_y = \sum_{j=1}^{N_2}\frac{\partial^2}{\partial y_j^2}, \end{gather*} $F=F(x,y,u,v)$ is potential function, $\nabla F=(\frac{\partial F}{\partial u}, \frac{\partial F}{\partial v})$, $\alpha\ge 0$, $\beta\ge 0$ and $\lambda$ is a positive parameter. Denoting $N(s)=N_1 + (s+1) N_2$, we assume that $N_1, N_2\ge 1$ and $N(\alpha), N(\beta)>2$. For $s\ge 0, G_s$ is a Grushin type operator \cite{ref5}. Its properties (such as degeneracy, hypoellipticity) were considered in \cite{ref5},\cite{ref11}. A semilinear problem with $G_s$ in scalar case was studied in \cite{ref10}. Thuy and Tri \cite{ref10} pointed out the critical Sobolev exponent and proved the existence theorem for subcritical case. Many authors investigated the existence of solutions for scalar cases or potential system cases with Laplace and p-Laplacian operator (see \cite{ref16,ref1,ref2,ref3,ref7, ref4,ref6,ref9} and references therein). On the other hand, existence result for systems in Hamiltonian form with $G_s$ was obtained in \cite{ref14} and \cite{ref12}. Our main goal in this paper is using the Moutain Pass scheme and Ekeland's variational principle as in \cite{ref15}, \cite{ref7} and \cite{ref9} to find the weak solutions for system \eqref{e1} in the suitable Sobolev space when $\lambda\in (0, \lambda^*)$ and observe on the behaviour of that solutions as $\lambda\to 0$. In particular, we consider the system \eqref{e1} with some classes of homogeneous and nonhomogeneous nonlinearities. To state our main result, we need some definitions and notations. \subsection{Definition 1} By $S_1^p(\Omega), 1\le p < +\infty$, we denote the set of all pair $(u,v) \in L^p(\Omega)\times L^p(\Omega)$ such that $\frac{\partial u}{\partial x_i}$, $\frac{\partial v}{\partial x_i}, |x|^{\alpha}\frac{\partial u}{\partial y_j}$, $|x|^{\beta}\frac{\partial v}{\partial y_j}\in L^p(\Omega)$ for all $i=1,\dots ,N_1$, and $j=1,\dots ,N_2$. For the norm in $S_1^p(\Omega)$, we take $$ \|(u,v)\|_{S^p_1 (\Omega)} = \Big[\int_{\Omega} (|u|^p + |\nabla_x u|^p + |x|^{p\alpha}|\nabla_y u|^p + |v|^p + |\nabla_x v|^p + |x|^{p\beta} |\nabla_y v|^p ) dxdy\Big]^{1/p} $$ where $$ \nabla_x = (\frac{\partial}{\partial x_1},\dots ,\frac{\partial}{\partial x_{N_1}}), \nabla_y = (\frac{\partial}{\partial y_1},\dots ,\frac{\partial}{\partial y_{N_2}}). $$ For $p=2$, the inner product in $S_1^2(\Omega)$ is defined by \begin{align*} & \langle(u,v), (\varphi, \psi)\rangle \\ & = \int_{\Omega} (u\varphi + \nabla_x u\nabla_x\varphi + |x|^{2\alpha}\nabla_y u\nabla_y\varphi + v\psi + \nabla_x v\nabla_x\psi + |x|^{2\beta}\nabla_y v \nabla_y \psi) dx\,dy. \end{align*} The space $S_{1,0}^p(\Omega)$ is defined as closure of $C_0^1(\Omega)\times C_0^1(\Omega)$ in space $S_1^p(\Omega)$. By standard arguments, one can prove that $S_1^p(\Omega) $ and $S_{1,0}^p(\Omega)$ are Banach spaces, $S_1^2(\Omega) $ and $S_{1,0}^2(\Omega)$ are Hilbert spaces. The following Sobolev embedding inequality was proved in \cite{ref10}. \begin{equation} \Big(\int_{\Omega} |u|^{q}dx\,dy\Big)^{1/q} \le C\Big[\int_{\Omega} (|\nabla_x u|^2 + |x|^{2s}|\nabla_y u|^2)dx\,dy\Big]^{1/2}, \label{e2} \end{equation} where $q=\frac {2N(s)}{N(s)-2}-\tau, C>0, s\ge 0$ and $N(s) = N_1 + (s+1)N_2$, provided small positive number $\tau$. Furthermore, the number $\frac{N(s)+2}{N(s)-2}$ is critical Sobolev exponent for the embedding in \eqref{e2}. Denoting by $$ L^{p,q}(\Omega) = \{(u,v): u\in L^p(\Omega), v\in L^q(\Omega)\}, $$ and endowing this space with the norm $$ \|(u,v)\|_{L^{p,q}(\Omega)} = \Big[\int_{\Omega} |u|^p dx\,dy\Big]^{1/p} + \Big[\int_{\Omega} |v|^q dx\,dy\Big]^{1/q}, $$ we have the conclusion in view of \eqref{e2} that $S^2_{1,0}(\Omega)\subset L^{\frac{2N(\alpha)}{N(\alpha)-2}-\tau_1, \frac{2N(\beta)}{N(\beta)-2}-\tau_2}(\Omega)$ and this embedding is a compact mapping for all small positive numbers $\tau_1$ and $\tau_2$ (see \cite{ref10}). \subsection{Definition 2} A pair $(u,v)\in S^2_{1,0}(\Omega)$ is called a weak solution of system \eqref{e1} if \begin{equation} \begin{aligned} \int_{\Omega} (\nabla_x u\nabla_x\varphi + |x|^{2\alpha}\nabla_y u\nabla_y\varphi)dx\,dy &= \lambda \int_{\Omega}\frac{\partial F}{\partial u}\varphi dx\,dy,\\ \int_{\Omega} (\nabla_x v\nabla_x\psi + |x|^{2\beta}\nabla_y v\nabla_y\psi)dx\,dy &= \lambda \int_{\Omega}\frac{\partial F}{\partial v}\psi dx\,dy \end{aligned} \label{e3} \end{equation} for every $\varphi, \psi \in C^{\infty}_0(\Omega)$. Since the system \eqref{e1} is in the gradient form, we intend to get its solutions as the critical points of the functional \begin{equation} \begin{aligned} I_{\lambda} (u,v) = & \frac 12 \int_{\Omega} (|\nabla_x u|^2 + |x|^{2\alpha}|\nabla_y u|^2 + |\nabla_x v|^2 + |x|^{2\beta}|\nabla_y v|^2)dx\,dy \\ & - \lambda \int_{\Omega} F(x,y,u,v) dx\,dy, \end{aligned} \label{e4} \end{equation} defined on reflexive Banach space $S_{1,0}^2(\Omega)$ with Fr\'echet derivative given by \begin{equation} \begin{aligned} \langle I'_{\lambda} (u,v), (\varphi, \psi)\rangle = & \int_{\Omega} (\nabla_x u\nabla_x\varphi + |x|^{2\alpha}\nabla_y u\nabla_y\varphi + \nabla_x v\nabla_x\psi + |x|^{2\beta}\nabla_y v\nabla_y\psi)dx\,dy \\ & - \lambda \int_{\Omega} (\frac{\partial F}{\partial u}\varphi + \frac{\partial F}{\partial v}\psi) dx\,dy. \label{e5} \end{aligned} \end{equation} Let $f=\frac{\partial F}{\partial u}$ and $g=\frac{\partial F}{\partial v}$ be two Carath\'eodory functions satisfying the following conditions: \begin{itemize} \item[(H1)] There exist positive constants $C_i$, for $i=1,\dots,6$ such that \begin{equation} \begin{gathered} |f(x,y,s,t)|\le C_1 + C_2|s|^{r_1} + C_3|t|^{r_2},\\ |g(x,y,s,t)|\le C_4 + C_5|s|^{r_3} + C_6|t|^{r_4} \end{gathered} \label{e6} \end{equation} for a.e. $(x,y)\in \Omega$ and for all $s,t\in \mathbb R$, where \begin{equation} 0 < r_1, r_2 < \frac{N(\alpha)+2}{N(\alpha)-2}; \quad 0 < r_3, r_4< \frac{N(\beta)+2}{N(\beta)-2}. \label{e7} \end{equation} \item[(H2)] $F(x,y,0,0) = 0$ and there are two positive constants $\mu >2$ and $M>0$ such that \begin{equation} 0<\mu F(x,y,u,v)\le uf(x,y,u,v) + vg(x,y,u,v) \label{e8} \end{equation} for a.e. $(x,y)\in \Omega$ and for all $u,v\in \mathbb{R}$ satisfying $|u|, |v| \ge M >0$. \item[(H3)] For a.e. $(x,y)\in \Omega$, \begin{equation} \lim_{|u|+|v|\to\infty}\frac{F(x,y,u,v)}{|u|^2+|v|^2}=+\infty. \label{e9} \end{equation} \end{itemize} The inequalities in \eqref{e6} and \eqref{e7} express the subcritical character of the system \eqref{e1} and guarantee the well-definiteness of the functional $I_{\lambda}$. It's now to state our main result. \begin{theorem}\label{th1} Under hypotheses (H1), (H2) and (H3), there exists a positive constant $\lambda^{*}$ such that for any $\lambda \in (0, \lambda^{*})$, the functional $I_{\lambda}$ has a nontrivial critical point $(u_{\lambda}, v_{\lambda})$ satisfying $\|(u_{\lambda},v_{\lambda})\|_{S^2_{1,0}(\Omega)}\to +\infty$ as $\lambda \to 0$. \end{theorem} Our work is organized as follows. In section 2, we prove some lemmas to establish the analysis framework for the proof of the main theorem in section 3. We shall make a note that, the operator $L_{\alpha,\beta}$ in system \eqref{e1} has some extensions preserved our proofs. In the last section, we are interested in some cases of nonlinearity of the system \eqref{e1}. \section{Preliminaries} We first recall standard definitions and notations. Let $X$ be a reflexive Banach space endowed with a norm $\|.\|$. Let $\langle.,.\rangle$ denote the duality pairing between $X$ and its dual $X^*$. We denote the weak convergence in $X$ by ``$\rightharpoonup$" and the strong convergence by ``$\to$". Let $I\in C^1(X,\mathbb R)$. We say $I$ satisfies the {\it Palais-Smale condition}, denoted by $(PS)$ condition, if every Palais-Smale sequence (a sequence $\{x_n\}\subset X$ such that $\{I(x_n)\}$ is bounded and $I^{'}(x_n)\to 0$ in dual space $X^{*}$) is relatively compact. Putting \begin{align*} \Phi (u,v) &= \frac 12 \int_{\Omega} (|\nabla_x u|^2 + |x|^{2\alpha}|\nabla_y u|^2 + |\nabla_x v|^2 + |x|^{2\beta}|\nabla_y v|^2)dx\,dy,\\ \Psi (u,v) & = \int_{\Omega} F(x,y,u,v) dx\,dy, \end{align*} we can write \begin{equation} I_{\lambda} (u,v) = \Phi (u,v) - \lambda \Psi (u,v). \label{e10} \end{equation} \begin{lemma} \label{lm2} Suppose $f$ and $g$ are continuous functions satisfying (H2). Then every Palais-Smale sequence of $I_{\lambda}$ is bounded. \end{lemma} \begin{proof} Let $\{(u_n, v_n)\}$ be a Palais-Smale sequence of $I_\lambda$, that is, \begin{equation} \Phi (u_n,v_n) - \lambda \Psi (u_n,v_n) \to c \label{e11} \end{equation} and \begin{equation} |\langle\Phi^{'}(u_n, v_n), (\xi, \eta)\rangle - \lambda \langle\Psi^{'}(u_n, v_n), (\xi, \eta)\rangle|\le \epsilon_n \|(\xi, \eta)\|_{S^2_{1,0}(\Omega)},\label{e12} \end{equation} for all $(\xi,\eta)\in S_{1,0}^2(\Omega)$, where $\epsilon_n \to 0$ as $n\to \infty$. From \eqref{e11}, we have \begin{equation} \mu \Phi (u_n,v_n) - \lambda \mu \Psi (u_n,v_n) \le \mu c + 1. \label{e13} \end{equation} Subtracting \eqref{e12}, with $(\xi, \eta)=(u_n, v_n)$, yields \begin{equation} \begin{aligned} \; & (\mu-2)\Phi (u_n, v_n) -\lambda [\mu \Psi(u_n, v_n) - \langle\Psi^{'}(u_n, v_n), (u_n, v_n)\rangle] \\ \; & \le \mu c + 1 + \epsilon_n \|(u_n, v_n)\|_{S^2_{1,0}(\Omega)}. \end{aligned}\label{e14} \end{equation} Assumption (H2) ensures that \begin{equation} \mu \Psi(u_n, v_n) - \langle\Psi^{'}(u_n, v_n), (u_n, v_n)\rangle \le 0. \label{e15} \end{equation} Therefore, \eqref{e14} implies $$ \frac{\mu-2}2\|(u_n, v_n)\|^2_{S^2_{1,0}(\Omega)} - \epsilon_n \|(u_n, v_n)\|_{S^2_{1,0}(\Omega)} \le \mu c + 1. $$ Consequently, $\{(u_n, v_n)\}$ is bounded in $S_{1,0}^2(\Omega)$. \end{proof} \begin{lemma} \label{lm3} Let assumption (H1) hold and $(u_n, v_n)\rightharpoonup (u,v)$ in $S_{1,0}^2(\Omega)$. Then $$ \lim_{n\to\infty} \langle \Psi^{'}(u_n, v_n), (u_n-u, v_n-v)\rangle = 0. $$ \end{lemma} \begin{proof} It suffices to prove that \begin{equation} \lim_{n\to \infty}\int_{\Omega} [f(x,y,u_n, v_n)(u_n - u) + g(x,y,u_n, v_n) (v_n - v)]dx\,dy = 0.\label{e16} \end{equation} We first show that there exists constant $M_1>0$ such that \begin{equation} \int_{\Omega} |f(x,y,u_n,v_n)|^p dx\,dy < M_1, \quad \text{for}\quad p=\frac{2N(\alpha)}{N(\alpha)+2}+\tau, \label{e17} \end{equation} if $\tau$ is positive and sufficiently small. By assumption (H1) and the fact that $pr_1, pr_2 \le \frac{2N(\alpha)}{N(\alpha)-2}-\delta(\tau)$, we have \begin{align*} \int_{\Omega} |f(x,y,u_n,v_n)|^p dx\,dy\le & \int_{\Omega}(C_1 + C_2 |u_n|^{pr_1} + C_3|v_n|^{pr_2})dx\,dy\\ \le & C(1+ \|(u_n, v_n)\|_{S^2_{1,0}(\Omega)}^{\frac{2N(\alpha)}{N(\alpha)-2} -\delta(\tau)}), \end{align*} where $C>0, \delta(\tau)$ is positive and $\delta(\tau)\to 0$ as $\tau\to 0$. Since $\{(u_n, v_n)\}$ is a bounded sequence, \eqref{e17} follows. Similarly, \begin{equation} \int_{\Omega} |g(x,y,u_n,v_n)|^q dx\,dy < M_2, \quad \text{for } q=\frac{2N(\beta)}{N(\beta)+2}+\tau, \label{e18} \end{equation} where $M_2>0$, $\tau$ is positive and sufficiently small. We are now in a position to prove \eqref{e16}. Let $p'$ be the conjugate exponent of $p$, using H\"{o}lder inequality, we have \begin{align*} &\int_{\Omega} \Big[|f(x,y,u_n, v_n)(u_n - u)| + |g(x,y,u_n, v_n)(v_n - v)|\Big] dx\,dy\\ & \le \Big(\int_{\Omega}|f(x,y,u_n, v_n)|^p dx\,dy \Big)^{1/p} \Big(\int_{\Omega}|u_n-u|^{p'}dx\,dy\Big)^{1/p'} \\ &\quad + \Big(\int_{\Omega}|g(x,y,u_n, v_n)|^q dx\,dy \Big)^{1/q} \Big(\int_{\Omega}|u_n-u|^{q'}dx\,dy\Big)^{1/q'} \\ & \le M_1^{1/p}\|u_n-u\|_{L^{p'}(\Omega)} + M_2^{1/q}\|v_n-v\|_{L^{q'}(\Omega)}\\ & \le M \|(u_n,v_n)-(u,v)\|_{L^{p',q'}(\Omega)} \end{align*} where $M>0$ and \begin{equation*} p' = \frac{2N(\alpha)+\tau[N(\alpha)+2]}{N(\alpha)-2+\tau[N(\alpha)+2]} = \frac{2N(\alpha)}{N(\alpha)-2} - \delta_1(\tau), \end{equation*} $ \delta_1(\tau) >0\quad\text{and}\quad\delta_1(\tau)\to 0\quad\text{as}\quad\tau\to 0$, \begin{equation*} q' = \frac{2N(\beta)+\tau[N(\beta)+2]}{N(\beta)-2+\tau[N(\beta)+2]} = \frac{2N(\beta)}{N(\beta)-2} - \delta_2(\tau), \end{equation*} $\delta_2(\tau) >0\quad\text{and}\quad\delta_2(\tau)\to 0\quad\text{as } \tau\to 0$. The compactness of the embedding $S^2_{1,0}(\Omega)\subset L^{\frac{2N(\alpha)}{N(\alpha)-2}-\delta_1(\tau), \frac{2N(\beta)}{N(\beta)-2}-\delta_2(\tau)}(\Omega)$ implies that there exists a subsequence, denoted also by $\{(u_n, v_n)\}$, such that $$ \|(u_n,v_n)-(u,v)\|_{L^{p',q'}(\Omega)} \to 0\quad\text{as}\quad n\to \infty. $$ The proof is complete. \end{proof} \begin{lemma}\label{lm4} If $(u_n, v_n)\rightharpoonup (u,v)$ in $S^2_{1,0}(\Omega)$ and $$ \lim_{n\to\infty} \langle \Phi' (u_n,v_n), (u_n-u, v_n-v)\rangle= 0, $$ then $(u_n, v_n)\to (u,v)$ in $S^2_{1,0}(\Omega)$. \end{lemma} \begin{proof} Let $\{(u_n, v_n)\}$ converge weakly to $(u,v)$. Denoting \begin{align*} J_n = & \int_{\Omega} [\nabla_x u\nabla_x(u_n-u) + |x|^{2\alpha}\nabla_y u \nabla_y(u_n-u)]dx\,dy \\ & + \int_{\Omega}[\nabla_x v\nabla_x(v_n-v) + |x|^{2\beta}\nabla_y v \nabla_y(v_n-v)]dx\,dy, \end{align*} we have $\lim_{n\to\infty}J_n=0$. On the other hand, \begin{align*} &\langle \Phi' (u_n,v_n), (u_n-u, v_n-v)\rangle\\ &= \int_{\Omega} [|\nabla_x (u_n-u)|^2 + |x|^{2\alpha}|\nabla_y (u_n-u)|^2]dx\,dy \\ &\quad + \int_{\Omega}[|\nabla_x (v_n-v)|^2 + |x|^{2\beta}|\nabla_y (v_n-v)|^2]dx\,dy + J_n. \end{align*} Hence $\|(u_n-u, v_n-v)\|_{S^2_{1,0}(\Omega)}\to 0$ as $n\to\infty$. The conclusion of Lemma \ref{lm4} is proved. \end{proof} \begin{lemma}\label{lm5} Let assumptions (H1) and (H2) hold. Then $I_{\lambda}$ satisfies the (PS) condition. \end{lemma} \begin{proof} Suppose that $\{(u_n, v_n)\}$ is a Palais-Smale sequence of $I_{\lambda}$. Lemma \ref{lm2} ensures that $\{(u_n, v_n)\}$ is bounded in $S^2_{1,0}(\Omega)$. Then, there exists a subsequence, denoted also $\{(u_n, v_n)\}$ converging weakly to $(u,v)$ in $S^2_{1,0}(\Omega)$. By the conclusion of Lemma \ref{lm3}, $$ \lim_{n\to\infty}\langle\Psi'(u_n, v_n), (u_n-u, v_n-v) = 0, $$ and the fact that \begin{align*} & \langle I_{\lambda}'(u_n, v_n), (u_n-u, v_n-v)\rangle\\ & = \langle \Phi'(u_n, v_n), (u_n-u, v_n-v)\rangle - \lambda\langle\Psi'(u_n, v_n), (u_n-u, v_n-v)\rangle, \end{align*} we obtain $$ \lim_{n\to\infty}\Phi'(u_n, v_n), (u_n-u, v_n-v)\rangle = 0. $$ Lemma \ref{lm4} allows us to conclude that $\{(u_n, v_n)\}$ converge strongly to $(u,v)$ in $S^2_{1,0}(\Omega)$. Hence, the functional $I_{\lambda}$ satisfies the (PS) condition. \end{proof} \section{Proof of the Existence Result} \begin{lemma}\label{lm6} Assume that the hypotheses of Theorem \ref{th1} hold. Then there exist positive numbers $\eta_{\lambda}$ and $\rho_{\lambda}$ such that $\eta_{\lambda} \to +\infty$ as $\lambda\to 0$ and, $$ I_{\lambda}(u,v) \ge \eta_{\lambda} \quad \text{for all }(u, v)\in S^2_{1,0} (\Omega) \text{ satisfying } \|(u,v)\|_{S^2_{1,0}(\Omega)}\ge\rho_{\lambda}. $$ Moreover, $I_{\lambda}(tu, tv)\to -\infty$ as $t\to +\infty$ for some $(u,v)\in S^2_{1,0}(\Omega)\backslash\{(0,0)\}$. \end{lemma} \begin{proof} Let $$ p = \frac{2N(\alpha)}{N(\alpha)+2} + \tau, \quad q = \frac{2N(\beta)}{N(\beta)+2} + \tau. $$ Using the same arguments in the proof of Lemma \ref{lm3}, we obtain \begin{equation} \begin{aligned} \int_{\Omega} |f(x,y,u,v)|^p dx\,dy & \le C_p(1+ \|(u,v)\|_{S^2_{1,0}(\Omega)}^{\frac{2N(\alpha)}{N(\alpha)-2} - \gamma_1(\tau)}), \\ \int_{\Omega} |g(x,y,u,v)|^q dx\,dy & \le C_q(1+ \|(u,v)\|_{S^2_{1,0} (\Omega)}^{\frac{2N(\beta)}{N(\beta)-2} - \gamma_2(\tau)}), \end{aligned} \label{e19} \end{equation} where $\gamma_1(\tau), \gamma_2(\tau)$ are positive and $\gamma_1(\tau), \gamma_2(\tau)\to 0$ as $\tau\to 0$. Now, from the inequalities in \eqref{e19}, one can estimate \begin{align*} &\int_{\Omega} F(x,y,u,v)dx\,dy \\ &\le C \int_{\Omega}[uf(x,y,u,v) + vg(x,y,u,v)]dx\,dy\\ \le & C \Big(\int_{\Omega}|f(x,y,u,v)|^p dx\,dy\Big)^{1/p} \Big(\int_{\Omega}|u|^{p'}dx\,dy\Big)^{1/p'} \\ & + C \Big(\int_{\Omega}|g(x,y,u,v)|^q dx\,dy\Big)^{1/q} \Big(\int_{\Omega}|v|^{q'}dx\,dy\Big)^{1/q'}\\ \le & C \Big(1+ \|(u,v)\|_{S^2_{1,0}(\Omega)}^{\frac{2N(\alpha)}{N(\alpha)-2} - \gamma_1(\tau)}\Big)^{1/p}\|u\|_{L^{p'}(\Omega)} \\ & + C \Big(1+ \|(u,v)\|_{S^2_{1,0}(\Omega)}^{\frac{2N(\beta)}{N(\beta)-2} - \gamma_2(\tau)}\Big)^{1/q}\|v\|_{L^{q'}(\Omega)}\\ \le & C \Big[\Big(1+ \|(u,v)\|_{S^2_{1,0}(\Omega)}^{\frac{2N(\alpha)} {N(\alpha)-2} - \gamma_1(\tau)}\Big)^{1/p} + \Big(1+ \|(u,v)\|_{S^2_{1,0}(\Omega)}^{\frac{2N(\beta)}{N(\beta)-2} - \gamma_2(\tau)}\Big)^{1/q}\Big]\\ \quad & \times (\|u\|_{L^{p'}(\Omega)} + \|v\|_{L^{q'}(\Omega)}).\\ \le & C \Big[\Big(1+ \|(u,v)\|_{S^2_{1,0}(\Omega)}^{\frac{2N(\alpha)} {N(\alpha)-2} - \gamma_1(\tau)}\Big)^{1/p} + \Big(1+ \|(u,v)\|_{S^2_{1,0}(\Omega)}^{\frac{2N(\beta)}{N(\beta)-2} - \gamma_2(\tau)}\Big)^{1/q}\Big]\|(u,v)\|_{S^2_{1,0}(\Omega)}, \end{align*} for some positive constant $C$. Note that, the Young inequality gives $$ A^{\frac 1p}\le \frac 1q + \frac 1p A,\,\, B^{\frac 1q}\le \frac 1p + \frac 1q B\,\text{for} \, A,B>0. $$ From these facts, we have $$ \int_{\Omega} F(x,y,u,v)dx\,dy \le C^*_1 + C^*_2\|(u,v)\|_{S^2_{1,0} (\Omega)}^{\frac{2N(\alpha)}{N(\alpha)-2} +1 - \gamma_1(\tau)} + C^*_3\|(u,v)\|_{S^2_{1,0}(\Omega)}^{\frac{2N(\beta)}{N(\beta)-2} + 1 - \gamma_2(\tau)}, $$ for some $C^*_1, C^*_2, C^*_3 >0$. Using the last inequality and taking \eqref{e10} into account, we get \begin{equation} \begin{aligned} I_{\lambda}(u,v) \ge & \frac 12 \|(u,v)\|_{S^2_{1,0}(\Omega)}^2 \\ & - \lambda [C^*_1 + C^*_2\|(u,v)\|_{S^2_{1,0}(\Omega)}^{\frac{2N(\alpha)} {N(\alpha)-2} +1 - \gamma_1(\tau)} + C^*_3\|(u,v)\|_{S^2_{1,0}(\Omega)} ^{\frac{2N(\beta)}{N(\beta)-2} + 1 - \gamma_2(\tau)}]. \end{aligned}\label{e19b} \end{equation} Choosing $(u,v)\in S^2_{1,0}(\Omega)$ such that $\|(u,v)\|_{S^2_{1,0}(\Omega)}=\lambda^{-s}$, with $s$ satisfying $$ 00$, there exists $K(M)>0$ such that $$ F(x,y,s,t)\ge M(s^2 + t^2), \quad \text{for all } s,t\in \mathbb{R} \text{ satisfying } |s|+|t|\ge K(M). $$ Let $(u,v)\in S^2_{1,0}(\Omega)$ with $\|(u,v)\|_{S^2_{1,0}(\Omega)}=1$ and $\int_{\Omega}(u^2+v^2)dx\,dy=a$. Then $$ I_{\lambda}(tu,tv) = \frac 12 t^2 - \lambda \int_{\Omega} F(x,y,tu,tv)dx\,dy $$ and \begin{equation} \int_{\Omega} F(x,y,tu,tv)dx\,dy\ge Mt^2\int_{\Omega\cap\{(x,y)\in\Omega:|u|+|v|\ge\frac{K(M)}t\}}(u^2+v^2)dx\,dy-b, \label{e20} \end{equation} where $b$ is a constant depending on $a$. For $t$ sufficient large, $$ \int_{\Omega\cap\{(x,y)\in\Omega: |u|+|v|\ge\frac{K(M)}t\}}(u^2+v^2)dx\,dy \ge \frac 12 a. $$ From this and \eqref{e20}, we arrive at the conclusion $$ I_{\lambda}(tu,tv) \le \frac 12 t^2 - \frac 12 a \lambda Mt^2 + b, $$ for $t$ sufficient large. Choosing $M=\frac 2{a\lambda}$ leads to $$ I_{\lambda}(tu,tv) \le b-\frac 12 t^2,\; \text{for $t$ sufficient large}. $$ Hence, $I_{\lambda}(tu,tv)\to -\infty$ as $t\to+\infty$. \end{proof} \begin{proof}[Proof of Theorem \ref{th1}] By Lemma \ref{lm5} and Lemma \ref{lm6}, we may apply the Mountain Pass Theorem \cite{ref8}. It follows that there exists $\lambda^*>0$ such that for all $\lambda\in (0, \lambda^*)$, the functional $I_{\lambda}$ has a critical point $(u_{\lambda}, v_{\lambda})$ satisfying $I_{\lambda}(u_{\lambda}, v_{\lambda})> \eta_{\lambda} > 0$ and $\|(u_{\lambda}, v_{\lambda})\|_{S^2_{1,0}(\Omega)}\ge\rho_{\lambda} =\lambda^{-s}\to+\infty$ as $\lambda \to 0$. \end{proof} \subsection*{Remark} (1)\; The operator $G_s$ can be extended to the more complicated form $$ \Delta_x + |x|^{2s_1}\Delta_y + |x|^{2s_2}\Delta_z, $$ in the domain $\Omega\subset\mathbb{R}^{N_1}\times\mathbb{R}^{N_2}\times \mathbb{R}^{N_3}$. Following \cite{ref10}, the critical exponent for this case is $$ \frac{N_1 + (s_1+1)N_2 + (s_2 + 1)N_3 + 2}{N_1 + (s_1+1)N_2 + (s_2 + 1)N_3 - 2}. $$ Generally, $G_s$ has the form $$ \Delta_{\omega_0} + \sum_{i=1}^m |\omega_0|^{2s_i}\Delta_{\omega_i}, $$ in the domain $\Omega=\{(\omega_0,\omega_1,\dots ,\omega_m)\}\subset \prod_{i=0}^m \mathbb R^{N_i}$. The associated critical exponent is given by $$ \frac{N_0 + \sum_{i=1}^m (s_i+1)N_i + 2}{N_0 + \sum_{i=1}^m (s_i+1)N_i - 2}. $$ Putting $\omega=(\omega_0,\omega_1,\dots ,\omega_m)$, we can preserve the hypotheses (H1)-(H3) and proceed with the functional \begin{align*} I_{\lambda} (u,v) = &\frac 12 \int_{\Omega} (|\nabla_{\omega_0}u|^2 + \sum_{i=1}^m|\omega_0|^{2s_i}|\nabla_{\omega_i}u|^2) d\omega \\ & +\frac 12 \int_{\Omega}(|\nabla_{\omega_0}v|^2 + \sum_{i=1}^m|\omega_0|^{2s_i}|\nabla_{\omega_i}v|^2)d\omega - \lambda\int_{\Omega}F(\omega,u,v) d\omega. \end{align*} (2) Using the same argument used above, we can deal with the system of $m$ unknowns \begin{gather*} L U = \lambda \nabla F \quad \text{in } \Omega, \\ U = 0 \quad \text{on } \partial \Omega, \end{gather*} with $U=(u_1,u_2,\dots ,u_m)$ $F=F(x,y,u_1,u_2,\dots ,u_m)$ and $$ L = \begin{pmatrix} G_{s_1} & 0 & \dots & 0 \\ 0 & G_{s_2} & \dots & 0 \\ \vdots & \vdots & \dots &\vdots \\ 0 & 0 & \dots & G_{s_m} \end{pmatrix}. $$ For this system, hypotheses (H1), (H2) and (H3) are replaced by \begin{itemize} \item[(H1')] All components of $\nabla F$ are Caratheodory functions satisfying \begin{gather*} \Big|\frac{\partial F}{\partial u_i}(x,y,U) \Big| \le C_{i0} + \sum_{j=1}^m C_{ij}|u_i|^{r_{ij}},\\ 0 \le r_{ij} < \frac{N(s_i)+2}{N(s_i)-2}, \quad i=1..n, \; j=1..m, \end{gather*} for a.e, $(x,y)\in \Omega$ and for all $U\in \mathbb{R}^n$. \item[(H2')] For a.e. $(x,y)\in\Omega$ and for all $U\in \mathbb{R}^n$ satisfying $|U|\ge M$, $F(x,y,0)=0$ and $0<\mu F\le \nabla F.U$, where $\mu, M$ are real numbers, $\mu >2$ and $M>0$. \item[(H3')] $F(x,y,U)$ is superlinear, i.e. $\lim_{|U|\to\infty}\frac{F(x,y,U)}{|U|^2}=+\infty$. The associated functional is represented by $$ I_{\lambda} (U) = \frac 12\int_{\Omega} \Big[\sum_{i=1}^m(|\nabla_x u_i|^2 + |x|^{2s_i}|\nabla_y u_i|^2)\Big]dx\,dy - \lambda \int_{\Omega} F(x,y,U)dx\,dy. $$ \end{itemize} \section{Some Special Cases of Nonlinearity} \subsection*{Homogeneous cases} Let $q\in\mathbb{R}$, $q>1$. The potential function $F(x,y,u,v)$ is called $q$-homogeneous in $(u,v)$ if $F(x,y,tu,tv)=t^q F(x,y,u,v)$ for a.e. $(x,y)\in\Omega$, for all $t>0$ and $(u,v)\in \mathbb{R}^2$. Assume that $F(x,y,u,v)\ge 0$ and $F$ is $q$-homogeneous in $(u,v)$. Furthermore, for fixed $(x,y)\in \Omega, F(x,y,.,.)\in C^1(\mathbb{R}^2)$ and for fixed $(u,v)\in \mathbb{R}^2$, $F(.,.,u,v)\in L^{\infty}(\Omega)$. Then the following properties of $F(x,y,u,v)$ are verified: \begin{enumerate} \item For a.e. $(x,y)\in \Omega$ and for all $(u,v)\in \mathbb{R}^2$, \begin{equation} m (|u|^q + |v|^q)\le F(x,y,u,v) \le M (|u|^q + |v|^q), \label{e21} \end{equation}\label{e21b} where \begin{gather} M= \mathop{\rm esssup}_{(x,y)\in\Omega} \max_{(u,v)\in\mathbb{R}^2} \{F(x,y,u,v): |u|^q + |v|^q=1\},\label{e21c}\\ m= \mathop{\rm essinf}_{(x,y)\in\Omega} \min_{(u,v)\in\mathbb{R}^2} \{F(x,y,u,v): |u|^q + |v|^q=1\}.\label{e21d} \end{gather} \item For all $(u,v)\in \mathbb{R}^2$ and a.e. $(x,y)\in \Omega$, \begin{equation} u\frac{\partial F(x,y,u,v)}{\partial u} + v\frac{\partial F(x,y,u,v)}{\partial v} = q F(x,y,u,v).\label{e22} \end{equation} \item \begin{equation} \text{$\nabla F$ is $(q-1)$- homogeneous in $(u,v)$.} \label{e23} \end{equation} \end{enumerate} It is easy to see that, for $q>2$, the condition (H2) is followed from \eqref{e22}, the {\it $q$-homogeneity} of $F$ implies the condition (H3). Moreover, we deduce from properties \eqref{e23} and \eqref{e21} that \begin{gather*} \Big|\frac{\partial F(x,y,u,v)}{\partial u}\Big|\le M_1 (|u|^{q-1} + |v|^{q-1}), \\ \Big|\frac{\partial F(x,y,u,v)}{\partial v}\Big|\le M_2 (|u|^{q-1} + |v|^{q-1}), \end{gather*} where $M_1, M_2$ are positive constants. Then, the condition \begin{equation} 20,\label{e28} \end{equation} for $\lambda$ sufficiently small. On the other hand, under the assumptions on $G$ and $H$, the following inequalities hold for all $(u,v)\in \mathbb{R}^2, (x,y)\in \Omega$: \begin{equation} \begin{gathered} m_G(|u|^p + |v|^p) \le G(x,y,u,v)\le M_G(|u|^p + |v|^p),\\ m_H(|u|^q + |v|^q) \le H(x,y,u,v)\le M_H(|u|^q + |v|^q), \end{gathered} \end{equation} where $M_G, M_H, m_G$ and $m_H$ are defined in \eqref{e21c} and \eqref{e21d}. This fact allows us to estimate \begin{align*} &I_{\lambda}(tu,tv)\\ & \le \frac 12 t^2\|(u,v)\|_{S^2_{1,0}(\Omega)} - t^pm_G\int_{\Omega}(|u|^p+|v|^p)dx\,dy -t^qm_H\int_{\Omega}(|u|^q+|v|^q)dx\,dy, \end{align*} for all $t>0$. Taking $(u,v)\in S^2_{1,0}(\Omega)$, $\|(u,v)\|_{S^2_{1,0}(\Omega)}=1$ we have \begin{equation*} I_{\lambda}(tu,tv) \le C_1 t^2 - C_2 t^p - C_3 t^q, \end{equation*} where $C_1, C_2$ and $C_3$ are positive constants. Since $1