\documentclass[reqno]{amsart} \usepackage{hyperref} \AtBeginDocument{{\noindent\small \emph{Electronic Journal of Differential Equations}, Vol. 2011 (2011), No. 30, pp. 1--12.\newline ISSN: 1072-6691. URL: http://ejde.math.txstate.edu or http://ejde.math.unt.edu \newline ftp ejde.math.txstate.edu} \thanks{\copyright 2011 Texas State University - San Marcos.} \vspace{9mm}} \begin{document} \title[\hfilneg EJDE-2011/30\hfil Degenerate and singular elliptic systems] {Existence of infinitely many solutions for degenerate and singular elliptic systems with indefinite concave nonlinearities} \author[N. T. Chung\hfil EJDE-2011/30\hfilneg] {Nguyen Thanh Chung} \address{Nguyen Thanh Chung \newline Department of Mathematics and Informatics, Quang Binh University, 312 Ly Thuong Kiet, Dong Hoi, Quang Binh, Vietnam} \email{ntchung82@yahoo.com} \thanks{Submitted May 14, 2010. Published February 18, 2011.} \subjclass[2000]{35J65, 35J20} \keywords{Degenerate and singular Elliptic system; weight function; \hfill\break\indent concave nonlinearity; infinitely many solutions} \begin{abstract} In this article, we consider degenerate and singular elliptic systems of the form \begin{gather*} - \operatorname{div}(h_1(x)\nabla u) = b_1(x)|u|^{r-2}u + F_u(x,u,v) \quad \text{in } \Omega,\\ - \operatorname{div}(h_2(x)\nabla v) = b_2(x)|v|^{r-2}v + F_v(x,u,v) \quad \text{in } \Omega, \end{gather*} where $\Omega$ is a bounded domain in $\mathbb{R}^N$, $N \geq 2$, with smooth boundary $\partial\Omega$; $h_i: \Omega \to [0, \infty)$, $h_i \in L^1_{\rm loc}(\Omega)$, and are allowed to have ``essential'' zeroes; $1 < r < 2$; the weight functions $b_i: \Omega \to \mathbb{R}$, may be sign-changing; and $(F_u,F_v) = \nabla F$. Using variational techniques, a variant of the Caffarelli - Kohn - Nirenberg inequality, and a variational principle by Clark \cite{Clark}, we prove the rxistence of infinitely many solutions in a weighted Sobolev space. \end{abstract} \maketitle \numberwithin{equation}{section} \newtheorem{theorem}{Theorem}[section] \newtheorem{lemma}[theorem]{Lemma} \newtheorem{remark}[theorem]{Remark} \newtheorem{definition}[theorem]{Definition} \allowdisplaybreaks \section{Introduction and Preliminaries} In this article, we are concerned with a class of degenerate and singular elliptic systems of the form \begin{equation}\label{e1.1} \begin{gathered} - \operatorname{div}(h_1(x)\nabla u) = b_1(x)|u|^{r-2}u + F_u(x,u,v) \quad \text{in } \Omega,\\ - \operatorname{div}(h_2(x)\nabla v) = b_2(x)|v|^{r-2}v + F_v(x,u,v) \quad \text{in } \Omega, \end{gathered} \end{equation} where $\Omega \subset \mathbb{R}^N$, with $N \geq 2$, is a bounded domain with smooth boundary $\partial\Omega$, and $(F_u,F_v) = \nabla F$. We point out that if $h_1(x) = h_2(x) \equiv 1$, the problem has been intensively studied; we refer to the interesting works \cite{AdriHami, AmbBreCer, BartWill, Guo, Wang, Wu1,Wu2}. In \cite{AdriHami,AmbBreCer, BartWill, Wang}, the authors considered \eqref{e1.1} with concave-convex nonlinearities in the case when the functions $b_i(x)$, $i = 1, 2$, are positive constants. Some existence and multiplicity results were obtained provided that the nonlinear term $f$ satisfies some global assumptions for all $x$ and $u$. A typical example of $f$ satisfying those global assumptions is $f(x,u) = |u|^{p-2}u$ with $2 < p \leq 2^\star = 2N/(N-2)$ and $N \geq 3$. When $b_i$, $i = 1, 2$ are sign-changing weighted functions, the problem was studied by Wu \cite{Wu1,Wu2}. There, with the help of the Nehari manifold, the author proved that the problem has at least two nontrivial nonnegative solutions, under some suitable conditions on the nonlinearities. In a recent paper Caldiroli et al. \cite{CaldMusi} considered the Dirichlet elliptic problem \begin{equation}\label{e:1.2} -\operatorname{div}(h(x)\nabla u) = \lambda u + g(x,u) \quad\text{in } \Omega, \end{equation} where $\Omega$ is a (bounded or unbounded) domain in $\mathbb{R}^N$ ($N\geq 2$), and $h$ is a nonnegative measurable weighted function that is allowed to have ``essential'' zeroes at some points in $\Omega$; i.e., the function $h$ can have at most a finite number of zeroes in $\Omega$. More precisely, the authors assumed that: \begin{itemize} \item[(H)] The function $h: \Omega \to [0, \infty)$ belongs to $L^1_{\rm loc}(\Omega)$ and there exists a constant $\phi \geq 0$ such that $$ \liminf_{x \to z} |x - z|^{-\phi}h(x) > 0 \quad \text{for all } z \in \overline\Omega. $$ \end{itemize} Thus, the function $h$ decreases more slowly than $|x - z|^\alpha$ near every point $z \in h^{-1}\{0\}$. It should be observed that a model example for such function is that $h(x) = |x|^{\alpha}$, (see \cite{Mihai, MihaRadu}). The case $\alpha = 0$ covers the ``isotropic'' case corresponding to the Laplacian operator. Caldiroli et al. \cite{CaldMusi} proved that if a function $h$ satisfies the condition (H), then there exist a finite set $Z = \{z_1, z_2, \dots, z_k\} \subset \overline \Omega$ and numbers $r, \delta > 0$ such that the balls $B_i = B_{r}(z_i)$ $(i = 1, 2, \dots,k)$ are mutually disjoint and \begin{gather*} h(x) \geq \delta |x-z_i|^{\alpha} \quad \forall x \in B_i, \quad i =1, 2, \dots,k, \\ h(x) \geq \delta \quad \forall x \in \overline\Omega \backslash \cup_{i = 1}^k B_i. \end{gather*} This says that the elliptic operators in system \eqref{e1.1} may be degenerate and singular. Such problems come from the consideration of standing waves in anisotropic Schr\"odinger systems. They arise in many areas of applied physics, including nuclear physics, field theory, solid waves and problems of false vacuum. These problems are introduced as models for several physical phenomena related to equilibrium of continuous media which somewhere be perfect insulators (see \cite[p. 79]{DautLion}). For more information and connection on problems of this type, the readers may consult in \cite{MurtStam, Stre} and the references therein. Regarding the nonlinear term $g(x,u)$, Caldiroli et al. assumed that $g: \Omega \times \mathbb{R} \to \mathbb{R}$ is a Carath\'eodory function satisfying the following conditions: \begin{itemize} \item[(G1)] $|g(x,u)| = O(|u|^{p-1})$ as $|u| \to \infty$, uniformly in $x \in \Omega$, where $2 < p < 2^\star_\phi = \frac{2N}{N -2+\phi}$, $\phi \in (0,2)$; \item[(G2)] $g(u) = o(u)$ as $|u| \to 0$, uniformly in $x\in \Omega$; \item[(G3)] There is $\mu > 2$, such that $$ 0 < \mu G(x,u) : = \int_0^u g(x,s)ds \leq g(x,u)u $$ uniformly in $x \in \Omega$, and for all $u \in \mathbb{R} \backslash \{0\}$. \end{itemize} By introducing some interesting results, using the mountain pass theorem \cite{AmbRab}, Caldiroli et al. obtained in \cite[Theorem 4.4]{CaldMusi} the existence of a nontrivial solution for \eqref{e:1.2} in a suitable function space, provided that $\lambda < \lambda_1(h)$, where $$ \lambda_1(h) : = \inf_{u \in H^1_0(\Omega) \backslash \{0\}} \frac{\int_\Omega h(x)|\nabla u|^2dx}{\int_\Omega|u|^2dx}. $$ The results in \cite{CaldMusi} were used by Zographopoulos \cite{Zog}, Zhang et al. \cite{ZhangWang} and Chung et al. \cite{ChungToan, Chung} to study the existence of solutions for a class of degenerate elliptic systems. Zographopoulos \cite{Zog} considered the degenerate semilinear elliptic system \begin{equation}\label{e:1.3} \begin{gathered} - \operatorname{div}(h_1(x)\nabla u) = \lambda \mu (x) |u|^{\gamma-1}|v|^{\delta+1}u \quad \text{in } \Omega \\ - \operatorname{div}(h_2(x)\nabla v) = \lambda \mu (x) |u|^{\gamma+1}|v|^{\delta-1}v \quad \text{in } \Omega \\ u = v = 0 \quad \text{on } \partial \Omega, \end{gathered} \end{equation} where the functions $h_i \in L^1_{\rm loc}(\Omega)$ and $h_i$ ($i = 1, 2$) are allowed to have ``essential'' zeroes at some points in $\Omega$, the function $\mu \in L^\infty(\Omega)$ and may change sign in $\Omega$, $\lambda$ is a positive parameter and the nonnegative constants $\gamma$, $\delta$ satisfy the following conditions: \begin{gather*} \gamma + 1 < p < 2^\star_\alpha, \quad \delta + 1 < q < 2^\star_\beta, \\ \frac{\gamma+1}{p} + \frac{\delta+1}{q} = 1, \quad \frac{\gamma+1}{2^\star_\alpha} + \frac{\delta+1}{2^\star_\beta} < 1, \\ 2^\star_\alpha = \frac{2N}{N-2+\alpha}, \quad 2^\star_\beta = \frac{2N}{N-2+\beta}, \quad \alpha, \beta \in (0,2). \end{gather*} Using arguments of Mountain pass type \cite{AmbRab}, the author showed the existence of a nontrivial solution of \eqref{e:1.3} in the supercritical case; i.e., \begin{equation}\label{e:1.4} \frac{\gamma+1}{2} + \frac{\delta+1}{2} > 1. \end{equation} In the critical case $\gamma = \delta = 0$, the author also established the existence of a positive principal egienvalue $\lambda_1$ for system \eqref{e:1.3} and some of its pertubations. Motivated by the results in \cite{BoccDeFi, CaldMusi, Cos, StavZogr, Zog}, Chung \cite{Chung} and Zhang et al. \cite{ZhangWang} obtained some existence results for \eqref{e1.1} under subcritical growth conditions and the primitive $F(x,u,v)$ being intimately related to with the first eigenvalue of a corresponding linear system. Finally, in the case when $\Omega$ is a bounded domain with smooth boundary, Chung et al. \cite{ChungToan} obtained the nonexistence and multiplicity of solutions for \eqref{e1.1} using the minimum principle combined with the mountain pass theorem \cite{AmbRab}. In the present paper, we consider problem \eqref{e1.1} with the degenerate potentials as in \cite{CaldMusi,ChungToan,Chung,ZhangWang, Zog}; i.e., $h_i: \Omega \to [0, \infty)$, $h_i \in L^1_{\rm loc}(\Omega)$, $h_i$ ($i = 1, 2$) are allowed to have ``essential'' zeroes at some points in $\Omega$. The problem will be investigated under the case $1 < r < 2$ and the weight functions $b_i: \Omega \to \mathbb{R}$, $i = 1, 2$, may be possibly sign-changing. Motivated by the interesting ideas in \cite{Guo,Wang}, we dot not require the nonlinear term $f$ satisfying any global assumptions for all $u$ as in \cite{CaldMusi,ChungToan, Chung, Wu1,Wu2, ZhangWang, Zog}. Thus, the result introduced here is a complete natural extension of the previous ones. In order to overcome the difficulties brought, we will use variational techniques rely essentially on a variant of the Caffarelli - Kohn - Nirenberg inequality in \cite{CaldMusi} combined with a variational principle by Clark \cite{Clark}, we prove the problem has infinitely many solutions in a weighted Sobolev space. As we mentioned above, throughout this paper, we assume that the functions $h_1$ and $h_2$ satisfy the following conditions: \begin{itemize} \item[(H1)] The function $h_1: \Omega \to [0, \infty)$ belongs to $L^1_{\rm loc}(\Omega)$ and there exists a constant $\alpha \geq 0$ such that $$ \liminf_{x \to z} |x - z|^{-\alpha}h_1(x) > 0 \quad \text{for all } z \in \overline\Omega. $$ \item[(H2)] The function $h_2: \Omega \to [0, \infty)$ belongs to $L^1_{\rm loc}(\Omega)$ and there exists a constant $\beta \geq 0$ such that $$ \liminf_{x \to z} |x - z|^{-\beta}h_2(x) > 0 \quad\text{for all } z \in \overline\Omega. $$ \end{itemize} Next, in order to state our main result, we propose some hypotheses on the nonlinearities as follows: \begin{itemize} \item[(B)] The functions $b_i: \Omega \to \mathbb{R}$, $i = 1, 2$, are continuous and there is a nonempty open subset $\Omega'$ of $\Omega$ such that $b_i(x) > 0$ for a.e. $x \in \Omega$; \item[(F1)] There are two positive constants $\rho_1$ and $\rho_2$, such that $F(x,u,v)$ is a $C^1$-function on $\Omega \times (-\rho_1,\rho_1) \times (-\rho_2,\rho_2)$, $\nabla F = (F_u,F_v)$, $F_u, F_v \in C(\Omega \times (-\rho_1,\rho_1) \times (-\rho_2,\rho_2), \mathbb{R})$, and $F(x,-u,-v) = F(x,u,v)$ for all $(u,v) \in \times (-\rho_1,\rho_1) \times (-\rho_2,\rho_2)$ and a.e. $x \in \Omega$; \item[(F2)] It holds that $$ \lim_{|u| \to 0}\frac{F_u(x,u,v)}{|u|^\gamma|v|^{\delta+1}} = 0,\quad \lim_{|v| \to 0}\frac{F_v(x,u,v)}{|s|^{\gamma+1}|v|^\delta} = 0 $$ uniformly for $x \in \Omega$, in which the positive constants $\gamma$ and $\delta$ are chosen such that $\frac{\gamma+1}{p} + \frac{\delta+1}{q} = 1$, $\frac{\gamma+1}{2^\star_\alpha} + \frac{\delta+1}{2^\star_\beta} < 1$, and $\gamma + 1 < p < 2^\star_\alpha = \frac{2N}{N-2+\alpha}$, $\delta + 1 < q< 2^\star_\beta = \frac{2N}{N-2+\beta}$, $\alpha, \beta \in (0,2)$. \end{itemize} We find that the condition (B) says the weight functions $b_i$, $i = 1, 2$ may being sign-changing in $\Omega$ while the conditions (F1) and (F2) say the assumptions imposed on the nonlinearities stisfies only for $u$ and $v$ small enough. Moreover, we do not require the Ambrosetti-Rabinowitz type condition as in \cite{CaldMusi} (see (G3)). It is clear that by the presence of the functions $h_1, h_2$, the solutions of system \eqref{e1.1} must be found in a suitable space. To this purpose, we define the Hilbert spaces $H^1_0(\Omega,h_1)$ and $H^1_0(\Omega,h_2)$ as the closures of $C^\infty_0(\Omega)$ with respect to the norms $$ \|u\|_{h_1} = \Big(\int_\Omega h_1(x)|\nabla u|^2 dx\Big)^{1/2} $$ for all $u \in C^\infty_0(\Omega)$ and $$ \|v\|_{h_2} = \Big(\int_\Omega h_2(x)|\nabla v|^2 dx\Big)^{1/2} $$ for all $v \in C^\infty_0(\Omega)$, respectively, and set $$ H = H^1_0(\Omega,h_1) \times H^1_0(\Omega,h_2) = \{w = (u,v): u \in H^1_0(\Omega,h_1), \; v \in H^1_0(\Omega, h_2) \}. $$ Then, it is clear that $H$ is a Hilbert space under the norm $$ \|w\|_H = \|u\|_{h_1} + \|v\|_{h_2} $$ for all $w = (u,v) \in H$, and with respect to the scalar product $$ \langle {\varphi,\psi} \rangle_{H} = \int_{\Omega}(h_1(x)\nabla \varphi_1\cdot\nabla \psi_1 + h_2(x)\nabla \varphi_2 \cdot\nabla \psi_2) dx $$ for all $\varphi = (\varphi_1, \varphi_2), \psi =(\psi_1, \psi_2) \in H$. The key in our arguments is the following lemma, which is introduced by Caldiroli et al. \cite{CaldMusi} as the generalization of the Caffarelli - Kohn - Nirenberg inequality. \begin{lemma}[{\cite[Proposition 2.5]{CaldMusi}}] \label{bd1.1} Let $\Omega$ be a bounded domain in $\mathbb{R}^N$, $N \geq 2$. Assume that the function $h : \Omega \to [0, +\infty)$ the condition (H), then there exists a constant $C_\phi > 0$ depending on $\phi$ such that $$ \Big(\int_\Omega|\varphi|^{2^\star_{\phi}}dx\Big)^{2/2^\star_{\phi}} \leq C_\phi\int_\Omega h(x)|\nabla \varphi|^2dx $$ for every $\varphi \in C^{\infty}_0(\Omega)$, where $2^{\star}_{\phi} = 2N/(N-2+\phi)$. \end{lemma} By Lemma \ref{bd1.1}, \cite[Propositions 3.2 and 3.4]{CaldMusi}, we have the following remark, which helps us to overcome the lack of compactness. \begin{remark}\label{cy1.2} \rm Assume that the hypotheses (H1) and (H2) are satisfied, then we conclude that \begin{itemize} \item[(i)] The embedding $H \hookrightarrow L^{2^\star_\alpha}(\Omega) \times L^{2^\star_\beta}(\Omega)$ is continuous. \item[(ii)] The embedding $H \hookrightarrow L^i(\Omega) \times L^j(\Omega)$ is compact for all $i \in [1, 2^\star_\alpha)$ and all $j \in [1, 2^\star_\beta)$. \end{itemize} \end{remark} \begin{definition}\label{dn1.3} \rm We say that $w = (u, v) \in H$ is a weak solution of \eqref{e1.1} if \begin{align*} &\int_{\Omega}(h_1(x)\nabla u\cdot \nabla \varphi_1 + h_2(x)\nabla v\cdot\nabla \varphi_2) dx -\int_\Omega (b_1(x)|u|^{r-2}u\varphi_1+ b_2(x)|v|^{r-2}v\varphi_2) dx \\ &- \int_{\Omega} (F_u(x,u,v)\varphi_1 + F_v(x,u,v)\varphi_2) dx = 0 \end{align*} for all $\varphi = (\varphi_1,\varphi_2) \in C^{\infty}_0(\Omega, \mathbb{R}^2)$. \end{definition} \begin{theorem}\label{dl1.4} Let $1 < r < 2$ and assume that the conditions {\rm (H1)--(H2), (B), (F1), (F2)} are satisfied. Then \eqref{e1.1} has a sequence of weak solutions $w_m = (u_m, v_m) \in H$, such that $\|w_m\|_{L^\infty(\Omega,\mathbb{R}^2)} \to 0$ as $m \to \infty$. Moreover, $J(w_m) < 0$ for all $m$ and $J(w_m) \to 0$ as $m \to \infty$, where \begin{align*} J(w_m) &= \frac{1}{2}\int_\Omega (h_1(x)|\nabla u_m|^2 + h_2(x)|\nabla v_m|^2)dx \\ &\quad - \frac{1}{r}\int_\Omega (b_1(x)|u_m|^r + b_2(x)|v_m|^r) dx - \int_\Omega F(x,u_m,v_m)dx. \end{align*} \end{theorem} It should be noticed that in \cite{ChungToan}, the authors had to require the condition $\gamma, \delta > 1$, which helps them to show the associated functional having the mountain pass geometry. In this article, we do not need this condition. So, our idea is to obtain the solutions of system \eqref{e1.1} using a variational principle by Clark \cite{Clark} which is stated in the following lemma. \begin{lemma}[\cite{Clark}]\label{bd1.5} Let $\Phi \in C^1(X,\mathbb{R})$ where $X$ is a Banach space. Assume that $\Phi$ satisfies the Palais - Smale condition, is even and bounded from below, and $\Phi(0) = 0$. If for any $k \in \mathbb N$, there exists a $k$-dimensional subspace $X_k$ and $\rho_k > 0$ such that $$ \sup_{X_k \cap S_{\rho_k}} \Phi < 0, $$ where $S_\rho = \{w \in X: \|w\| = \rho\}$, then $\Phi$ has a sequence of critical values $c_k < 0$ satisfying $c_k \to 0$ as $k \to \infty$. \end{lemma} \section{Proof of the Main result} Let $\lambda_1$ be the first eigenvalue of the following Dirichlet problem (see \cite[Lemma 2.3]{ZhangWang}, for $\mu(x) \equiv 1$, or \cite{ChungToan, Zog}), \begin{gather*} - \operatorname{div}(h_1(x)\nabla u) = \lambda |u|^{\gamma-1}|v|^{\delta+1}u \quad \text{in } \Omega, \\ - \operatorname{div}(h_2(x)\nabla v) = \lambda |u|^{\gamma+1}|v|^{\delta-1}v \quad \text{in } \Omega, \\ u = v = 0 \quad \text{on } \partial \Omega, \end{gather*} where the functions $h_1(x)$ and $h_2(x)$ as in (H1) and (H2), $\gamma$ and $\delta$ are two positive real numbers satisfying the condition (F2). Then, we have $\lambda_1 > 0$ and it is given by \begin{equation}\label{e2.1} \lambda_1 = \inf_{ w = (u, v) \in H \backslash\{(0,0)\}} \frac{\int_{\Omega}(\frac{\gamma+1}{p}h_1(x)|\nabla u|^2 + \frac{\delta +1}{q}h_2(x)|\nabla v|^2)dx} {\int_{\Omega}|u|^{\gamma+1}|v|^{\delta+1} dx} \end{equation} and the associated eigenfunction $w_0 = (u_0, v_0)$ is componentwise nonnegative and is unique (up to multiplication by a nonzero scalar). We first modify $F, F_u,, F_v$ so that the nonlinearities are defined for all $(x,u) \in \Omega \times \mathbb{R}$. \begin{lemma}\label{bd2.1} Assume that the hypotheses (F1) and (F2) are satisfied. Then, for any $\lambda \in (0,\lambda_1)$, there exist two constants $\rho'_1 \in (0,\frac{\rho_1}{2})$, $\rho'_2 \in (0, \frac{\rho_2}{2})$, and a function $\hat{F}(x,u,v)$ is of $C^1$ on $\Omega \times \mathbb{R} \times \mathbb{R}$, odd in $(u,v)$, such that \begin{equation}\label{e2.2} \hat{F}_u(x,u,v) = \frac{\partial \hat{F}}{\partial u}(x,u,v) = F_u(x,u,v), \quad \hat{F}_v(x,u,v) = \frac{\partial \hat{F}}{\partial v }(x,u,v) = F_v(x,u,v), \end{equation} for all $|u| \leq \rho'_1$ and $|v| \leq \rho'_2$, \begin{equation} \hat{F}(x,u,v)u + \hat{F}_v(x,u,v)v - r \hat{F}(x,u,v) \leq \frac{(2-r)\lambda}{2}|u|^{\gamma+1}|v|^{\delta+1}, \label{e2.3} \end{equation} for $(x,u,v) \in \Omega \times \mathbb{R}$, and \begin{equation} |\hat{F}(x,u,v)| \leq \frac{\lambda}{2}|u|^{\gamma+1}|v|^{\delta+1}, \quad (x,u,v) \in \Omega \times \mathbb{R}. \label{e2.4} \end{equation} \end{lemma} \begin{proof} For any $\lambda \in (0, \lambda_1)$ and $r \in (1, 2)$ we set $\theta = \frac{(2-r)\lambda}{2}$ and choose $\epsilon \in (0, \frac{\theta}{24})$. By the hypothesis (F2), there exist two positive constants $\rho'_1$ and $\rho'_2$, such that for any $|u| \leq 2\rho'_1$, and $|v| \leq 2 \rho'_2$, we have \begin{equation}\label{e2.5} |F_u(x,u,v)u| \leq \epsilon |u|^{\gamma+1}|v|^{\delta+1} \text{ and } |F_v(x,u,v)u| \leq \epsilon |u|^{\gamma+1}|v|^{\delta+1}. \end{equation} It follows that \begin{align*} |F(x,u,v)| & = |F(x,u,v) - F(x,0,0)| \\ &= |F_u(x,\theta_1 u,\theta_2v)u + F_v(x,\theta_1 u,\theta_2 v)v| \\ & \leq \left|\epsilon |\theta_1 u|^\gamma |\theta_2 v|^{\delta+1}u + \epsilon |\theta_1 u|^{\gamma +1}|\theta_2 v|^\delta v\right|\\ &\leq 2\epsilon |u|^{\gamma+1} |v|^{\delta+1}, \quad 0 < \theta_1, \theta_2 < 1. \end{align*} Now, we choose a cut-off function $\eta \in C^1(\mathbb{R}\times \mathbb{R}, \mathbb{R})$ so that it is even and satisfies the following conditions: \begin{gather*} \eta(s,t) = 1 \quad \text{for all } |s| \leq \rho'_1, \; |t| \leq \rho'_2, \\ \eta(s,t) = 0 \quad \text{for all } |s| \geq 2\rho'_1,\; |t| \geq 2\rho'_2\\ |\eta'_s(s,t)| \leq \frac{2}{\rho'_1}, \quad |\eta'_t(s,t)| \leq \frac{2}{\rho'_2}, \quad \eta'_s(s,t)s \leq 0, \quad \eta'_t(s,t)t \leq 0. \end{gather*} Let $\overline \theta \in \big(0, \frac{\theta}{4(\gamma + \delta +12)}\big)$ be fixed, we define \begin{gather}\label{e2.6} F_\infty(u) : = \overline\theta |u|^{\gamma+1}|v|^{\delta+1}, \\ \hat{F}(x,u,v) : = \eta(u,v) F(x,u,v) + (1-\eta(u,v))F_\infty(u,v), \label{e2.7}\\ \hat{F}_u(x,u,v) : = \frac{\partial \hat{F}}{\partial u}(x,u,v), \quad \hat{F}_v(x,u,v) = \frac{\partial \hat{F}}{\partial v}(x,u,v) \label{e2.8}. \end{gather} Then, it is easy to verify that \begin{align}\label{e2.9} |\hat{F}(x,u,v)| \leq (\epsilon + \overline\theta)|u|^{\gamma+1}|v|^{\delta+1} \leq \frac{\lambda}{2} |u|^{\gamma+1}|v|^{\delta+1} \end{align} for all $(x,u,v) \in \Omega \times\mathbb{R} \times \mathbb{R}$, On the other hand, for any $|u| \leq 2 \rho'_1$ and $|v| \leq 2 \rho'_2$, we have \begin{align*} \hat{F}_u(x,u,v) &= \eta'_u(u,v)F(x,u,v) + \eta(u,v)F_u(x,u,v) \\ &\quad + (1-\eta(u,v))F'_{\infty,u}(u,v) - \eta'_u(u,v)F_\infty(u,v) \end{align*} and \begin{align*} \hat{F}_v(x,u,v) &= \eta'_v(u,v)F(x,u,v) + \eta(u,v)F_v(x,u,v) \\ &\quad + (1-\eta(u,v))F'_{\infty,v}(u,v) - \eta'_v(u,v)F_\infty(u,v). \end{align*} It follows that \begin{align*} &\hat{F}_u (x,u,v)u + \hat{F}_v(x,u,v)v - r\hat{F}(x,u,v)\\ &= \eta'_u(u,v) u F(x,u,v) + \eta(u,v) F_u(x,u,v)v \\ &\quad + (1-\eta(u,v))F'_{\infty,u}(u,v)u - \eta'_u(u,v) uF_\infty(u,v) \\ &\quad + \eta'_v(u,v) vF(x,u,v) + \eta(u,v) F_v(x,u,v)v + (1-\eta(u,v))F'_{\infty,v}(u,v)v \\ &\quad - \eta'_v(u,v) vF_\infty(u,v) - r\eta(u,v) F(x,u,v) - r (1- \eta(u,v)) F_\infty(u,v). \end{align*} Combining this with relations \eqref{e2.6}-\eqref{e2.9}, a simple computation shows that \begin{align*} \hat{F}_u(x,u,v)u + \hat{F}_v(x,u,v)v - r\hat{F}(x,u,v) & \leq [12\epsilon + (\gamma + \delta +12)\overline\theta]|u|^{\gamma+1} |v|^{\delta+1} \\ & \leq \theta |u|^{\gamma+1}|v|^{\delta+1} \\ & = \frac{(2-r)\lambda}{2}|u|^{\gamma+1}|v|^{\delta+1}. \end{align*} Thus, the numbers $\rho'_1$, $\rho'_2$ and the function $\hat{F}(x,u,v)$, defined by \eqref{e2.7} satisfy all the properties stated in the lemma. \end{proof} Next, we consider the modified elliptic problem \begin{equation}\label{e2.10} \begin{gathered} - \operatorname{div}(h_1(x)\nabla u) = b_1(x)|u|^{r-2}u + \hat{F}_u(x,u,v) \quad \text{in } \Omega,\\ - \operatorname{div}(h_2(x)\nabla v) = b_2(x)|v|^{r-2}v + \hat{F}_v(x,u,v) \quad \text{in } \Omega, \end{gathered} \end{equation} where $\hat{F}_u(x,u,v)$ and $\hat{F}_v(x,u,v)$ are given by \eqref{e2.8}. Then the solutions of problem \eqref{e2.10} correspond to critical points of the $C^1$ functional $\hat{J} : H \to \mathbb{R}$, defined by \begin{equation} \label{e2.11} \begin{split} \hat{J}(w) &= \frac{1}{2}\int_\Omega (h_1(x)|\nabla u|^2 + h_2(x)|\nabla v|^2)dx \\ &\quad - \frac{1}{r}\int_\Omega (b_1(x)|u|^r + b_2(x)|v|^r)dx - \int_\Omega \hat{F}(x,u,v)dx, \\ & = \Lambda (w) - \hat I(w), \end{split} \end{equation} where \begin{equation} \label{e2.12} \begin{gathered} \Lambda(w) = \frac{1}{2}\int_\Omega (h_1(x)|\nabla u|^2 + h_2(x)|\nabla v|^2)dx \\ \hat I(w) = \frac{1}{r}\int_\Omega (b_1(x)|u|^r + b_2(x) |v|^r)dx+\int_\Omega \hat{F}(x,u,v)dx \end{gathered} \end{equation} for all $w = (u,v) \in H$. \begin{lemma}\label{bd2.2} The functional $\hat{J}$ is coercive and bounded from below in $H$. \end{lemma} \begin{proof} We deduce by the definition $\hat{J}$ and \eqref{e2.9} that \begin{equation} \label{e2.13} \begin{split} \hat{J}(w) & = \frac{1}{2}\int_\Omega (h_1(x)|\nabla u|^2 + h_2(x)|\nabla v|^2)dx - \frac{1}{r}\int_\Omega (b_1(x)|u|^r + b_2(x)|v|^r)dx \\ &\quad - \int_\Omega \hat{F}(x,u,v)dx\\ & \geq \frac{1}{2}\int_\Omega (h_1(x)|\nabla u|^2 + h_2(x)|\nabla v|^2)dx - \frac{1}{r}\int_\Omega (b_1(x)|u|^r + b_2(x)|v|^r)dx \\ & \quad - \frac{\lambda}{2}\int_\Omega |u|^{\gamma+1}|v|^{\delta+1}dx \\ & \geq \frac{1}{2}\big(1 - \frac{\lambda}{\lambda_1}\big) \int_\Omega (h_1(x)|\nabla u|^2+ h_2(x)|\nabla v|^2)dx\\ &\quad - \frac{1}{r}\int_\Omega (b_1(x)|u|^r + b_2(x)|v|^r)dx \end{split} \end{equation} for all $w = (u,v) \in H$. Since $1 < r < 2$ and $0 < \lambda < \lambda_1$, relation \eqref{e2.13} implies that the functional $\hat{J}$ is coercive and bounded from below. \end{proof} \begin{lemma}\label{bd2.3} The functional $\hat{J}$ satisfies the Palais-Smale condition in $H$. \end{lemma} \begin{proof} Let $\{w_m\} = \{(u_m,v_m)\} \subset H$ be a sequence such that \begin{equation}\label{e2.14} \hat{J}(w_m) \to \overline c, \quad D\hat{J}(w_m) \to 0 \quad \text{ in } H \text{ as } m \to \infty. \end{equation} Since $\hat{J}$ is coercive, the sequence $\{w_m\}$ is bounded in $H$. Since $H$ is a Hilbert space, there exists $w = (u,v) \in H$ such that, passing to a subsequence, still denoted by $\{w_m\}$, it converges weakly to $w$ in $H$. Hence, $\{\|w_m - w\|_H\}$ is bounded. This and \eqref{e2.14} imply that \begin{equation}\label{e2.15} \lim_{m \to \infty}D\hat{J}(w_m)(w_m - w) = 0. \end{equation} From the proof of Lemma \ref{bd2.1}, there are positive constants $C_1, C_2$, depending on $\gamma, \delta$ and such that \begin{equation}\label{e2.16} |\hat{F}_u(x,u,v)| \leq C_1|u|^\gamma |v|^{\delta+1}, \quad |\hat{F}_v(x,u,v)| \leq C_2|u|^{\gamma+1}|v|^\delta \end{equation} for all $(u,v) \in H$. Hence, \begin{equation} \label{e2.17} \begin{split} \int_{\Omega}\hat{F}_u(x,u_m,v_m)(u_m - u) dx & \leq C_1 \int_{\Omega}|u_m|^{\gamma}|v_m|^{\delta+1}|u-u_m|dx \\ & \leq C_1 \|u_m\|^{\gamma}_{L^p(\Omega)}\|v_m\|^{\delta +1}_{L^q(\Omega)}\| u_m - u\|_{L^p(\Omega)}, \end{split} \end{equation} and \begin{equation} \label{e2.18} \begin{split} \int_{\Omega}\hat{F}_v(x,u_m,v_m)(v_m - v) dx & \leq C_2 \int_{\Omega}|u_m|^{\gamma+1}|v_m|^{\delta}|u-u_m|dx \\ & \leq C_2 \|u_m\|^{\gamma+1}_{L^p(\Omega)}\|v_m\|^{\delta}_{L^q(\Omega)}\| v_m - v\|_{L^q(\Omega)}. \end{split} \end{equation} It follows from relations \eqref{e2.17} and \eqref{e2.18} that \begin{align*} |D\hat I(w_m) (w_m - w)| & = \Big|\int_\Omega [\hat{F}_u(x,u_m,v_m)(u_m - u) +\hat{F}_v(x,u_m,v_m)(v_m - v)]dx\Big| \\ & \leq C_1 \|u_m\|^{\gamma}_{L^p(\Omega)}\|v_m\|^{\delta +1}_{L^q(\Omega)} \| u_m - u\|_{L^p(\Omega)} \\ & \quad + C_2 \|u_m\|^{\gamma+1}_{L^p(\Omega)}\|v_m\|^{\delta}_{L^q(\Omega)} \| v_m - v\|_{L^q(\Omega)}. \end{align*} where the functional $\hat I$ is given by \eqref{e2.12}. Therefore, we can show by Remark \ref{cy1.2} that \begin{equation}\label{e2.19} \lim_{m\to \infty}D\hat I(w_m) (w_m - w) = 0. \end{equation} Combining relations \eqref{e2.15}, \eqref{e2.19} with the fact that $$ D\Lambda(w_m) (w_m - w) = D\hat{J}(w_m) (w_m - w) + D\hat I(w_m) (w_m - w) $$ imply that \begin{equation}\label{e2.20} \lim_{m\to \infty}D\Lambda(w_m) (w_m - w) = 0, \end{equation} where the functional $\Lambda$ is given by \eqref{e2.12}. Hence, by the convexity of the functional $\Lambda$, we have \begin{equation} \label{e2.21} \begin{split} \Lambda (w) - \lim_{m \to \infty}\sup\Lambda (w_m) & = \lim_{m \to \infty}\inf(\Lambda (w) - \Lambda (w_m)) \\ & \geq \lim_{m \to \infty}D\Lambda(w_m) (w - w_m)= 0 \end{split} \end{equation} and the weak lower semi-continuity of $\Lambda$ implies that \begin{equation}\label{e2.22} \lim_{m \to \infty}\Lambda (w_m) = \Lambda (w). \end{equation} We now assume by contradiction that $\{w_m\}$ does not converge strongly to $w$ in $H$, then there exist a constant $\epsilon > 0$ and a subsequence of $\{w_m\}$, still denoted by $\{w_m\}$, such that $\| w_m - w\| \geq \epsilon$. We have \begin{equation}\label{e2.23} \frac{1}{2}\Lambda (w) + \frac{1}{2}\Lambda (w_m) - \Lambda \big(\frac{w_m + w}{2}\big) = \frac{1}{4} \|w_m -u\|^2 \geq \frac{1}{4}\epsilon^2. \end{equation} Letting $m \to \infty$, relation \eqref{e2.23} gives \begin{equation}\label{e2.24} \lim_{m \to \infty}\sup \Lambda \big(\frac{w_m + w}{2}\big) \leq \Lambda (w) - \frac{1}{4} \epsilon^2. \end{equation} We remark that the sequence $\{\frac{w_m + w}{2}\}$ also converges weakly to $w$ in $H$. So, we have $$ \Lambda (w) \leq \lim_{m \to \infty}\inf \Lambda \big(\frac{w_{m} + w}{2}\big), $$ which contradicts \eqref{e2.24}. Therefore, $\{w_m\}$ converges strongly to $w$ in $H$ and the functional $\hat{J}$ satisfies the Palais-Smale condition in $H$. \end{proof} \begin{lemma}\label{bd2.4} $\hat{J}(w) = 0 = D\hat{J}(w)(w)$ if and only if $w = (0,0)$, where $w = (u,v) \in H$. \end{lemma} \begin{proof} It is clear that if $w = (u,v) = (0,0)$ then $\hat{J}(w) = 0 = D\hat{J}(w)(w)$. Next, we assume $\hat{J}(w) = 0 = D\hat{J}(w)(w)$. By the definition of the functional $\hat{J}$ and \begin{multline*} D\hat{J}(w)(w) = \int_\Omega (h_1(x)|\nabla u|^2 + h_2(x)|\nabla v|^2)dx \\ - \int_\Omega (b_1(x)|u|^r + b_2(x)|v|^r) dx - \int_\Omega [\hat{F}_u(x,u,v)u + \hat{F}_v(x,u,v)v]dx. \end{multline*} we obtain by Lemma \ref{bd2.1} that \begin{equation} \label{e2.25} \begin{split} &\big(\frac{1}{r}- \frac{1}{2}\big) \int_\Omega (a_1(x)|\nabla u|^2 + a_2(x)|\nabla v|^2)dx \\ & = \int_\Omega \Big(\frac{1}{r}\hat{F}_u(x,u,v)u + \frac{1}{r}\hat{F}_v(x,u,v)v - \hat{F}(x,u,v)\Big) dx \\ & \leq \frac{\lambda (2-r)}{2r} \int_\Omega |u|^{\gamma+1}|v|^{\delta+1}dx, \end{split} \end{equation} Then since $0 < \lambda < \lambda_1$, where $\lambda_1$ is given by \eqref{e2.1}, it implies that $u = 0$ and $v = 0$. \end{proof} \begin{proof}[Proof of Theorem \ref{dl1.4}] To apply Lemma \ref{bd1.5} to the functional $\hat{J}$, we only need to find any $k \in \mathbb N$, a subspace $H_k$ of $H$ and $\rho_k > 0$ such that $$ \sup_{H_k \cap S_{\rho_k}} \hat{J} < 0. $$ Indeed, for any $k \in \mathbb N$ we find $k$ linearly independent functions $e_1, e_2, \dots, e_k \in C^\infty_0(\Omega', \mathbb{R}^2)$, $e_i = (e^{(1)}_i,e^{(2)}_i)$, $i = 1, 2, \dots, k$, and define the subspace $$ H_k : = \operatorname{span}\{e_1, e_2, \dots, e_k\}. $$ By (B), we may assume that $b_i(x) \geq b_0 > 0$, $i = 1, 2$ in $\cup_{i = 1}^k \text{ supp } e_i$ for some constant $b_0$. For any $w = (u,v) \in H_k$, using \eqref{e2.1} in Lemma \ref{bd2.1}, we have \begin{align*} \hat{J}(w) & = \frac{1}{2}\int_\Omega (a_1(x)|\nabla u|^2 + a_2(x)|\nabla v|^2)dx- \frac{1}{r}\int_\Omega (b_1(x)|u|^r + b_2(x)|v|^r) dx\\ &\quad - \int_\Omega \hat{F}(x,u,v)dx \\ & \leq \frac{1}{2}\int_\Omega (a_1(x)|\nabla u|^2 + a_2(x)|\nabla v|^2)dx - \frac{b_0}{r} (\|u\|^r_{L^r(\Omega)} + \|v\|^r_{L^r(\Omega)}) \\ &\quad + \frac{\lambda}{2}\int_\Omega |u|^{\gamma+1}|v|^{\delta+1}dx \\ &\leq \frac{1}{2}\big(1+ \frac{\lambda}{\lambda_1}\big) \int_\Omega (a_1(x)|\nabla u|^2 + a_2(x)|\nabla v|^2)dx -\frac{b_0}{r} (\|u\|^r_{L^r(\Omega)} + \|v\|^r_{L^r(\Omega)}), \end{align*} which implies the existence of $\rho_k$ such that $\sup_{H_k \cap S_{\rho_k}} \hat{J} < 0$ since the dimension of $H_k$ is finite. By Lemma \ref{bd2.1}, there exists a sequence of negative critical values $c_k$ of $\hat{J}$ satisfying $c_k \to 0$ as $k\to \infty$. For any $k$, let $w_k = (u_k,v_k)$ be a critical point of $\hat{J}$ associated with $c_k$. Then, $w_k$, $k \geq 1$ are exactly the solutions of problem \eqref{e1.1} and they form a Palais-Smale sequence. Without loss of genarality, we may assume that $w_k \to w = (u,v)$ in $H$ as $k \to\infty$. Then $w$ satisfies $$ \hat{J}(w) = 0 = D\hat{J}(w)(w). $$ Therefore, $w = (0,0)$ according to Lemma \ref{bd2.4} and $w_k \to 0$ in $H$ as $k \to \infty$. Standard elliptic estimates show that $w_k \to 0$ in $L^\infty(\Omega,\mathbb{R}^2)$ as $k \to \infty$. Finally, relations \eqref{e2.2} and \eqref{e2.10} help us to conclude that $w_k$ with $k$ large enough are the solutions of problem \eqref{e1.1}. The proof of Theorem \ref{dl1.4} is now complete. \end{proof} \subsection*{Acknowledgments} The author would like to thank the anonymous referees for their suggesions and helpful comments which improved the presentation of this article. This work was supported by National Foundation for Science and Technology Development (NAFOSTED). \begin{thebibliography}{00} \bibitem{AdriHami}{K. Adriouch and A. EL Hamidi}, {The Nehari manifold for systems of nonlinear elliptic equations}, \emph{Nonlinear Anal.}, \textbf{64} (2006), 2149-2164. \bibitem{AmbBreCer}{A. Ambrosetti, H. Brezis and C. Cerami}, {Combined effects of concave and convex nonlinearities in some elliptic problems}, \emph{J. Funct. Anal.}, \textbf{122} (1994), 519-543. \bibitem{AmbRab}{A. Ambrosetti and P. H. Rabinowitz}, {Dual variational methods in critical points theory and applications}, \emph{J. Funct. Anal.}, \textbf{4} (1973), 349-381. \bibitem{BartWill}{T. Bartsch and M. Willem}, {On an elliptic equation with concave and convex nonlinearities}, \emph{Proc. Amer. Math. Soc.}, \textbf{123} (1995), 3555-3561. \bibitem{BoccDeFi}{L. Boccardo and D. G. De Figueiredo}, {Some remarks on a system of quasilinear elliptic equations}, \emph{Nonlinear Diff. Equ. Appl. (NoDEA)}, \textbf{9} (2002), 309 - 323. \bibitem{CaldMusi}{P. Caldiroli and R. Musina}, {On a variational degenerate elliptic problem}, \emph{Nonlinear Diff. Equ. Appl. (NoDEA)}, \textbf{7} (2000), 189-199. \bibitem{ChungToan}{N. T. Chung and H. T. Toan}, {On a class of degenerate and singular elliptic systems in bounded domains}, \emph{J. Math. Anal. Appl.}, \textbf{360}(2) (2009), 422-431. \bibitem{Chung}{N. T. Chung}, {On the existence of weak solutions for a degenerate and singular elliptic system in $\mathbb{R}^N$}, \emph{Acta Appl. Math.}, \textbf{110}(1) (2010), 47-56. \bibitem{Clark}{D. C. Clark}, {A variant of the Ljusternik-Schnirelmann theory}, \emph{Indiana Univ. Math. J.}, \textbf{22} (1972), 65-74. \bibitem{Cos}{D. G. Costa}, {On a class of elliptic systems in $\mathbb{R}^N$}, \emph{Electron. J. Diff. Eqns.} \textbf{07} (1994), 1-14. \bibitem{DautLion}{R. Dautray, J. L. Lions}, {Mathematical Analysis and Numerical Methods for Science and Technology I: Physical Origins and Classical Methods}, Springer-Verlag, Berlin, 1985. \bibitem{Guo}{Z. Guo}, {Elliptic equations with indefinite concave nonlinearities near the origin}, \emph{J. Math. Anal. Appl.}, \textbf{367} (2010), 273-277. \bibitem{Mihai}{M. Mih\u{a}ilescu}, {Nonlinear eigenvalue problems for some degenerate elliptic operators on $\mathbb{R}^N$}, \emph{Bull. Belg. Math. Soc.}, \textbf{12} (2005), 435-448. \bibitem{MihaRadu}{M. Mih\u{a}ilescu and V. R\u{a}dulescu}, {Ground state solutions of nonlinear singular Schr\"odinger equations with lack compactness}, \emph{Math. Methods Appl. Sci.}, \textbf{26} (2003), 897 - 906. \bibitem{MurtStam}{M. K. V. Murthy and G. Stampachia}, {Boundary value problems for some degenerate elliptic operators}, \emph{Ann. Mat. Pure Appl.}, \textbf{80} (1968), 1 - 122. \bibitem{Oh}{Y.-G. Oh}, {Existence of semiclassical bound states of nonlinear Schr\"odinger equations with potentials of the class $(V_a)$}, \emph{Comm. Partial Differential Equations}, \textbf{13}(12) (1988), 1499 - 1519. \bibitem{RaduSmet}{V. Radulescu and D. Smets}, \emph{Critical singular problems on infinite cones}, \emph{Nonlinear Anal.}, \textbf{54}(6) (2003), 1153-1164. \bibitem{StavZogr}{N. M. Stavrakakis and N. B. Zographopoulos}, {Existence results for quasilinear elliptic systems in $\mathbb{R}^N$}, \emph{Electron. J. Diff. Eqns.}, \textbf{39} (1999), 1-15. \bibitem{Stre}{E. W. Stredulinsky}, {Weighted inequalities and degenerate elliptic partial differential equations}, {Springer-Verlag, Berlin, New York}, 1984. \bibitem{Wang}{Z. Q. Wang}, {Nonlinear boundary value problems with concave nonlinearities near the origin}, \emph{Nonlinear Diff. Equ. Appl. (NoDEA)}, \textbf{8} (2001), 15-33. \bibitem{Wu1}{T. F. Wu}, {The Nehari manifold for a semilinear elliptic system involving sign-changing weight functions}, \emph{Nonlinear Anal.}, \textbf{68} (2008), 1733-1745. \bibitem{Wu2}{T. F. Wu}, {On semilinear elliptic equations involving concave-convex nonlinearities and sign-changing weight function}, \emph{J. Math. Anal. Appl.}, \textbf{318} (2006), 253-270. \bibitem{ZhangWang}{G. Zhang and Y. Wang}, {Some existence results for a class of degenerate semilinear elliptic systems}, \emph{J. Math. Anal. Appl.}, \textbf{333} (2007), 904 - 918. \bibitem{Zog}{N. B. Zographopoulos}, {On a class of degenerate potential elliptic systems}, \emph{Nonlinear Diff. Equ. Appl. (NoDEA)}, \textbf{11} (2004), 191-199. \end{thebibliography} \end{document}