\documentclass[reqno]{amsart} \usepackage{hyperref} \AtBeginDocument{{\noindent\small \emph{Electronic Journal of Differential Equations}, Vol. 2008(2008), No. 119, pp. 1--10.\newline ISSN: 1072-6691. URL: http://ejde.math.txstate.edu or http://ejde.math.unt.edu \newline ftp ejde.math.txstate.edu (login: ftp)} \thanks{\copyright 2008 Texas State University - San Marcos.} \vspace{9mm}} \begin{document} \title[\hfilneg EJDE-2008/119\hfil Existence of weak solutions] {Existence of weak solutions for a nonuniformly elliptic nonlinear system in $\mathbb{R}^N$} \author[N. T. Chung\hfil EJDE-2008/119\hfilneg] {Nguyen Thanh Chung} \address{Nguyen Thanh Chung \newline Department of Mathematics and Informatics, Quang Binh University, 312 Ly Thuong Kiet, Dong Hoi, Vietnam} \email{ntchung82@yahoo.com} \thanks{Submitted March 27, 2008. Published August 25, 2008.} \subjclass[2000]{35J65, 35J20} \keywords{Nonuniformly elliptic; nonlinear systems; mountain pass theorem; \hfill\break\indent weakly continuously differentiable functional} \begin{abstract} We study the nonuniformly elliptic, nonlinear system \begin{gather*} - \mathop{\rm div}(h_1(x)\nabla u)+ a(x)u = f(x,u,v) \quad \text{in } \mathbb{R}^N,\\ - \mathop{\rm div}(h_2(x)\nabla v)+ b(x)v = g(x,u,v) \quad \text{in } \mathbb{R}^N. \end{gather*} Under growth and regularity conditions on the nonlinearities $f$ and $g$, we obtain weak solutions in a subspace of the Sobolev space $H^1(\mathbb{R}^N, \mathbb{R}^2)$ by applying a variant of the Mountain Pass Theorem. \end{abstract} \maketitle \numberwithin{equation}{section} \newtheorem{theorem}{Theorem}[section] \newtheorem{bd}[theorem]{Lemma} \newtheorem{remark}[theorem]{Remark} \newtheorem{proposition}[theorem]{Proposition} \newtheorem{definition}[theorem]{Definition} \section{Introduction} We study the nonuniformly elliptic, nonlinear system \begin{equation}\label{e:1.1} \begin{gathered} - \mathop{\rm div}(h_1(x)\nabla u) + a(x)u = f(x,u,v) \quad \text{in } \mathbb{R}^N, \\ - \mathop{\rm div}(h_2(x)\nabla v) + b(x)v = g(x,u,v) \quad \text{in } \mathbb{R}^N, \end{gathered} \end{equation} where $N\geq 3$, $h_i \in L^1_{\rm loc}(\mathbb{R}^N)$, $h_i (x) \geq 1$ $i = 1, 2$; $a, b \in C(\mathbb{R}^N)$. We assume that there exist $a_0, b_0 > 0$ such that \begin{equation} \label{e:1.2} \begin{gathered} a(x) \geq a_0, \quad b(x) \geq b_0, \quad \forall x \in \mathbb{R}^N,\\ a(x) \to \infty, \quad b(x) \to \infty \quad \text{as } |x| \to \infty. \end{gathered} \end{equation} System \eqref{e:1.1}, with $h_1(x) = h_2(x) = 1$, has been studied by Costa \cite{Cos}. There, under appropriate growth and regularity conditions on the functions $f(x,u,v)$ and $g(x,u,v)$, the weak solutions are exactly the critical points of a functional defined on a Hilbert space of functions $u, v$ in $H^1(\mathbb{R}^N)$. In the scalar case, the problem $$ -\mathop{\rm div}(|x|^{\alpha}\nabla u) + b(x)u = f(x,u) \quad \text{in } \mathbb{R}^N, $$ with $N \geq 3$ and $\alpha \in (0, 2)$, has been studied by Mih\u{a}ilescu and R\u{a}dulescu \cite{MR}. In this situation, the authors overcome the lack of compactness of the problem by using the the Caffarelli-Kohn-Nirenberg inequality. In this paper, under condition \eqref{e:1.2}, we consider \eqref{e:1.1} which may be a nonuniformly elliptic system. We shall reduce \eqref{e:1.1} to a uniformly elliptic system by using appropriate weighted Sobolev spaces. Then applying a variant of the Mountain pass theorem in \cite{Duc}, we prove the existence of weak solutions of system \eqref{e:1.1} in a subspace of $H^1(\mathbb{R}^N, \mathbb{R}^2)$. To prove our main results, we introduce the following some hypotheses: \begin{itemize} \item[(H1)] There exists a function $F(x,w) \in C^1(\mathbb{R}^N \times \mathbb{R}^2, \mathbb{R})$ such that $\frac{\partial F}{\partial u} = f(x,w)$, $\frac{\partial F}{\partial v} = g(x,w)$, for all $x \in \mathbb{R}^N$, $w= (u, v) \in \mathbb{R}^2$. \item[(H2)] $f(x,w), g(x,w) \in C^1(\mathbb{R}^N \times \mathbb{R}^2,\mathbb{R})$, $f(x,0,0) = g(x,0,0) = 0$ for all $x \in \mathbb{R}^N$, there exists a positive constant $\tau_0$ such that $$ |\nabla f(x,w)| + |\nabla g(x,w)| \leq \tau_0|w|^{p-1} $$ for all $x \in \mathbb{R}^N$, $w = (u,v) \in \mathbb{R}^2$. \item[(H3)] There exists a constant $\mu > 2$ such that $$ 0 < \mu F(x,w) \leq w \nabla F(x,w) $$ for all $x \in \mathbb{R}^N$ and $w \in \mathbb{R}^2\backslash \{(0,0)\}$. \end{itemize} Let $H^1(\mathbb{R}^N, \mathbb{R}^2)$ be the usual Sobolev space under the norm $$ \|w\|^2 = \int_{\mathbb{R}^N}(|\nabla u|^2 + |\nabla v|^2 + |u|^2 + |v|^2)dx, \quad w = (u, v) \in H^1(\mathbb{R}^N, \mathbb{R}^2)\,. $$ Consider the subspace $$ E = \{(u,v) \in H^1(\mathbb{R}^N, \mathbb{R}^2) : \int_{\mathbb{R}^N}(|\nabla u|^2 + |\nabla v|^2 + a(x)|u|^2 + b(x)|v|^2)dx < \infty\}. $$ Then $E$ is a Hilbert space with the norm $$ \|w\|_E^2 = \int_{\mathbb{R}^N}(|\nabla u|^2 + |\nabla v|^2 + a(x)|u|^2 + b(x)|v|^2)dx. $$ By \eqref{e:1.2} it is clear that $$ \|w\|_E \geq m_0 \|w\|_{H^1(\mathbb{R}^N, \mathbb{R}^2)}, \quad \forall w \in E, m_0 > 0, $$ and the embeddings $E \hookrightarrow H^1(\mathbb{R}^N, \mathbb{R}^2) \hookrightarrow L^q(\mathbb{R}^N, \mathbb{R}^2)$, $2 \leq q \leq 2^*$ are continuous. Moreover, the embedding $E \hookrightarrow L^2(\mathbb{R}^N, \mathbb{R}^2)$ is compact (see \cite{Cos}). We now introduce the space $$ H = \{(u,v) \in E: \int_{\mathbb{R}^N}(h_1(x)|\nabla u|^2 + h_2(x)|\nabla v|^2 + a(x)|u|^2 + b(x)|v|^2)dx < \infty\} $$ endowed with the norm $$ \|w\|^2_H = \int_{\mathbb{R}^N}(h_1(x)|\nabla u|^2 + h_2(x)|\nabla v|^2 + a(x)|u|^2 + b(x)|v|^2)dx. $$ \begin{remark}\label{rmk:1.1} \rm Since $h_1(x) \geq 1$, $h_2(x) \geq 1$ for all $x \in \mathbb{R}^N$ we have $\|w\|_E \leq \|w\|_H$ with $\forall w \in H$ and $C_0^{\infty}(\mathbb{R}^N, \mathbb{R}^2) \subset H$. \end{remark} \begin{proposition}\label{prop:1.2} The set $H$ is a Hilbert space with the inner product $$ \langle {w_1, w_2} \rangle = \int_{\mathbb{R}^N}(h_1(x)\nabla u_1 \nabla u_2+ h_2(x) \nabla v_1\nabla v_2 + a(x)u_1u_2 + b(x)v_1v_2)dx $$ for all $w_1 = (u_1, v_1)$, $w_2 = (u_2, v_2) \in H$. \end{proposition} \begin{proof} It suffices to check that any Cauchy sequences $\{w_m\}$ in $H$ converges to $w \in H$. Indeed, let $\{w_m\} = \{(u_m, v_m)\}$ be a Cauchy sequence in $H$. Then \begin{align*} &\lim_{m,k \to \infty} \int_{\mathbb{R}^N} \left(h_1(x)|\nabla u_m - \nabla u_k|^2+ h_2(x) |\nabla v_m - \nabla v_k|^2\right) dx \\ & + \lim_{m,k \to \infty} \int_{\mathbb{R}^N}\left( a(x)|u_m-u_k|^2 + b(x)|v_m-v_k|^2\right) dx = 0 \end{align*} and $\{\|w_m\|_H\}$ is bounded. Moreover, by Remark \ref{rmk:1.1}, $\{w_m\}$ is also a Cauchy sequence in $E$. Hence the sequence $\{w_m\}$ converges to $w = (u, v) \in E$; i.e., \begin{align*} &\lim_{m \to \infty} \int_{\mathbb{R}^N} \left(|\nabla u_m - \nabla u|^2+ |\nabla v_m - \nabla v|^2\right) dx \\ &+ \lim_{m \to \infty} \int_{\mathbb{R}^N} \left( a(x)|u_m - u|^2 + b(x)|v_m - v|^2\right)dx = 0. \end{align*} It follows that $\{\nabla w_m = (\nabla u_m, \nabla v_m)\}$ converges to $\nabla w = (\nabla u, \nabla v)$ and $\{w_m\}$ converges to $w$ in $L^2 (\mathbb{R}^N, \mathbb{R}^2)$. Therefore $\{ \nabla w_m (x) \}$ converges to $\{\nabla w(x)\}$ and $\{w_m(x)\}$ converges to $w(x)$ for almost everywhere $x \in \mathbb{R}^N$. Applying Fatou's lemma we get \begin{align*} & \int_{\mathbb{R}^N}(h_1(x)|\nabla u|^2 + h_2(x) |\nabla v|^2 + a(x)|u|^2 + b(x)|v|^2)dx \\ & \leq \liminf_{m \to \infty}\int_{\mathbb{R}^N}(h_1(x) |\nabla u_m|^2 + h_2(x) |\nabla v_m|^2 + a(x)|u_m|^2 + b(x)|v_m|^2)dx < \infty. \end{align*} Hence $w = (u,v) \in H$. Applying again Fatou's lemma \begin{align*} 0 & \leq \lim_{m \to \infty} \int_{\mathbb{R}^N} \left(h_1(x)|\nabla u_m - \nabla u|^2+ h_2(x) |\nabla v_m - \nabla v|^2\right) dx \\ & \quad + \lim_{m \to \infty} \int_{\mathbb{R}^N} \left( a(x)|u_m - u|^2 + b(x)|v_m - v|^2\right) dx \\ & \leq \lim_{m \to \infty}\Big[\liminf_{k \to \infty} \int_{\mathbb{R}^N}\left( h_1(x)|\nabla u_m - \nabla u_k|^2 + h_2(x) |\nabla v_m - \nabla v_k|^2\right) dx \Big] \\ &\quad + \lim_{m \to \infty} \Big[\liminf_{k \to \infty} \int_{\mathbb{R}^N} \left( a(x)|u_m - u_k|^2 + b(x)|v_m - v_k|^2\right) dx \Big] = 0. \end{align*} We conclude that $\{w_m\}$ converges to $w = (u,v)$ in $H$. \end{proof} \begin{definition}\label{def:1.3} \rm We say that $w = (u,v) \in H$ is a weak solution of system \eqref{e:1.1} if \begin{align*} &\int_{\mathbb{R}^N}(h_1(x) \nabla u \nabla \varphi + h_2(x) \nabla v\nabla \psi + a(x)u \varphi + b(x)v\psi )dx \\ &\quad - \int_{\mathbb{R}^N}(f(x,u,v) \varphi + g(x,u,v)\psi) dx=0 \end{align*} for all $\Phi =(\varphi, \psi) \in H$. \end{definition} Our main result is stated as follows. \begin{theorem}\label{thm:1.4} Assuming \eqref{e:1.2} and {\rm (H1)--(H3)} are satisfied, the system \eqref{e:1.1} has at least one non-trivial weak solution in $H$. \end{theorem} This theorem will be proved by using variational techniques based on a variant of the Mountain pass theorem in \cite{Duc}. Let us define the functional $J : H \to \mathbb{R}$ given by \begin{equation} \label{e:1.3} \begin{aligned} J(w) & = \frac{1}{2}\int_{\mathbb{R}^N}(h_1(x)|\nabla u|^2 + h_2(x)|\nabla v|^2 + a(x)|u|^2 + b(x)|v|^2)dx\\ &\quad - \int_{\mathbb{R}^N}F(x,u,v)dx \\ & = T(w) - P(w) \quad \text{for } w =(u,v) \in H, \end{aligned} \end{equation} where \begin{gather}\label{e:1.4} T(w) = \frac{1}{2}\int_{\mathbb{R}^N}(h_1(x)|\nabla u|^2 + h_2(x)|\nabla v|^2 + a(x)|u|^2 + b(x)|v|^2)dx, \\ P(w) = \int_{\mathbb{R}^N}F(x,u,v)dx. \end{gather} \section{Existence of weak solutions} In general, due to $h(x) \in L^1_{\rm loc}(\mathbb{R}^N)$, the functional $J$ may be not belong to $C^1(H)$ (in this work, we do not completely care whether the functional $J$ belongs to $C^1(H)$ or not). This means that we cannot apply directly the Mountain pass theorem by Ambrosetti-Rabinowitz (see \cite{AR}). In the situation, we recall the following concept of weakly continuous differentiability. Our approach is based on a weak version of the Mountain pass theorem by Duc (see \cite{Duc}). \begin{definition}\label{def:2.1} \rm Let $J$ be a functional from a Banach space $Y$ into $\mathbb{R}$. We say that $J$ is weakly continuously differentiable on $Y$ if and only if the following conditions are satisfied \begin{itemize} \item[(i)] $J$ is continuous on $Y$. \item[(ii)] For any $u \in Y$, there exists a linear map $DJ(u)$ from $Y$ into $\mathbb{R}$ such that $$ \lim_{t \to 0}\frac{J(u+tv) - J(u)}{t} = \langle {DJ(u),v}\rangle, \quad \forall v \in Y. $$ \item[(iii)] For any $v \in Y$, the map $u \mapsto \langle {DJ(u),v} \rangle$ is continuous on $Y$. \end{itemize} \end{definition} We denote by $C^1_w(Y)$ the set of weakly continuously differentiable functionals on $Y$. It is clear that $C^1(Y) \subset C^1_w(Y)$, where $C^1(Y)$ is the set of all continuously Frechet differentiable functionals on $Y$. The following proposition concerns the smoothness of the functional $J$. \begin{proposition}\label{prop:2.2} Under the assumptions of Theorem \ref{thm:1.4}, the functional $J(w), w \in H$ given by \eqref{e:1.3} is weakly continuously differentiable on $H$ and \begin{equation} \label{e:2.1} \begin{aligned} \langle {DJ(w),\Phi} \rangle &= \int_{\mathbb{R}^N}(h_1(x) \nabla u \nabla \varphi+ h_2(x) \nabla v\nabla \psi + a(x)u \varphi + b(x)v\psi )dx \\ &\quad - \int_{\mathbb{R}^N}(f(x,u,v) \varphi + g(x,u,v)\psi) dx \end{aligned} \end{equation} for all $w = (u,v)$, $\Phi =(\varphi, \psi) \in H$. \end{proposition} \begin{proof} By conditions (H1)--(H3) and the embedding $H \hookrightarrow E$ is continuous, it can be shown (cf. \cite[Theorem A.VI]{BL}) that the functional $P$ is well-defined and of class $C^1(H)$. Moreover, we have $$ \langle {DP(w),\Phi} \rangle = \int_{\mathbb{R}^N}(f(x,u,v)\varphi + g(x,u,v)\psi)dx $$ for all $w = (u,v)$, $\Phi =(\varphi, \psi) \in H$. Next, we prove that $T$ is continuous on $H$. Let $\{w_m\}$ be a sequence converging to $w$ in $H$, where $w_m = (u_m, v_m)$, $m =1, 2, \dots$, $w =(u,v)$. Then \begin{align*} &\lim_{m \to \infty}\int_{\mathbb{R}^N}[h_1(x)|\nabla u_m - \nabla u|^2 + h_2(x)|\nabla v_m - \nabla v|^2] dx \\ & + \lim_{m \to \infty}\int_{\mathbb{R}^N} [ a(x)|u_m - u|^2+ b(x)|v_m - v|^2]dx = 0 \end{align*} and $\{\|w_m\|_H\}$ is bounded. Observe further that \begin{align*} &\big|\int_{\mathbb{R}^N}h_1(x)|\nabla u_m|^2dx - \int_{\mathbb{R}^N}h_1(x)|\nabla u|^2dx\big| \\ &= \big|\int_{\mathbb{R}^N}h_1(x)(|\nabla u_m|^2- |\nabla u|^2)dx\big| \\ &\leq \int_{\mathbb{R}^N}h_1(x)||\nabla u_m| - |\nabla u||(|\nabla u_m| + |\nabla u|)dx \\ &\leq \int_{\mathbb{R}^N}h_1(x)|\nabla u_m - \nabla u||\nabla u_m|dx + \int_{\mathbb{R}^N}h_1(x)|\nabla u_m - \nabla u||\nabla u|dx \\ &\leq \Big(\int_{\mathbb{R}^N}h_1(x)|\nabla u_m - \nabla u|^2dx\Big) ^{1/2}\Big(\int_{\mathbb{R}^N}h_1(x)|\nabla u_m|^2dx\Big)^{1/2} \\ &\quad + \Big(\int_{\mathbb{R}^N}h_1(x)|\nabla u_m - \nabla u|^2dx\big)^{1/2} \Big(\int_{\mathbb{R}^N}h_1(x)|\nabla u|^2dx\Big)^{1/2} \\ & \leq (\|w_m\|_H + \|w\|_H )\| w_m - w \|_H. \end{align*} Similarly, we obtain \begin{gather*} \big|\int_{\mathbb{R}^N}h_2(x)|\nabla v_m|^2dx - \int_{\mathbb{R}^N}h_2(x)|\nabla v|^2dx\big| \leq (\|w_m\|_H + \|w\|_H )\| w_m - w \|_H, \\ \big|\int_{\mathbb{R}^N}a(x)|u_m|^2dx - \int_{\mathbb{R}^N}a(x)| u|^2dx\big| \leq (\|w_m\|_H + \|w\|_H )\| w_m - w \|_H, \\ \big|\int_{\mathbb{R}^N}b(x)|v_m|^2dx - \int_{\mathbb{R}^N}b(x)|v|^2dx\big| \leq (\|w_m\|_H + \|w\|_H )\| w_m - w \|_H. \end{gather*} From the above inequalities, we obtain $$ |T(w_m) - T(w)| \leq 4 (\|w_m\|_H + \|w\|_H)\|w_m - w\|_H \to 0 \quad \text{as } m \to \infty. $$ Thus $T$ is continuous on $H$. Next we prove that for all $w =(u,v)$, $\Phi=(\varphi, \psi) \in H$, $$ \langle {DJ(w),\Phi} \rangle = \int_{\mathbb{R}^N}(h_1(x) \nabla u \nabla \varphi+ h_2(x) \nabla v\nabla \psi + a(x)u \varphi + b(x)v\psi )dx. $$ Indeed, for any $w = (u,v)$, $\Phi = (\varphi,\psi) \in H$, any $t \in (-1,1) \backslash \{0\}$ and $x \in \mathbb{R}^N$ we have \begin{align*} \big|\frac{h_1(x)|\nabla u + t\nabla \varphi|^2 - h_1(x)|\nabla u|^2}{t}\big| & = \big|2 \int_0^1h_1(x)(\nabla u + st \nabla \varphi) \nabla \varphi ds\big| \\ & \leq 2h_1(x)(|\nabla u| + |\nabla \varphi|)|\nabla \varphi| \\ & \leq h_1(x)|\nabla u|^2 + 3h_1(x)|\nabla \varphi|^2. \end{align*} Since $h_1(x)|\nabla u|^2$, $h_1(x)|\nabla \varphi|^2 \in L^1(\mathbb{R}^N)$, $g(x) = h_1(x)|\nabla u|^2 + 3h_1(x)|\nabla \varphi|^2$ $\in L^1(\mathbb{R}^N)$. Applying Lebesgue's Dominated convergence theorem we get \begin{equation}\label{e:2.2} \lim_{t \to 0}\int_{\mathbb{R}^N}\frac{h_1(x)|\nabla u +t\nabla \varphi|^2 - h_1(x)|\nabla u|^2}{t}dx = 2 \int_{\mathbb{R}^N}h_1(x)\nabla u \nabla \varphi dx. \end{equation} Similarly, we have \begin{gather}\label{e:2.3} \lim_{t \to 0}\int_{\mathbb{R}^N}\frac{h_2(x)|\nabla v +t\nabla \psi|^2 - h_2(x)|\nabla v|^2}{t}dx = 2 \int_{\mathbb{R}^N}h_2(x)\nabla v \nabla \psi dx,\\ \label{e:2.4} \lim_{t \to 0}\int_{\mathbb{R}^N}\frac{a(x)|u +t \varphi|^2 - a(x)|u|^2}{t}dx = 2 \int_{\mathbb{R}^N}a(x)u\varphi dx, \\ \label{e:2.5} \lim_{t \to 0}\int_{\mathbb{R}^N}\frac{b(x)| v +t \psi|^2 - b(x)|v|^2}{t}dx = 2 \int_{\mathbb{R}^N}b(x)v \psi \,dx. \end{gather} Combining \eqref{e:2.2}-\eqref{e:2.5}, we deduce that \begin{align*} \langle {DT(w),\Phi} \rangle & = \lim_{t \to 0} \frac{T(w +t \Phi) - T(w)}{t} \\ & = \int_{\mathbb{R}^N}\left(h_1(x)\nabla u \nabla \varphi + h_2(x)\nabla v \nabla \psi + a(x)u\varphi + b(x)v\psi\right)dx. \end{align*} Thus $T$ is weakly differentiable on $H$. Let $\Phi=(\varphi,\psi) \in H$ be fixed. We now prove that the map $w \mapsto \langle {DT(w),\Phi} \rangle$ is continuous on $H$. Let $\{w_m\}$ be a sequence converging to $w$ in $H$. We have \begin{align*} &\big|\langle {DT(w_m),\Phi} \rangle -\langle {DT(w),\Phi} \rangle \big|\\ &\leq \int_{\mathbb{R}^N} h_1(x)|\nabla u_m -\nabla u||\nabla \varphi|dx + \int_{\mathbb{R}^N} h_2(x)|\nabla v_m -\nabla v||\nabla \psi|dx \\ &\quad + \int_{\mathbb{R}^N} a(x)|u_m - u||\varphi| dx + \int_{\mathbb{R}^N} b(x)|v_m - v||\psi| dx. \end{align*} It follows by applying Cauchy's inequality that \begin{equation}\label{e:2.6} |\langle {DT(w_m),\Phi} \rangle -\langle {DT(w),\Phi} \rangle| \leq 4\|\Phi\|_H\| w_m - w\|_H \to 0 \quad \text{as } m \to \infty. \end{equation} Thus the map $w \mapsto \langle {DT(w),\Phi} \rangle$ is continuous on $H$ and we conclude that functional $T$ is weakly continuously differentiable on $H$. Finally, $J$ is weakly continuously differentiable on $H$. \end{proof} \begin{remark}\label{rmk:2.3} \rm From Proposition~\ref{prop:2.2} we observe that the weak solutions of system \eqref{e:1.1} correspond to the critical points of the functional $J(w), w \in H$ given by \eqref{e:1.3}. Thus our idea is to apply a variant of the Mountain pass theorem in \cite{Duc} for obtaining non-trivial critical points of $J$ and thus they are also the non-trivial weak solutions of system \eqref{e:1.1}. \end{remark} \begin{proposition}\label{prop:2.4} The functional $J(w), w \in H$ given by \eqref{e:1.3} satisfies the Palais-Smale condition. \end{proposition} \begin{proof} Let $\{w_m=(u_m, v_m)\}$ be a sequence in $H$ such that $$ \lim_{m \to \infty}J(w_m) = c, \quad \lim_{m \to \infty}\|DJ(w_m)\|_{H^*} = 0. $$ First, we prove that $\{w_m\}$ is bounded in $H$. We assume by contradiction that $\{w_m\}$ is not bounded in $H$. Then there exists a subsequence $\{w_{m_j}\}$ of $\{w_m\}$ such that $\|w_{m_j}\|_H \to \infty$ as $j \to \infty$. By assumption (H3) it follows that \begin{align*} &J(w_{m_j}) - \frac{1}{\mu} \langle {DJ(w_{m_j}),w_{m_j}} \rangle\\ & = \big(\frac{1}{2} - \frac{1}{\mu}\big)\|w_{m_j}\|^2_H + \big(\frac{1}{\mu} \langle {DP(w_{m_j}),w_{m_j}} \rangle - P(w_{m_j})\big) \\ & \geq \gamma_0 \|w_{m_j}\|^2_H, \end{align*} where $\gamma_0 = \frac{1}{2} - \frac{1}{\mu}$. This yields \begin{equation} \label{e:2.7} \begin{aligned} J(w_{m_j}) & \geq \gamma_0 \|w_{m_j}\|^2_H + \frac{1}{\mu} \langle {DJ(w_{m_j}),w_{m_j}} \rangle \\ & \geq \gamma_0 \|w_{m_j}\|^2_H - \frac{1}{\mu} \|DJ(w_{m_j})\|_{H^*}.\|w_{m_j}\|_H \\ & = \|w_{m_j}\|_H\big(\gamma_0\|w_{m_j}\|_H - \frac{1}{\mu} \|DJ(w_{m_j})\|_{H^*}\big). \end{aligned} \end{equation} Letting $j \to \infty$, since $\|w_{m_j}\|_H \to \infty$, $\|DJ(w_{m_j})\|_{H^*} \to 0$ we deduce that $J(w_{m_j}) \to \infty$, which is a contradiction. Hence $\{w_{m}\}$ is bounded in $H$. Since $H$ is a Hilbert space and $\{w_m\}$ is bounded in $H$, there exists a subsequence $\{w_{m_k}\}$ of $\{w_{m}\}$ weakly converging to $w$ in $H$. Moreover, since the embedding $H \hookrightarrow E$ is continuous, $\{w_{m_k}\}$ is weakly convergent to $w$ in $E$. We shall prove that \begin{equation}\label{e:2.8} T(w) \leq \liminf_{k \to \infty} T(w_{m_k}). \end{equation} Since the embedding $E \hookrightarrow L^2(\mathbb{R}^N, \mathbb{R}^2)$ is compact, $\{w_{m_k}\}$ converges strongly to $w$ in $L^2(\mathbb{R}^N, \mathbb{R}^2)$. Therefore, for all $\Omega \subset\subset \mathbb{R}^N$, $\{w_{m_k}\}$ converges strongly to $w$ in $L^1(\Omega, \mathbb{R}^2)$. Besides, for any $\Phi = (\varphi, \psi) \in E$ we have \begin{align*} &\big|\int_{\Omega} \left( a(x)(u_{m_k}-u)\varphi + b(x)(v_{m_k}-v)\psi\right)dx\big|\\ &\leq \max\Big(\sup_{\Omega}a(x), \sup_{\Omega}b(x)\Big) \Big(\int_{\Omega}|u_{m_k} - u||\varphi| dx + \int_{\Omega}|v_{m_k} - v||\psi| dx\Big). \end{align*} Applying Cauchy inequality we obtain \begin{align*} &\big|\int_{\Omega}\left( a(x)(u_{m_k}-u)\varphi + b(x)(v_{m_k} - v) \psi\right) dx\big| \\ &\leq \gamma_1\|\Phi\|_{L^2(\mathbb{R}^N,\mathbb{R}^2)} \|w_{m_k}-w\|_{L^2(\mathbb{R}^N,\mathbb{R}^2)}, \end{align*} where $\gamma_1 = \max(\sup_{\Omega}a(x), \sup_{\Omega}b(x)) > 0$. Letting $k \to \infty$ we get \begin{equation}\label{e:2.9} \lim_{k\to \infty}\int_{\Omega}\left( a(x)(u_{m_k}-u)\varphi + b(x)(v_{m_k} - v)\psi \right) dx = 0. \end{equation} On the other hand, since $w_{m_k}$ converges weakly to $w$ in $E$; i.e., \begin{align*} &\lim_{k \to \infty}\int_{\mathbb{R}^N} \left[(\nabla u_{m_k}-\nabla u)\nabla\varphi + (\nabla v_{m_k} -\nabla v)\nabla\psi \right] dx \\ &+ \lim_{k \to \infty}\int_{\mathbb{R}^N}\left[a(x)(u_{m_k} - u)\varphi + b(x)(v_{m_k}- v)\psi \right] dx = 0 \end{align*} for all $\Phi = (\varphi, \psi) \in E$, by (\ref{e:2.9}) and $C_0^{\infty}(\mathbb{R}^N,\mathbb{R}^2)\subset H \subset E$ we infer that $$ \lim_{k \to \infty}\int_{\Omega}\left[(\nabla u_{m_k} -\nabla u)\nabla\varphi + (\nabla v_{m_k}-\nabla v)\nabla\psi\right]dx = 0, $$ for all $\Omega \subset \subset \mathbb{R}^N$. This implies that $\{\nabla w_{m_k}\}$ converges weakly to $\nabla w$ in $L^1(\Omega,\mathbb{R}^2)$. Applying \cite[Theorem 1.6]{Str}, we obtain $$ T(w) \leq \liminf_{k\to \infty} T(w_{m_k}). $$ Thus (\ref{e:2.8}) is proved. We now prove that \begin{equation}\label{e:2.10} \lim_{k\to \infty}\langle {DP(w_{m_k}),w_{m_k}-w} \rangle = \lim_{k\to \infty}\int_{\mathbb{R}^N}\nabla F(x,w_{m_k}).(w_{m_k}-w)dx = 0. \end{equation} Indeed, by (H2), we have \begin{align*} &|\nabla F(x, w_{m_k})(w_{m_k} - w)|\\ & = |f(x,w_{m_k})(u_{m_k} - u) + g(x,w_{m_k})(v_{m_k} - v)| \\ & \leq |\nabla f(x,\theta_1 w_{m_k})||w_{m_k}||u_{m_k} - u| + |\nabla g(x,\theta_2 w_{m_k})||w_{m_k}||v_{m_k} - v| \\ & \leq A_1|w_{m_k}|^p|u_{m_k} - u| + A_2|w_{m_k}|^p|v_{m_k} - v| \\ & \leq A_3|w_{m_k}|^p|w_{m_k} - w|, \quad 0 < \theta_1, \theta_2 < 1 \end{align*} where $A_i$ ($i = 1, 2, 3$) are positive constants. Set $2^* = \frac{2N}{N-2}$, $p_1 = \frac{2^*}{p - 1}$, $p_2 = p_3 = \frac{2p_1}{p_1 - 1}$. We have $p_1 > 1$, $2 < p_2, p_3 < 2^*$ and $\frac{1}{p_1} + \frac{1}{p_2} + \frac{1}{p_3} = 1$. Therefore, \begin{align*} \lim_{k\to \infty}\int_{\mathbb{R}^N}\nabla F(x,w_{m_k}).(w_{m_k}-w)dx & \leq A_3\int_{\mathbb{R}^N}|w_{m_k}|^{p - 1}|w_{m_k} - w||w_{m_k}|dx \\ & \leq A_3\|w_{m_k}\|_{L^{2^*}}^{p-1}\|w_{m_k} - w\|_{L^{p_2}} \|w_{m_k}\|_{L^{p_3}}. \end{align*} On the other hand, using the continuous embeddings $H \hookrightarrow E \hookrightarrow L^q(\mathbb{R}^N)$, $2 \leq q \leq 2^*$ together with the interpolation inequality (where $\frac{1}{p_2} = \frac{\delta}{2} + \frac{1 - \delta}{2^*}$), it follows that $$ \|w_{m_k} - w\|_{L^{p_2}(\mathbb{R}^N)} \leq \|w_{m_k} - w\|_{L^{2}(\mathbb{R}^N)}^{\delta}.\|w_{m_k} - w\|_{L^{2^*}}^{1-\delta}. $$ Since the embedding $E \hookrightarrow L^2(\mathbb{R}^N)$ is compact we have $\|w_{m_k} - w\|_{L^{2}(\mathbb{R}^N)} \to 0$ as $k \to \infty$. Hence $\|w_{m_k} - w\|_{L^{p_2}(\mathbb{R}^N)} \to 0$ as $k \to \infty$ and \eqref{e:2.10} is proved. On the other hand, by \eqref{e:2.10} and (\ref{e:2.1}) it follows $$ \lim_{k \to \infty}\langle {DT(w_{m_k}),w_{m_k}-w} \rangle = 0. $$ Hence, by the convex property of the functional $T$ we deduce that \begin{align}\label{e:2.11} T(w) - \lim_{k\to \infty} \sup T(w_{m_k}) & = \lim_{k \to \infty}\inf\left[T(w) - T(w_{m_k})\right] \\ & \geq \lim_{k \to \infty}\langle {DT(w_{m_k}),w - w_{m_k}} \rangle = 0. \end{align} Relations (\ref{e:2.8}) and (\ref{e:2.11}) imply \begin{equation}\label{e:2.12} T(w) = \lim_{k\to \infty} T(w_{m_k}). \end{equation} Finally, we prove that $\{w_{m_k}\}$ converges strongly to $w$ in $H$. Indeed, we assume by contradiction that $\{w_{m_k}\}$ is not strongly convergent to $w$ in $H$. Then there exist a constant $\epsilon_0 > 0$ and a subsequence $\{w_{m_{k_j}}\}$ of $\{w_{m_k}\}$ such that $\|w_{m_{k_j}} - w\|_H \geq \epsilon_0 >0$ for any $j = 1,2,\dots$. Hence \begin{equation}\label{e:2.13} \frac{1}{2}T(w_{m_{k_j}}) + \frac{1}{2}T(w) - T\Big(\frac{w_{m_{k_j}} + w}{2}\Big) = \frac{1}{4} \|w_{m_{k_j}} - w\|^2_H \geq \frac{1}{4}\epsilon^2_0. \end{equation} With the same arguments as in the proof of (\ref{e:2.8}), and remark that the sequence $\{\frac{w_{m_{k_j}}+w}{2}\}$ converges weakly to $w$ in $E$, we have \begin{equation}\label{e:2.14} T(w) \leq \liminf_{j \to \infty} T\Big(\frac{w_{m_{k_j}}+w}{2}\Big). \end{equation} Hence letting $j \to \infty$, from (\ref{e:2.12}) and (\ref{e:2.13}) we infer that \begin{equation}\label{e:2.15} T(w) - \liminf_{j \to \infty} T\Big(\frac{w_{m_{k_j}}+w}{2}\Big) \geq \frac{1}{4}\epsilon^2_0. \end{equation} Relations (\ref{e:2.14}) and (\ref{e:2.15}) imply $0 \geq \frac{1}{4}\epsilon_0^2 >0$, which is a contradiction. Therefore, we conclude that $\{w_{m_k}\}$ converges strongly to $w$ in $H$ and $J$ satisfies the Palais - Smale condition on $H$. \end{proof} To apply the Mountain pass theorem we shall prove the following proposition which shows that the functional $J$ has the Mountain pass geometry. \begin{proposition}\label{prop:2.5} (i) There exist $\alpha > 0$ and $r > 0$ such that $J(w) \geq \alpha$, for all $w \in H$ with $\|w\|_H = r$. (ii) There exists $w_0 \in H$ such that $\|w_0\|_H > r$ and $J(w_0) < 0$. \end{proposition} \begin{proof} (i) From (H3), it is easy to see that \begin{gather}\label{e:2.16} F(x,z) \geq \min_{|s|=1}F(x,s).|z|^{\mu} > 0 \quad \forall x \in \mathbb{R}^N \text{ and } |z| \geq 1, z \in \mathbb{R}^2,\\ \label{e:2.17} 0 < F(x,z) \leq \max_{|s| = 1} F(x,s) . |z|^{\mu} \quad \forall x \in \mathbb{R}^N \text{ and } 0 < |z| \leq 1, \end{gather} where $\max_{|s| = 1}F(x,s) \leq C$ in view of (H2). It follows from (\ref{e:2.17}) that \begin{equation}\label{e:2.18} \lim_{|z| \to 0}\frac{F(x,z)}{|z|^2} = 0 \quad\text{uniformly for } x \in \mathbb{R}^N. \end{equation} By using the embeddings $H \hookrightarrow E \hookrightarrow L^2(\mathbb{R}^N, \mathbb{R}^2)$, with simple calculations we infer from (\ref{e:2.18}) that $\inf_{\|w\|_H = r } J(w) = \alpha> 0$ for $r > 0$ small enough. This implies (i). (ii) By (\ref{e:2.16}), for each compact set $\Omega \subset \mathbb{R}^N$ there exists $\overline{c} = \overline{c}(\Omega)$ such that \begin{equation}\label{e:2.19} F(x,z) \geq \overline{c}|z|^{\mu} \quad \text{for all } x \in \Omega, |z| \geq 1. \end{equation} Let $0 \ne \Phi =(\varphi, \psi) \in C^1(\mathbb{R}^N, \mathbb{R}^2)$ having compact support, for $t > 0$ large enough, from (\ref{e:2.19}) we have \begin{equation}\label{e:2.20} J(t\Phi) = \frac{1}{2}t^2\|\Phi\|^2_H - \int_{\mathbb{R}^N}F(x,t\Phi) dx \leq \frac{1}{2}t^2\|\Phi\|^2_H - t^{\mu}\overline{c} \int_{\Omega}|\Phi|^{\mu} dx, \end{equation} where $\overline{c} = \overline{c}(\Omega)$, $\Omega = (\mathop{\rm supp}\varphi \cup \mathop{\rm supp}\psi)$. Then (\ref{e:2.20}) and $\mu > 2$ imply (ii). \end{proof} \begin{proof}[Proof of Theorem \ref{thm:1.4}] It is clear that $J(0) = 0$. Furthermore, the acceptable set $$ G = \{\gamma \in C([0,1], H) : \gamma (0) = 0, \gamma (1) = w_0\}, $$ where $w_0$ is given in Proposition \ref{prop:2.5}, is not empty (it is easy to see that the function $\gamma(t) = t\omega_0 \in G$). By Proposition \ref{prop:1.2} and Propositions \ref{prop:2.2}-\ref{prop:2.5}, all assumptions of the Mountain pass theorem introduced in \cite{Duc} are satisfied. Therefore there exists $\hat{w} \in H$ such that $$ 0 < \alpha \leq J(\hat{w}) = \inf\{\max J(\gamma([0,1])): \gamma \in G\} $$ and $\langle {DJ(\hat{w}), \Phi} \rangle = 0$ for all $\Phi \in H$; i.e., $\hat{w}$ is a weak solution of system \eqref{e:1.1}. The solution $\hat{w}$ is a non-trivial solution by $J(\hat{w}) \geq \alpha > 0$. The proof is complete. \end{proof} \subsection*{Acknowledgments} The author would like to thank Professor Hoang Quoc Toan, Department of Mathematics, Mechanics and Informatics, College of Science, Vietnam National University, for his interest, encouragement and helpful comments. \begin{thebibliography}{00} \bibitem{AF} {A. Abakhti-Mchachti., J. Fleckinger-Pell\'{e}}; \emph{Existence of positive solutions for non-cooperative semilinear elliptic systems defined on an unbounded domain}, {Pitman research Notes in Maths.,} {\bf 266} (1992), 92-106. \bibitem{Ada} {R. A. Adams}; \emph{Sobolev spaces}, {Academic Press, New York - San Francisco - London}, 1975. \bibitem{ACE} {A. Ambrosetti., K.C. Chang., I. Ekeland}; \emph{Nonlinear functional analysis and Applications to differential equations}, {Proceedings of the second school ICTP, Trieste, Italy}, 1997. \bibitem{AR} {A. Ambrosetti., P. H. Rabinowitz}; \emph{Dual variational methods in critical points theory and applications}, {J. Funct. Anal.} {\bf 4} (1973), 349-381. \bibitem{BL} {H. Berestycki., P. L. Lions}; \emph{Nonlinear scalar field equations, I. Existence of a ground state}, {Arch. Rat. Mech. Anal.}, {\bf 82} (1984), 313-345. \bibitem{Bre} {H. Brezis}; \emph{Analyse fonctionnelle th\'{e}ori et applications}, {Massion}, 1992. \bibitem{Cos} {D. G. Costa}; \emph{On a class of elliptic systems in $R^n$}, {Electronic Journal of Differential Equations} \textbf{1994} (1194) No. 07, 1-14. \bibitem{Duc}{D. M. Duc}; \emph{Nolinear singular elliptic equations}, {J. London. Math. Soc.} {\bf 40}(2) (1989), 420-440. \bibitem{GP} {L. Gas\'{i}nski., N. S. Parageorgiou}; \emph{Nonsmooth critical point theory and nonlinear boundary value problems}, {Vol. 8, A CRC Press Company, London - New York,} 2005. \bibitem{GTr} {D. Gilbarg., N. C. Trudinger}; \emph{Elliptic Partial Differential Equations of second order}, {Second Edition, Springer-Verlag, Berlin}, 1983. \bibitem{MR} {M. Mih\u{a}ilescu., V. R\u{a}dulescu}; \emph{Ground state solutions of nonlinear singular Schr\"odinger equations with lack compactness}, {Mathematical methods in the Applied Sciences,} {\bf 26} (2003), 897-906. \bibitem{NM} {P. De N\'{a}poli., M. C. Mariani}; \emph{Mountain pass solutions to equations of p-Laplacian type}, {Nonlinear Analysis,} {\bf 54} (2003), 1205-1219. \bibitem{Nir} {L. Nirenberg}; \emph{Variational and topology methods in nonlinear problems}, {Bull. Amer. Math. Soc.} {\bf 4} (1981), 267-302. \bibitem{Rab} {P. H. Rabinowitz}; \emph{Minimax methods and their applications to Partial Differential Equations}, {Nonlinear analysis, a collection of papers in honor of Erich Rothe, Academic Press, New York}, 1978. \bibitem{Str} {M. Struwe}; \emph{Variational Methods}, {Second Edition, Springer-Verlag}, 2000. \bibitem{VD} {N. T. Vu., D. M. Duc}; \emph{Nonuniformly elliptic equations of p-Laplacian type}, {Nonlinear Analysis} {\bf 61} (2005), 1483-1495. \bibitem{Zei} {E. Zeidler}; \emph{Nonlinear Functional Analysis and its applications}, {Vol. III, Springer-Verlag}, 1985. \end{thebibliography} \end{document}