\documentclass[reqno]{amsart} \usepackage{hyperref} \AtBeginDocument{{\noindent\small \emph{Electronic Journal of Differential Equations}, Vol. 2012 (2012), No. 111, pp. 1--16.\newline ISSN: 1072-6691. URL: http://ejde.math.txstate.edu or http://ejde.math.unt.edu \newline ftp ejde.math.txstate.edu} \thanks{\copyright 2012 Texas State University - San Marcos.} \vspace{9mm}} \begin{document} \title[\hfilneg EJDE-2012/111\hfil Multiplicity of positive solutions] {Multiplicity of positive solutions for quasilinear elliptic p-Laplacian systems} \author[A. Aghajani, J. Shamshiri \hfil EJDE-2012/111\hfilneg] {Asadollah Aghajani, Jamileh Shamshiri} % in alphabetical order \address{Asadollah Aghajani \newline School of Mathematics, Iran University of Science and Technology, P.O. Box 16846-13114, Narmak, Tehran, Iran} \email{aghajani@iust.ac.ir} \address{Jamileh Shamshiri \newline Department of Mathematics, Karaj Branch, Islamic Azad University, Karaj, Iran} \email{jamileshamshiri@gmail.com} \thanks{Submitted June 19, 2012. Published July 2, 2012.} \subjclass[2000]{35B38, 34B15, 35J92} \keywords{Critical points; nonlinear boundary value problems; \hfill\break\indent quasilinear p-Laplacian problem; fibering map; Nehari manifold} \begin{abstract} We study the existence and multiplicity of solutions to the elliptic system \begin{gather*} -\operatorname{div}(|\nabla u|^{p-2} \nabla u)+m_1(x)|u|^{p-2}u =\lambda g(x,u) \quad x\in \Omega,\\ -\operatorname{div}(|\nabla v|^{p-2} \nabla v)+m_2(x)|v|^{p-2}v=\mu h(x,v) \quad x\in \Omega,\\ |\nabla u|^{p-2}\frac{\partial u}{\partial n}=f_u(x,u,v),\quad |\nabla v|^{p-2}\frac{\partial v}{\partial n}=f_v(x,u,v), \end{gather*} where $\Omega\subset \mathbb{{R}}^N$ is a bounded and smooth domain. Using fibering maps and extracting Palais-Smale sequences in the Nehari manifold, we prove the existence of at least two distinct nontrivial nonnegative solutions. \end{abstract} \maketitle \numberwithin{equation}{section} \newtheorem{theorem}{Theorem}[section] \newtheorem{lemma}[theorem]{Lemma} \newtheorem{corollary}[theorem]{Corollary} \newtheorem{remark}[theorem]{Remark} \newtheorem{definition}[theorem]{Definition} \allowdisplaybreaks \section{Introduction} In this article, we study the existence and multiplicity of positive solutions of the quasilinear elliptic system $$\label{e1.1} \begin{gathered} -\operatorname{div}(|\nabla u|^{p-2} \nabla u)+m_1(x)|u|^{p-2}u =\lambda g(x,u) \quad x\in \Omega,\\ -\operatorname{div}(|\nabla v|^{p-2} \nabla v)+m_2(x)|v|^{p-2}v =\mu h(x,v) \quad x\in \Omega,\\ |\nabla u|^{p-2} \frac{\partial u}{\partial n}=f_u(x,u,v),\quad |\nabla v|^{p-2}\frac{\partial v}{\partial n}=f_v(x,u,v) x\in\partial\Omega, \end{gathered}$$ where $\lambda, \mu>0$, $p>2$, $\Omega\subset\mathbb{{R}}^N$ is a bounded domain in $\mathbb{{R}}^N$ with the smooth boundary $\partial\Omega$, $\frac{\partial}{\partial n}$ is the outer normal derivative, $m_1, m_2 \in C(\bar{\Omega})$ are positive bounded functions together with the following assumptions on the functions $f,g$ and $h$: \begin{itemize} \item[(A1)] $\frac{\partial^2}{\partial t^2} f(x,t|u|,t|v|)|_{t=1}\in C(\partial\Omega \times \mathbb{R}^2)$ and for $u, v\in L^p(\partial\Omega)$, the integral $\int_{\partial\Omega}\frac{\partial^2}{\partial t^2} \big(f(x,t|u|,t|v|)\big)dx$ has the same sign for every $t>0$. \item[(A2)] There exists $C_1>0$ such that $$f(x,u,v)\leq\frac{1}{r} \frac{\partial}{\partial t} f(x,tu,tv)|_{t=1}\leq\frac{1}{r(r-1)}\frac{\partial^2}{\partial t^2} f(x,tu,tv)|_{t=1}\leq C_1(u^r+v^r),$$ where $p2$, $p^*=\infty$ if $N\leq p$) for all $(x,u,v)\in {\partial\Omega} \times \mathbb{R^+}\times \mathbb{R^+}$. \item[(A3)] $\frac{\partial}{\partial t} f(x,tu,tv)|_{t=0}\geq0$ and $\lim_{t\to\infty}\frac{\frac{\partial}{\partial t} f(x,tu,tv)}{t^{p-1}}= \eta(x,u,v)$ uniformly respect to $(x,u,v)$, where $\eta(x,u,v)\in C(\bar{\Omega}\times\mathbb{R}^2)$ and $|\eta(x,u,v)|>\theta>0$, a.e. for all $(x,u,v)\in {\Omega} \times \mathbb{R^+}\times \mathbb{R^+}$. \item[(A4)] $g(x,u), h(x,v) \in C^1({\Omega} \times \mathbb{R})$ such that $g(x,0)\geq0$, $h(x,v)\geq0$, $g(x,0)\not\equiv0$ and there exist $C_2>0$, $C_3>0$ such that, $|g(x,u)| \leq C_2(1+u^{p-1})$ and $|h(x,v)|\leq C_3(1+v^{p-1})$, where $x\in \Omega$, $u,v \in{\mathbb{R^+}}$ and $p>2$. \item[(A5)] For $u,v\in W^{1,p}(\Omega)$, $\int_{\Omega}\frac{\partial}{\partial u} g(x,t|u|)u^2dx$ and $\int_{\Omega}\frac{\partial}{\partial v} h(x,t|v|)v^2dx$ have the same sign for every $t>0$ and there exist $C_4>0$, $C_5>0$ such that $|g_u(x,u)|\leq C_4u^{p-2}$ and $|h_v(x,v)|\leq C_5v^{p-2}$ for all $(x,u,v)\in \Omega\times\mathbb{R^+}\times\mathbb{R^+}$. \end{itemize} \begin{remark} \label{rmk1.1} \rm Equations involving positively homogeneous functions have been considered in many papers, such as \cite{3, 10, 17, 18, 27, 28}. It is clear that, if $f(x,u,v)$ is a positively homogeneous function of degree $r(r>p>2)$, that is, $f(x,tu,tv)=t^rf(x,u,v)$ $(t>0)$, then it satisfies conditions (A1)--(A3). Note that for such an $f$ we have $f_uu+f_vv=rf(x,u,v)\leq r K_f(|u|^r+|v|^r),$ where $K_f=\max\{f(x,u,v):(x,u,v)\in\overline{\Omega}\times\mathbb{R}^2,\; |u|^r+|v|^r=1\}.$ \end{remark} In recent years, there have been many papers concerned with the existence and multiplicity of positive solutions for the elliptic equations (systems) with nonlinear boundary conditions. The results relating to these problems can be found in \cite{1, 4, 6, 9, 11, 12, 13, 16, 19, 20, 21, 22, 23, 24, 25, 26, 29, 30, 31} and the references therein. For instance, Drabek and Schindler \cite{15} showed the existence of positive, bounded and smooth solutions of the following $p$-Laplacian equation \begin{gather*} -\Delta_pu+ b |u|^{p-2}u= f(.,u) \quad\text{in } \Omega,\\ \Re u=0 \quad\text{on } \partial\Omega , \end{gather*} where $\Re u=|\nabla u|^{p-2} \frac{\partial u}{\partial \nu} +b_{0}|u|^{p-2}u$, $\Omega\subset \mathbb{R}^{N}$ is a bounded domain and $11$, $2<\alpha+\beta<2^*$ and the functions $f, g, h$ satisfy the following conditions: \begin{itemize} \item $f\in C(\overline{\Omega})$ with $\|f\|_\infty=1$ and $f^+=max\{f,0\}\not\equiv 0$, \item $g,\,h\in C(\partial\Omega)$ with $\|g\|_\infty=\|h\|_\infty=1$, $g^\pm=\max\{\pm g,0\}\not\equiv 0$ and $h^\pm=\max\{\pm h,0\}\not\equiv 0$. \end{itemize} They found that the above problem has at least two nonnegative solutions if the pair $(\lambda, \mu)$ belongs to a certain subset of $\mathbb{R}^2$. Note that the function $\frac{1}{\alpha+\beta}f(x)|u|^\alpha|v|^\beta$ with the above given conditions is positively homogeneous of degree $r=\alpha+\beta$ and clearly satisfies our conditions (A1)--(A3). Recently, Shen and Zhang \cite{28} considered the semilinear p-Laplacian system \begin{gather*} -\Delta_p u=\frac{1}{p^*}\frac{\partial F(x,u,v)}{\partial u}+\lambda |u|^{q-2}u \quad\text{in }\Omega,\\ -\Delta_p v=\frac{1}{p^*}\frac{\partial F(x,u,v)}{\partial v}+\mu |v|^{q-2}v \quad\text{in } \Omega, \\ u>0, \quad v>0 \quad \text{in } \Omega,\\ u=v=0 \quad\text{on }\partial\Omega, \end{gather*} where $\Omega$ is bounded domain in $\mathbb{R}^N$ with smooth boundary, $F\in C^1(\overline{\Omega},\times (\mathbb{R}^+)^2)$ is positively homogeneous of degree $p^*$, they proved that this system has at least two positive solutions when the pair of parameters $(\lambda,\mu)$ belongs to certain subset of $\mathbb{R}^2$. In this article, the main difficulty will be the nonlinearity of $f(x,u,v)$, $g(x,u)$ and $h(x,v)$ in problem \eqref{e1.1} and the lack of separability. To overcome this difficultly, we need to restrict the problem \eqref{e1.1} to assumptions (A1) and (A5). Here we present some examples for $f(x,u,v)$ satisfying the conditions (A1)--(A3). \begin{gather*} f_1(x,u,v) \in C^1({{\partial\Omega}} \times \mathbb{R}^2,\mathbb{R}), \\ f_1(x,tu,tv) = t^r f_1(x,u,v) \quad\text{for } (x,u,v)\in {\partial\Omega} \times \mathbb{R^+}\times \mathbb{R^+} \text{ and } t>0,\\ f_2(x,u,v)=a_1(x)(-a_2(x)+\sqrt[q]{(a_2(x)^q+u^{qr}+v^{qr})}, \\ a_i(x)\in C({\partial\Omega}), \quad a_i(x)\geq0, \quad q>1, \quad q\in\mathbb{N},\\ f_3(x,u,v)=b(x)\frac{u^{q+r}+v^{q+r}}{1+u^q+v^q}, \quad b(x)\in C({\partial\Omega}), \quad b(x)\geq0, \quad r\geq0. \end{gather*} Now we present some examples for $g(x,u)$ and $h(x,v)$ satisfying the conditions (A4) and (A5): $Q_1(x,z)=\frac{-a_1(x)z^{p+r}}{1+a_2(x)z^{2}}+a_3(x)$ with $a_i(x)\in C(\overline{\Omega})$, $a_i(x)\geq 0$, $a_3(x)\not\equiv 0$, $\max\{2-p,-1\}\leq r\leq 1$. $Q_2(x,z)=b_1(x)\tan^{-1}(b_2(x)z^{p+k})\ln[1+z^{2k}]+b_3(x)$ with $b_i(x)\in C(\overline{\Omega})$, $b_i(x)\geq 0$, $b_3(x)\not\equiv 0$, $\frac{p}{2}\leq k\in \mathbb{N}$. $Q_3(x,z)=c_1(x)\sqrt[r]{(1+c_2(x)z^{2k})^{p-1}}$ with $c_i(x)\geq 0$, $c_i(x)\in C(\overline{\Omega})$, $c_1(x)\not\equiv 0$, $k\in\mathbb{N}$, $0<2k\leq r$. $Q_4(x,z)=\frac{-e_1(x)z^{p-1}}{4+\cot^{-1}(e_2(x)z^k)}+e_3(x)$ $e_i(x)\in C(\overline{\Omega})$, $e_i(x)\geq0$, $e_3(x)\not\equiv 0$, $k\geq0$. Here our main tool is the Nehari manifold method which is similar to the fibering method by Drabek and Pohozaev \cite{14}. The main idea in our proofs lies in dividing the Nehari manifold associated with the Euler functional for problem \eqref{e1.1} into two disjoint parts and then considering the infima of this functional on each part and by extracting Palais-Smale sequences we show that there exists at least one solution on each part. Define the Sobolev space $$\label{e1.2} W:=W^{1,p}(\Omega)\times W^{1,p}(\Omega),$$ endowed with the norm $$\|(u,v)\|_{W}=\Big(\int_{\Omega}(|\nabla u|^p+m_1(x)|u|^p)dx +\int_{\Omega}(|\nabla v|^p+m_2(x)|v|^p)dx\Big)^{1/p},$$ which is equivalent to the standard norm. We use the standard $\mathrm{L}^r(\Omega)$ spaces whose norms are denoted by $\|u\|_{r}$. Throughout this paper, we denote $S_q$ and $\bar{S}_{q}$ the best Sobolev and the best Sobolev trace constants for the embedding of $W^{1,p}(\Omega)$ into $\mathrm{L}^{q}(\Omega)$ and $W^{1,p}(\Omega)$ into $\mathrm{L}^{q}(\partial\Omega)$, respectively. So we have $$\label{e1.3} \frac{(\|(u,v)\|^p_{W})^{q}}{(\int_{\partial\Omega}(|u|^{q}+|v|^{q})dx )^p} \geq \frac{1}{2^p\bar{S}^{pq}_q} \quad \text{and} \quad \frac{(\|(u,v)\|^p_{W})^{q}}{(\int_{\Omega}(|u|^{q}+|v|^{q})dx )^p} \geq \frac{1}{2^p{S}^{pq}_{q}}.$$ Before stating our main results, we mention the following remarks. \begin{remark} \label{rmk1.2} \rm Notice that using conditions (A4) and (A5), for all $(x,u,v)\in\Omega \times \mathbb{R}^+\times \mathbb{R}^+$, we have \begin{itemize} \item[(A6)] $(r-1)g(x,u)-ug_u(x,u)\leq C_6(1+u^{p-1})$ and $(r-1)h(x,v)-vh_v(x,v)\leq C_7(1+v^{p-1})$. \item[(A7)] $G(x,u)-\frac{1}{r} g(x,u)u \leq C_8(1+u^p)$ and $H(x,v)-\frac{1}{r} h(x,v)v \leq C_9(1+v^p)$, where $$\label{e1.4} G(x,u)=\int_{0}^{u}{g(x,s)ds}, \quad H(x,v)=\int_{0}^{v}{h(x,s)ds}.$$ \end{itemize} \end{remark} \begin{remark} \label{rmk1.3} \rm It should be mentioned that using condition {\rm (A3)} we have $$|\frac{\partial}{\partial t}f(x,tw_1,tw_2)|\leq (1+|\eta(x,w_1,w_2)|)t^{r-1}$$ for $t$ sufficiently large and $(x,w_1,w_2)\in \bar{\Omega}\times (\mathbb{R^+})^2$, hence taking $w_1=\frac{|u|}{|u|+|v|}$, $w_2=\frac{|v|}{|u|+|v|}$ and $t=|u|+|v|$ for $|u|$ and $|v|$ sufficiently large we arrive at \begin{align*} \big|f_u(x,|u|,|v|)|u|+f_v(x,|u|,|v|)|v|\big| &\leq \big(1+|\eta(x,\frac{|u|}{|u|+|v|},\frac{|v|}{|u|+|v|})|\big)(|u|+|v|)^r\\ &\leq A_0(|u|^r+|v|^r), \end{align*} where $A_0=2^r \max\{1+|\eta(x,|u|,|v|)|: |u|+|v|=1\}$ and $(x,u,v)\in \bar{\Omega}\times \mathbb{R}^2$. Furthermore, if we assume that $f\in C^2 (\bar{\Omega}\times \mathbb{R^+}^2)$, then there exists $A_1>0$ such that $$\label{e1.5} |\frac{\partial}{\partial t}f(x,tu,tv)|_{t=1} =|f_u(x,u,v)u+f_v(x,u,v)v| \leq A_1(1+|u|^r+|v|^r),$$ where $(x,u,v)\in \bar{\Omega}\times(\mathbb{R^+})^2$. \end{remark} The purpose of this paper is to prove the following results. \begin{theorem} \label{thm1.1} There exists $K^*\subset (\mathbb{R}^+)^2$ such that for each $(\lambda,\mu)\in K^*$ problem \eqref{e1.1} has at least one positive solution. \end{theorem} \begin{theorem} \label{thm1.2} There exists $K^{**}\subset K^*$ such that for each $(\lambda,\mu)\in K^{**}$ problem \eqref{e1.1} has at least two distinct positive solutions. \end{theorem} This paper is organized as follows. In section 2 we point out some notation and preliminary results and give some properties of Nehari manifold and fibering maps. In section 3 a fairly complete description of the Nehari manifold and fibering maps associated with the problem is given, and finally Theorems \ref{thm1.1} and \ref{thm1.2} are proved in Section 4. \section{Preliminaries and auxiliary results} First, we define the weak solution of problem \eqref{e1.1} as follows. \begin{definition} \label{def2.1} \rm A pair of functions $(u,v)\in W$ ($W$ is given by \eqref{e1.2}) is said to be a weak solution of \eqref{e1.1}, whenever \begin{align*} &\int_{\Omega}{\big(|\nabla u|^{p-2}\nabla u.\nabla {\varphi}_1}+m_1(x)|u|^{p-2}u{\varphi}_1\big)dx\\ &- \lambda\int_{\Omega}g(x,u)\varphi_1dx- \int_{\partial\Omega}{f_u(x,u,v){{\varphi}_1}}dx=0, \\ &\int_{\Omega}{\big(|\nabla v|^{p-2} \nabla v.\nabla {\varphi}_2}+m_2(x)|v|^{p-2}v{\varphi}_2\big)dx\\ &-\mu\int_{\Omega}h(x,v)\varphi_2dx- \int_{\partial\Omega}{f_v(x,u,v){{\varphi}_2}}dx =0, \end{align*} for all $(\varphi_1,\varphi_2)\in W$. \end{definition} Associated with problem \eqref{e1.1}, we consider the energy functional $J_{\lambda,\mu}:W\to\mathbb{R}$ $$\label{e2.1} J_{\lambda,\mu}(u,v)=\frac{1}{p} M(u,v)- F(u,v)-\lambda \int_{\Omega}G(x,|u|)dx-\mu\int_{\Omega}H(x,|v|)dx,$$ where $G(x,u)$ and $H(x,v)$ are introduced in \eqref{e1.4} and $$\label{e2.2} \begin{gathered} M(u,v)=\int_{\Omega}(|\nabla u|^p+m_1(x) |u|^p)dx+\int_{\Omega}(|\nabla v|^p+m_2(x) |v|^p)dx, \\ F(u,v)=\int_{\partial\Omega}f(x,|u|,|v|)dx. \end{gathered}$$ If $J_{\lambda,\mu}$ is bounded from below and $J_{\lambda,\mu}$ has a minimizer on $W$, then this minimizer is a critical point of $J_{\lambda,\mu}$, so it is a solution of \eqref{e1.1}. Since $J_{\lambda,\mu}$ is unbounded from below on whole space $W$, it is useful to consider the functional on the Nehari manifold $$\label{e2.3} \mathcal{N}_{\lambda,\mu}(\Omega)=\{(u,v)\in W\setminus\{(0,0)\} : \langle J_{\lambda,\mu}'(u,v),(u,v) \rangle=0\},$$ where $\langle , \rangle$ denotes the usual duality between $W$ and $W^{-1}$ ( $W^{-1}$ is the dual space of the Sobolev space $W$). We recall that any nonzero solution of problem \eqref{e1.1} belongs to $\mathcal{N}_{\lambda,\mu}(\Omega)$. Moreover, by definition, we have that $(u,v)\in \mathcal{N}_{\lambda,\mu}(\Omega)$ if and only if $$\label{e2.4} \begin{split} &M(u,v)- \int_{\partial\Omega}\big(f_u(x,|u|,|v|)|u|+f_v(x,|u|,|v|)|v|\big)dx\\ &-\lambda\int_{\Omega}g(x,|u|)|u|dx-\mu\int_{\Omega}h(x,|v|)|v|dx=0. \end{split}$$ Furthermore, we have the following result. \begin{theorem} \label{thm2.1} $J_{\lambda,\mu}$ is coercive and bounded from below on $\mathcal{N}_{\lambda,\mu}(\Omega)$ for $\lambda$ and $\mu$ sufficiently small. \end{theorem} \begin{proof} Let $(u, v)\in \mathcal{N}_{\lambda,\mu}(\Omega)$, then by (A2), (A7), \eqref{e1.3} and \eqref{e2.1}--\eqref{e2.4}, we obtain \begin{align*} J_{\lambda,\mu}(u, v) &\geq (\frac{1}{p}-\frac{1}{r})M(u,v) -\lambda\int_{\Omega}\Big(G(x,|u|)-\frac{1}{r}g(x,|u|)|u|\Big)dx\\ &\quad -\mu\int_{\Omega}\Big(H(x,|v|)-\frac{1}{r}h(x,|v|)|v|\Big)dx\\ &\geq(\frac{1}{p}-\frac{1}{r})\|(u,v)\|_W^p-\lambda\int_{\Omega}C_8(1+|u|^p)dx -\mu\int_{\Omega}C_9(1+|v|^p)dx\\ &\geq \frac{r-p}{rp}\|(u,v)\|_W^p-(C_8\lambda+C_9\mu)|\Omega| -(C_8\lambda+C_9\mu)2{{S}}_p^p \|(u,v)\|_W^p, \end{align*} thus $J_{\lambda,\mu}$ is coercive and bounded from below on $\mathcal{N}_{\lambda,\mu}(\Omega)$ provided that $0\leq (C_8\lambda+C_9\mu)2{{S}}_p^p<(r-p)/(rp)$. \end{proof} It can be proved that the points in $\mathcal{N}_{\lambda,\mu}(\Omega)$ correspond to the stationary points of the fibering map $\phi_{u,v}(t): [0,\infty)\to \mathbb{R}$ defined by $\phi_{u,v}( t )= J_{\lambda,\mu}(tu,tv)$, which were introduced by Drabek and Pohozaev in \cite{14} and also discussed in Brown and Zhang \cite{8}. Using \eqref{e2.1} for $(u,v)\in W$, we have \label{e2.5} \begin{gathered} \begin{aligned} \phi_{u,v}(t)&=J_{\lambda,\mu}(tu,tv)\\ &=\frac{t^p}{p} M(u,v)-F(tu,tv)-\lambda\int_{\Omega}G(x,t|u|)dx- \mu\int_{\Omega}H(x,t|v|)dx, \end{aligned}\\ \begin{aligned} \phi_{u,v}'(t) &= t^{p-1}M(u,v)- \int_{\partial\Omega}\nabla f(x,t|u|,t|v|).(|u|,|v|)dx\\ &\quad -\lambda\int_{\Omega}g(x,t|u|)|u|dx- \mu\int_{\Omega}h(x,t|v|)|v|dx, \end{aligned}\\ \begin{aligned} \phi_{u,v}''(t)&= (p-1)t^{p-2}M(u,v)- \int_{\partial\Omega}\mathcal{F}(x,tu,tv)dx\\ &\quad -\lambda\int_{\Omega}g_u(x,t|u|)u^2dx- \mu\int_{\Omega}h_v(x,t|v|)v^2dx, \end{aligned} \end{gathered} where $$\label{e2.6} \begin{gathered} \nabla f(x,u,v):=(\frac{\partial f}{\partial u},\frac{\partial f}{\partial v}), \\ \mathcal{F}(x,tu,tv):=\frac{\partial^2}{\partial t^2} \big(f(x,t|u|,t|v|)=f_{uu}u^2+f_{vv}v^2+2f_{uv}|uv|. \end{gathered}$$ Hence it is natural to divide $\mathcal{N}_{\lambda,\mu}$ into three subsets $\mathcal{N}_{\lambda,\mu}^{+},\, \mathcal{N}_{\lambda,\mu}^{-}\ and \ \mathcal{N}_{\lambda,\mu}^{0}$ which correspond to local minima, local maxima and points of inflection of the fibering maps and so we define $$\label{e2.7} \begin{gathered} \mathcal{N}_{\lambda,\mu}^{+}=\{(u,v)\in \mathcal{N}_{\lambda,\mu}(\Omega): \phi_{u,v}''(1)> 0\},\\ \mathcal{N}_{\lambda,\mu}^{-}=\{(u,v)\in \mathcal{N}_{\lambda,\mu}(\Omega): \phi_{u,v}''(1)< 0\},\\ \mathcal{N}_{\lambda,\mu}^{0}=\{(u,v)\in \mathcal{N}_{\lambda,\mu}(\Omega): \phi_{u,v}''(1)=0\}. \end{gathered}$$ The following lemma shows that minimizers for $J_{\lambda,\mu}(u,v)$ on $N_{\lambda,\mu}(\Omega)$ are usually critical points for $J_{\lambda,\mu}$, as proved by Brown and Zhang in \cite{8} or in Aghajani et al. \cite{2}. \begin{lemma} \label{lem2.1} Let $(u_{0},v_0)$ be a local minimizer for $J_{\lambda,\mu}(u,v)$ on $\mathcal{N}_{\lambda,\mu}(\Omega)$. If $(u_{0},v_0)$ is not in $\mathcal{N}_{\lambda,\mu}^{0}(\Omega)$, then $(u_{0},v_0)$ is a critical point of $J_{\lambda,\mu}$. \end{lemma} Motivated by the above lemma, we give conditions for $\mathcal{N}_{\lambda,\mu}^{0}= \emptyset$. \begin{lemma} \label{lem2.2} There exists $K_0\subset (\mathbb{R^+})^2$ such that for all $(\lambda ,\mu)\in K_0$, we have $\mathcal{N}_{\lambda,\mu}^{0}= \emptyset$. \end{lemma} \begin{proof} Suppose the contrary, that is there exists $(\lambda ,\mu)$ such that $\mathcal{N}_{\lambda,\mu}^{0}\neq \emptyset$. Then for $(u,v)\in \mathcal{N}_{\lambda,\mu}^{0}$ by \eqref{e2.5}--\eqref{e2.7} we have $$\label{e2.8} \begin{split} \phi'_{u,v}(1)&= M(u,v)-\int_{\partial\Omega}\big(\nabla f(x,|u|,|v|).(|u|,|v|)\big)dx\\ &\quad -\lambda\int_{\Omega}g(x,|u|)|u|dx -\mu\int_{\Omega}h(x,|v|)|v|dx=0, \end{split}$$ and by \eqref{e2.7} $\phi''_{u,v}(1)=0$, so $$\label{e2.9} \begin{split} &(p-1)M(u,v)-\int_{\partial\Omega}\big(f_{uu}u^2+f_{vv}v^2+2f_{uv}|uv|\big)dx\\ &-\lambda\int_{\Omega}g_u(x,|u|)u^2dx -\mu\int_{\Omega}h_v(x,|v|)v^2dx=0, \end{split}$$ using (A2) in \eqref{e2.9} we obtain $$\label{e2.10} \begin{split} &(p-1)M(u,v)-(r-1)\int_{\partial\Omega}\nabla f(x,|u|,|v|).(|u|,|v|)dx \\ &-\lambda\int_{\Omega}g_u(x,|u|)u^2dx -\mu\int_{\Omega}h_v(x,|v|)v^2dx\geq0. \end{split}$$ Using \eqref{e1.3}, \eqref{e2.8}, \eqref{e2.10} and condition (A6) we obtain \begin{align*} (r-p)M(u,v) &\leq \lambda\int_{\Omega}\big((r-1)g(x,|u|)-g_u(x,|u|)|u|\big)|u|dx\\ &\quad + \mu\int_{\Omega}\big((r-1)h(x,|v|)-h_v(x,|v|)|v|\big)|v|dx\\ &\leq 2\lambda C_6 \int_{\Omega}(1+|u|^p)dx+2\mu C_7 \int_{\Omega}(1+|v|^p)dx\\ &\leq (2\lambda C_6+2\mu C_7)|\Omega|+(2\lambda C_6+2\mu C_7)2{S}_p^p\|(u,v)\|_W^p, \end{align*} which concludes $$\label{e2.11} M(u,v)\leq\Big(\frac{(2\lambda C_6+2\mu C_7)|\Omega|}{(r-p)-(4\lambda C_6+4\mu C_7) {{S}}_p^p}\Big).$$ Moreover, \eqref{e1.3}, \eqref{e2.2} together (A2) imply $$\label{e2.12} \begin{split} &\int_{\partial\Omega}\big(f_{uu}u^2+f_{vv}v^2+2f_{uv}|uv|\big)dx\\ &\leq r(r-1)\int_{\partial\Omega} C_1(|u|^r+|v|^r)dx\leq 2r(r-1)C_1 \bar{S}_r^r\|(u,v)\|_W^r, \end{split}$$ hence using \eqref{e2.12} in \eqref{e2.9} and taking into account (A5) and \eqref{e1.3} we obtain $$\label{e2.13} M(u,v)\leq L\|(u,v)\|_W^r+(\lambda L'+\mu L'') M(u,v),$$ where $$\label{e2.14} L=\frac{2r(r-1)C_1 \bar{S}_r^r}{p-1}, \quad L'=\frac{C_4{S}^p_p}{p-1}, \quad L''=\frac{C_5{S}^p_p}{p-1}.$$ From \eqref{e2.13} we obtain $$\label{e2.15} M(u,v)\geq \Big(\frac{1-\lambda L'-\mu L''}{L}\Big)^\frac{p}{r-p},$$ so using \eqref{e2.11} we must have \begin{equation*} \Big(\frac{1-\lambda L'-\mu L''}{L}\Big)^\frac{p}{r-p}\leq\Big(\frac{(2\lambda C_6+2\mu C_7)|\Omega|}{(r-p)-(4\lambda C_6+4\mu C_7) {{S}}_p^p}\Big), \end{equation*} which is a contradiction for $\lambda ,\mu$ sufficiently small. So there exists $K_0\subset(\mathbb{R}^+)^2$ such that for $(\lambda ,\mu )\in K_0$, $\mathcal{N}_{\lambda,\mu}^{0}= \emptyset$. \end{proof} \begin{definition} \label{def2.2} \rm A sequence $y_n=(u_{n},v_n)\subset W$ is called a Palais-Smale sequence if $I_{\lambda,\mu}(y_{n})$ is bounded and $I'_{\lambda,\mu}(y_{n})\to 0$ as $n\to\infty$. If $I_{\lambda,\mu}(y_{n})\to c$ and $I'_{\lambda,\mu}(y_{n})\to 0$, then $y_n$ is a $(PS)_c$-sequence. It is said that the functional $I_{\lambda,\mu}$ satisfies the Palais-Smale condition (or $(PS)_{c}$-condition), if each Palais-Smale sequence ($(PS)_c$-sequence) has a convergent subsequence. \end{definition} Now we prove the boundedness of Palais-Smale sequences. \begin{lemma} \label{lem2.3} If $\{(u_n,v_n)\}$ is a $(PS)_{c}$-sequence for $J_{\lambda,\mu}$, then $\{(u_n,v_n)\}$ is bounded in $W$ provided that $(\lambda,\mu)\in K_1=\{(\lambda,\mu): r-p-4r(C_8\lambda+C_9\mu){{S}}_p^p>0\}$. \end{lemma} \begin{proof} Using \eqref{e1.3}, \eqref{e2.5}, (A2) and (A7) we have \begin{align*} &J_{\lambda,\mu} (u_n,v_n) -\frac{1}{r}\langle J_{\lambda,\mu}'(u_n,v_n),(u_n,v_n)\rangle\\ &\geq\frac{r-p}{rp}M(u_n,v_n) -\lambda\int_{\Omega}(G(x,|u_n|)-\frac{1}{r}g(x,|u_n|)|u_n|)dx\\ &\quad -\mu\int_{\Omega}(H(x,|v_n|)-\frac{1}{r}h(x,|v_n|)|v_n|)dx\\ &\geq\frac{r-p}{rp} M(u_n,v_n)-\lambda\int_{\Omega}C_8(1+|u_n|^p)dx -\mu\int_{\Omega}C_9(1+|v_n|^p)dx \\ &\geq \frac{r-p-4r(C_8\lambda +C_9\mu){{S}}_p^p}{rp}\|(u_n,v_n)\|_W^p-(C_8\lambda+C_9\mu)|\Omega|, \end{align*} so for $(\lambda,\mu)\in K_1$, $\{(u_{n},v_n)\}$ is bounded in $W$. \end{proof} \begin{lemma} \label{lem2.4} There exists $K_2\subset \mathbb{R}^2$ such that if $(\lambda,\mu)\in K_2$ and $(u,v)\in N_{\lambda,\mu}^-$, then $\int_{\partial\Omega}\mathcal{F}(x,u,v)dx>0$, where $\mathcal{F}(x,u,v)$ is defined by \eqref{e2.6}. \end{lemma} \begin{proof} Suppose otherwise, then $-\int_{\partial\Omega}\mathcal{F}(x,u,v)dx\geq0$ and from \eqref{e2.5} and \eqref{e2.7} we obtain \begin{align*} \phi''_{u,v}(1)&=(p-1)M(u,v)-\int_{\partial\Omega}\mathcal{F}(x,u,v)dx\\ &\quad -\lambda\int_{\Omega}g_u(x,|u|)u^2dx -\mu\int_{\Omega}h_v(x,|v|)v^2dx<0, \end{align*} so by \eqref{e1.3}, \eqref{e2.2}, \eqref{e2.14} and condition (A5) we have \begin{align*} \|(u,v)\|_W^p &\leq\frac{\lambda}{p-1}\int_{\Omega}g_u(x,|u|)u^2dx +\frac{\mu}{p-1}\int_{\Omega}h_v(x,|v|)v^2dx \\ &\leq(\lambda L ' +\mu L'')\|(u,v)\|_W^p, \end{align*} which is a contradiction for $(\lambda,\mu)\in K_2=\{(\lambda,\mu): \ \lambda L' +\mu L"<1\}$. \end{proof} \section{Properties of Nehari manifold and fibering maps} To obtain a better understanding of the behavior of fibering maps, we will describe the nature of the derivative of the fibering maps for all possible signs of $\int_{\partial\Omega}\mathcal{F}(x,tu,tv)dx$ (by (A1) and \eqref{e2.6}, $\int_{\partial\Omega}\mathcal{F}(x,tu,tv)dx$ has the same sign for every $t>0$). Define the functions $R(t)$ and $S(t)$ as follows \begin{gather} \label{e3.1} R(t):=\frac{1}{p} t^pM(u,v)- F(tu,tv) \quad (t>0),\\ \label{e3.2} S(t):=\lambda\int_{\Omega}G(x,t|u|)dx+\mu\int_{\Omega}H(x,t|v|)dx \quad (t>0), \end{gather} then from \eqref{e2.5} it follows that $\phi_{u,v}(t)=R(t)-S(t)$. Moreover, $\phi'_{u,v}(t)=0$ if and only if $R'(t)=S'(t)$, where $$\label{e3.3} R'(t)=t^{p-1}M(u,v)-\int_{\partial\Omega}\Big(f_u(x,t|u|,t|v|)|u| +f_v(x,t|u|,t|v|)|v|\Big)dx,$$ and $$\label{e3.4} S'(t)=\lambda\int_{\Omega}g(x,t|u|)|u|dx+\mu\int_{\Omega}h(x,t|v|)|v|dx.$$ In the next result we see that, $\phi_{u,v}$ and $\phi'_{u,v}$ take on positive values for all nonzero $(u,v)\in W$ whenever, $\lambda$ and $\mu$ belong to a certain subset of $\mathbb{R}^2$. \begin{lemma} \label{lem3.1} There exists $K_3\subset (\mathbb{R}^+)^2$ such that for all nonzero $(u,v)\in W$, $\phi_{u,v}(t)$ and $\phi'_{u,v}(t)$ take on positive values whenever $(\lambda,\mu)\in K_3$. \end{lemma} \begin{proof} First we show that $\phi_{u,v}(t)$ takes on positive values, for all possible signs of $\int_{\partial\Omega}\mathcal{F}(x,tu,tv)dx$. If $\int_{\partial\Omega}\mathcal{F}(x,tu,tv)dx\leq0$, then by \eqref{e3.1} $R''(t)\geq0$ and using \eqref{e3.2}, $R(t)>S(t)$ for t sufficiently large, so $\phi_{u,v}(t)>0$ for t sufficiently large . Now, suppose there exists $(u,v)\in W$ such that $\int_{\partial\Omega}\mathcal{F}(x,tu,tv)dx\geq0$. Condition (A2) together \eqref{e3.1} imply that \begin{equation*} R(t)\geq \frac{1}{p} t^pM(u,v)-C_1t^r\int_{\partial\Omega}(|u|^r+|v|^r)dx. \end{equation*} Define $$\label{e3.5} K(t):=\frac{1}{p} t^pM(u,v)-C_1t^r\int_{\partial\Omega}(|u|^r+|v|^r)dx \quad (t>0),$$ we obtain $R(t)\geq K(t)$, and by elementary calculus, we see that $K(t)$ takes a maximum value at $$\label{e3.6} t_{\rm max}=\Big(\frac{M(u,v)}{ rC_1\int_{\partial\Omega}(|u|^r+|v|^r)dx)}\Big)^{\frac{1}{r-p}},$$ then follows by \eqref{e3.1}, \eqref{e3.6}, \eqref{e1.3} and \eqref{e2.2} that $$\label{e3.7} \begin{split} R(t_{\rm max}) &\geq K(t_{\rm max}) =\frac{r-p}{rp}\Big(\frac{(\|(u,v)\|^p_{W})^{r}}{\big(rC_1\int_{\partial\Omega} (|u|^r+|v|^r)dx\big)^p}\Big)^{\frac{1}{r-p}}\\ &\geq\frac{r-p}{rp}\Big(\frac{1}{(2rC_1)^p{\bar{S}}^{rp}_{r}}\Big)^{\frac{1}{r-p}}=\delta_1, \end{split}$$ where $\delta_1$ is independent of $(u,v)$. Now from \eqref{e3.6}, \eqref{e3.7} and \eqref{e1.3} for $1\leq \alpha0$ for all nonzero $(u,v)\in W$. Now we prove that $\phi'_{u,v}(t)$ takes on positive values. If $\int_{\partial\Omega}\mathcal{F}(x,tu,tv)dx\leq0$, then using \eqref{e3.1}, \eqref{e3.2} $\phi'_{u,v}(t)\geq0$ for t sufficiently large. Suppose that, there exists $(u,v)\in W$ such that $\int_{\partial\Omega}\mathcal{F}(x,tu,tv)dx\geq0$. To verify that $\phi'_{u,v}(t)>0$, it is sufficient to show that $tR'(t)>tS'(t)$. Using (A2) and \eqref{e3.3} we have \begin{equation*} tR'(t)\geq t^pM(u,v)-rC_1t^r\int_{\partial\Omega}(|u|^r+|v|^r)dx. \end{equation*} In view of \eqref{e3.5}, we write $$\label{e3.11} \bar{K}(t):= t^pM(u,v)-rC_1t^r\int_{\partial\Omega}(|u|^r+|v|^r)dx \ \ (t>0),$$ so $tR'(t)>\bar{K}(t)$ and by elementary calculus we can show that $\bar{K}(t)$ achieves its maximum at $$\label{e3.12} \tau_{\rm max} =\Big(\frac{pM(u,v)}{r^2C_1\int_{\partial\Omega}(|u|^r+|v|^r)dx}\Big)^{\frac{1}{r-p}}.$$ Using \eqref{e1.3}, \eqref{e3.3}, \eqref{e3.11} and \eqref{e3.12}, we arrive at $$\label{e3.13} \begin{split} \tau_{\rm max}R'(\tau_{\rm max}) &=(\frac{p}{r^2C_1})^\frac{p}{r-p}(\frac{r-p}{r})\Big(\frac{(\| (u,v)\|^p)^{r}}{(\int_{\partial\Omega}(|u|^r+|v|^r)dx) ^p}\Big) ^{\frac{1}{r-p}}\\ &\geq(\frac{p}{r^2C_1})^\frac{p}{r-p}(\frac{r-p}{r}) \Big(\frac{1}{2^p{\bar{S}}_r^{rp}}\Big)^{\frac{1}{r-p}} =\delta_2>0, \end{split}$$ where ${\delta}_2$ is independent of $(u,v)$. Using \eqref{e1.3}, \eqref{e3.12} and \eqref{e3.13}, and by some calculations very Similar to \eqref{e3.8}, we obtain $$\label{e3.14} (\tau_{\rm max})^{\beta}\int_{\Omega}(|u|^{\beta}+|v|^\beta)dx \leq c_2(\tau_{\rm max}R'(\tau_{\rm max})\big)^{\frac{\beta}{p}},$$ for $1\leq \beta<2^*$. Then using \eqref{e1.3}, \eqref{e3.4}, \eqref{e3.14} and condition (A4) we find \begin{align*} \tau_{\rm max}S'(\tau_{\rm max}) &=\lambda\tau_{\rm max}\int_{\Omega}g(x,\tau_{\rm max}|u|)|u|dx+ \mu\tau_{\rm max}\int_{\Omega}h(x,\tau_{\rm max}|v|)|v|dx\\ &\leq \lambda\int_{\Omega}C_1(|t_{\rm max}u|+|t_{\rm max}u|^p)dx+ \mu\int_{\Omega}C_3(|t_{\rm max}v|+|t_{\rm max}v|^p)dx\\ &\leq (\lambda e_0+\mu e_1)\Big((t_{\rm max}R'(t_{\rm max}))^\frac{1}{p} +t_{\rm max}R'(t_{\rm max})\Big), \end{align*} where $e_0$ and $e_1$ are independent of $(u,v)$, so from the above inequality and \eqref{e3.13}, we obtain \begin{align*} &\tau_{\rm max} \phi'_{u,v}(\tau_{\rm max})= \tau_{\rm max}R'(\tau_{\rm max})-\tau_{\rm max}S'(\tau_{\rm max})\\ &\geq \tau_{\rm max}R'(\tau_{\rm max})\Big(1- (\lambda e_0+\mu e_1)\big((\tau_{\rm max}R_\lambda'(\tau_{\rm max}))^\frac{1-p}{p} +1\big)\Big)\\ &\geq \delta_2\Big(1- (\lambda e_0+\mu e_1)\big(\delta_2^\frac{1-p}{p} +1\big)\Big), \end{align*} Clearly for all nonzero $(u,v)\in W$, $\tau_{\rm max} \phi'_{u,v}(\tau_{\rm max})>0$ provided that $2(\lambda e_0+\mu e_1)\big(\delta_2^\frac{1}{p} +\delta_2\big)<\delta_2$. Using the above inequality and \eqref{e3.10}, we obtain that if $(\lambda,\mu)\in K_3$, where $$\label{e3.15} K_3=\{(\lambda,\mu): 2(\lambda B_0+\mu B_1)(1+\delta_1)<\delta_1 \text{ and } 2(\lambda e_0+\mu e_1)\big(\delta_2^\frac{1}{p} +\delta_2\big)<\delta_2\},$$ then $\phi_{u,v}(t)$ and $\phi'_{u,v}(t)$ take on positive values for all nonzero $(u,v)\in W$ and this completes the proof. \end{proof} \begin{corollary} \label{coro3.1} If $(\lambda,\mu)\in K_2\cap K_3$, then there exists $\varepsilon>0$ such that $J_{\lambda,\mu}(u,v)>\epsilon$ for all $(u,v)\in \mathcal{N}_{\lambda,\mu}^{-}$. \end{corollary} \begin{proof} If $(u,v)\in N_{\lambda,\mu}^-$, then by lemma \ref{lem2.4}, $\int_{\partial\Omega}\mathcal{F}(x,u,v)dx>0$. Also due to (A1) and (A5), $\phi_{u,v}$ has a positive global maximum at $t=1$ and so by \eqref{e2.5}, \eqref{e3.10} and \eqref{e3.15} $$J_{\lambda,\mu}(u,v)=\phi_{u,v}(1)\geq \phi_{u,v}(t_{\rm max}) \geq\delta_1\big (1-(\lambda B_0+\mu B_1)(\delta_1^{-1}+1)\big)\geq \delta_1/2= \varepsilon>0.$$ \end{proof} From (A1) and \eqref{e2.6}, $\int_{\partial\Omega}\mathcal{F}(x,tu,tv)dx$ has the same sign for every $t>0$, so we have the following corollary. \begin{corollary} \label{coro3.2} for $(u,v)\in W\setminus \{(0,0)\}$ we have \begin{itemize} \item[(i)] If $\int_{\partial\Omega}\mathcal{F}(x,tu,tv)dx\leq0$, then there exists $t_1$ such that $(t_1u,t_1v)\in N_{\lambda,\mu}^{+}$ and $\phi_{u,v}(t_1)<0$. \item[(ii)] If $\int_{\partial\Omega}\mathcal{F}(x,tu,tv)dx\geq0$ and $(\lambda,\mu)\in K_3$, then there exist $00$ such that $\phi'_{u,v}(t_1)=0$. Now using (A1) and (A5), for $0t_1$, $\phi'_{u,v}(t)>0$, therefore $(t_1u,t_1v)\in N_{\lambda,\mu}^{+}$ and $\phi_{u,v}(t_1)<\phi_{u,v}(0)=0$. (ii) Using \eqref{e2.5}, (A3), (A5) and the assumption that $\int_{\partial\Omega}\mathcal{F}(x,tu,tv)dx\geq0$ we obtain $\lim_{t\to\infty}\phi'_{u,v}(t)=- \infty$, $\phi'_{u,v}(0)<0$ and by Lemma \ref{lem3.1} we have $\phi'_{u,v}(\tau)>0$ for suitable $\tau>0$, so using again the intermediate value theorem concludes that there exist $t_1$ and $t_2$ such that $0p\geq2$. Hence from the compactness of the embeddings $W^{1,p}\hookrightarrow L^{\alpha}(\Omega)$ and $W^{1,p}\hookrightarrow L^{\alpha}(\partial\Omega)$ for $1\leq \alpha \phi'_{u_1,v_1}(t_1)=0$ for $n$ sufficiently large. Since $\{(u_{n},v_n)\}\subseteq N^{+}_{\lambda,\mu}(\Omega)$, by considering the possible fibering maps it is easy to see that, $\phi'_{u_{n},v_n}(t)<0$ for $01$, but $(t_1u_1,t_1 v_1)\in N^{+}_{\lambda,\mu}$ and so \begin{align*} &J_{\lambda,\mu}(t_1u_1,t_1 v_1) =\phi_{u_1,v_1}(t_1)<\phi_{u_1,v_1}(1)\\ &<\lim_{n \to \infty}\phi_{u_{n},v_n}(1) = \lim_{n \to \infty}J_{\lambda,\mu}(u_{n},v_n) = \inf_{(u,v)\in \mathcal{N}_{\lambda,\mu}^{+}} J_{\lambda,\mu}(u,v), \end{align*} which is a contradiction. Therefore, $(u_{n}v_n)\to (u_1,v_1)$ in $W$ and this concludes that $J_{\lambda,\mu}(u_1,v_1)= \lim_{n \to \infty}J_{\lambda,\mu}(u_{n},v_n)= \inf_{(u,v)\in \mathcal{N}_{\lambda,\mu}^{+}} J_{\lambda,\mu}(u,v).$ Thus $(u_1,v_1)$ is a minimizer for $J_{\lambda,\mu}$ on $\mathcal{N}_{\lambda,\mu}^{+}(\Omega)$. (ii) By Corollary \ref{coro3.1} we have $J_{\lambda,\mu}(u,v)\geq\varepsilon>0$ for all $(u,v)\in \mathcal{N}_{\lambda,\mu}^{-}$, so $\inf_{(u,v)\in\mathcal{N}_{\lambda,\mu}^{-}}J_{\lambda,\mu}(u,v)>0,$ hence, there exists a minimizing sequence $\{(u_{n},v_n)\}\subseteq \mathcal{N}_{\lambda,\mu}^{-}(\Omega)$ such that $$\label{e4.3} \lim_{n \to \infty}J_{\lambda,\mu}(u_{n},v_n)= \inf_{(u,v)\in \mathcal{N}_{\lambda,\mu}^{-}} J_{\lambda,\mu}(u,v)>0.$$ Similar to the argument in the proof of (i) we find that $\{(u_{n},v_n)\}$ is bounded in $W$ and also the results obtained in \eqref{e4.1} are satisfied for $\{(u_{n},v_n)\}$ and $\{(u_2,v_2)\}$. Since $(u_{n},v_n)\in \mathcal{N}_{\lambda,\mu}^{-}(\Omega)$, so by \eqref{e2.7}, $\phi''_{u_n, v_n}(1)<0$, letting $n\to\infty$, by \eqref{e2.5}, Remark \ref{rmk4.1} and the above argument we see that $$\label{e4.4} \begin{split} \phi''_{u_2,v_2}(1) &=M(u_2,v_2)-\int_{\partial\Omega}\mathcal{F}(x,u_2,v_2)dx -\lambda\int_{\Omega}g_{u}(x,|u_2|)u_2^2dx\\ &\quad -\mu\int _{\Omega}h_{v}(x,|v_2|)v_2^2dx\leq0. \end{split}$$ On the other hand for $(u_{n},v_n)\in \mathcal{N}_{\lambda,\mu}^{-}$, by Lemma \ref{lem2.4}, $\int_{\partial\Omega}\mathcal{F}(x,u_{n},v_n)dx>0$. Letting $n\to\infty$, we see that $\int_{\partial\Omega}\mathcal{F}(x,u_2,v_2)dx\geq0$. We claim that $\int_{\partial\Omega}\mathcal{F}(x,u_2,v_2)dx\neq0$. If $\int_{\partial\Omega}\mathcal{F}(x,u_2,v_2)dx=0$, then by (A4), \eqref{e1.3}, \eqref{e2.14} and \eqref{e4.4} we have \begin{equation*} M(u_2,v_2)\leq\lambda\int _{\partial\Omega}g_{u}(x,|u_2|)u_2^2dx+\mu\int _{\partial\Omega}h_{v}(x,|v_2|)v_2^2dx\leq(\lambda L+\mu L')M(u_2,v_2), \end{equation*} which is a contradiction for $(\lambda,\mu) \in K_2$. So $\int_{\partial\Omega}\mathcal{F}(x,u_2,v_2)dx>0$ and by Corollary \ref{coro3.2} (ii) there exists $t_2>0$ such that $(t_2u_2,t_2v_2)\in \mathcal{N}_{\lambda,\mu}^{-}(\Omega)$. We claim that $(u_{n},v_n)\to (u_2,v_2)$ in $W$. Suppose that this is false, so we have $$\label{e4.5} M(u_2,v_2)< \liminf_{n\to \infty }M(u_{n},v_n).$$ However, $(u_{n},v_n)\in \mathcal{N}_{\lambda,\mu}^{-}$ and so $J_{\lambda,\mu}(u_{n},v_n)\geq J_{\lambda,\mu}(tu_{n},tv_n)$ for all $t\geq0$. Therefore, considering \eqref{e2.5}, \eqref{e4.3}--\eqref{e4.5} and Remark \eqref{e4.1}, we can write \begin{align*} &J_{\lambda,\mu}(t_2u_2,t_2v_2)\\ &=\frac{t_2^p}{p} M(u_2,v_2)- F(t_2u_2,t_2v_2)-\lambda \int_{\Omega}H(x,t_2|u_2|)dx-\mu\int_{\Omega}G(x,t_2|v_2|)dx\\ & <\lim_{n \to \infty}\Big(\frac{t_2^p}{p} M(u_n,v_n)-F(t_2u_n,t_2v_n)-\lambda \int_{\Omega}H(x,t_2|u_n|)dx\\ &\quad -\mu\int_{\Omega}G(x,t_2|v_n|)dx\Big)\\ & = \lim_{n \to \infty}J _{\lambda,\mu}(t_2u_{n},t_2v_n)\leq \lim_{n \to \infty} J_{\lambda,\mu}(u_{n},v_n)= \inf_{(u,v)\in \mathcal{N}_{\lambda,\mu}^{-}} J_{\lambda,\mu}(u,v), \end{align*} which is a contradiction. So, $(u_{n},v_n)\to (u_2,v_2)$ in $W$ and the proof is complete. \end{proof} \begin{proof}[Proof of Theorem \ref{thm1.1}] By Lemma \ref{lem4.1} (i) there exists $(u_1,v_1)\in N_{\lambda,\mu}^{+}(\Omega)$ such that $J_{\lambda,\mu}(u_1,v_1)=\inf_{(u,v) \in N_{\lambda,\mu}^{+}} J_{\lambda,\mu}(u,v)$ and by Lemmas \ref{lem2.1} and \ref{lem2.2}, $(u_1,v_1)$ is a critical point of $J_{\lambda,\mu}$ on $W$ and hence is a weak solution of problem \eqref{e1.1}. On the other hand $J_{\lambda,\mu}(u,v)=J_{\lambda,\mu}(|u|,|v|)$, so we may assume that $(u_1,v_1)$ is a positive solution and the proof is complete. \end{proof} \begin{proof}[Proof of Theorem \ref{thm1.2}] By Lemma \ref{lem4.1} there exist $(u_1,v_1)\in N_{\lambda,\mu}^{+}(\Omega)$ and $(u_2,v_2)\in N_{\lambda,\mu}^{-}(\Omega)$ such that $$J_{\lambda,\mu}(u_1,v_1)=\inf_{(u,v) \in N_{\lambda,\mu}^{+}} J_{\lambda,\mu}(u,v),\quad J_{\lambda,\mu}(u_2,v_2)=\inf_{(u,v) \in N_{\lambda,\mu}^{-}}J_{\lambda,\mu}(u,v).$$ By Lemmas \ref{lem2.1} and \ref{lem2.2}, $(u_1,v_1)$ and $(u_2,v_2)$ are critical points of $J_{\lambda,\mu}$ on $W$ and hence are weak solutions of problem \eqref{e1.1}. Similar to the proof of Theorem \ref{thm1.1}, we may assume that $(u_1,v_1)$ and $(u_2,v_2)$ are positive solutions. Also since $N_{\lambda,\mu}^{+}\cap N_{\lambda,\mu}^{-}={\emptyset}$, this implies that $(u_1,v_1)$ and $(u_2,v_2)$ are distinct and the proof is complete. \end{proof} \begin{thebibliography}{99} \bibitem{1} S. Adachi, K. Tanaka; \emph{Four positive solutions for the semilinear elliptic equations $-\Delta u +u= a(x)u^p +f(x)$ in $\mathbb{R}^N$}, Calc. var. 11 (2000) 63-95 . \bibitem{2} A. Aghajani, J. Shamshiri, F. M. Yaghoobi; \emph{Existence and multiplicity of positive solutions for a class of nonlinear elliptic problems}, Turk. J. Math (2012) doi:10.3906/mat-1107-23. \bibitem{3} A. Aghajani, F. M. Yaghoobi, J. Shamshiri; \emph{Existence and multiplicity of nontrivial nonnegative solutions for a class of guasilinear p-Laplacian systems}, Dynamics of Continuous, Discrete and Impulsive Systems Series A: Mathematical Analysis 19 (2012) 383-396. \bibitem{4} P. A. Binding, P. Drabek, Y. X. Huang; \emph{On Neumann boundary value problems for some quasilinear elliptic equations}, Nonlinear Analysis 42 (2000) 613-629. \bibitem{5} H. Brezis; \emph{Functional Analysis}, Sobolev Spaces and Partial Differential Equations, Springer, New York, 2010. \bibitem{6} K. J. Brown, T.-F. Wu; \emph{A fibering map approach to a semilinear elliptic boundary value problem}, J. Differential Equations 69 (2007) 1-9. \bibitem{7} K. J. Brown, T.-F. Wu; \emph{A semilinear elliptic system involving nonlinear boundary condition and sign-changing weight function}, J. Math. Anal. Appl. 337 (2008) 1326-1336. \bibitem{8} K. J. Brown, Y. Zhang; \emph{The Nehari manifold for a semilinear elliptic problem with a sign changing weight function}, J. Differential Equations 193 (2003) 481-499. \bibitem{9} Z. Chen, W. Zou; \emph{Ground states for a system of of Schrodinger equations with critical exponent}, Journal of Functional Analysis 262 (2012) 3091-3107. \bibitem{10} C. M. Chu, C. L. Tang; \emph{Existence and multiplicity of positive solutions for semilinear elliptic systems with Sobolev critical exponents}, Nonlinear Anal 71 (2009) 5118-5130. \bibitem{11} F. C. Cirstea, Y. Du; \emph{Isolated singularities for weighted quasilinear elliptic equations}, Journal of Functional Analysis 259 (2010) 174-202. \bibitem{12} D. Daners, P. Drabek; \emph{A priori estimates for a class of quasi-linear of elliptic equations}, Amer. Math. Soc 361, 12 (2009) 6475-6500. \bibitem{13} P. Drabek; \emph{Resonance Problems for the p-Laplacian}, Journal of Functional Analysis 169 (1999) 189-200. \bibitem{14} P. Drabek, S. I. Pohozaev; \emph{Positive solutions for the p-Laplacian: application of the fibering method}, Proc. Royal Soc. Edinburgh Sect. A 127 (1997) 703-726. \bibitem{15} P. Drabek, I. Schindler; \emph{Positive solutions for the p-Laplacian with Robin boundary conditions on irregular domains}, Applied Mathematics Letters 24 (2011) 588-591. \bibitem{16} I. Ekeland; \emph{On the variational principle}, J. Math. Anal. Appl. 47 (1974) 324-353. \bibitem{17} G. M. Figueiredo, M. F. Furtado; \emph{Multiple positive solutions for a quasilinear system of Schr\"odinger equations}, Nonlinear differ. equ. appl. 15 (2008) 309-333. \bibitem{18} M. F. Furtado, J. P. P. da Silva; \emph{Multiplicity of solutions for homogeneous elliptic systems with critical growth}, J. Math. Anal. Appl. 385 (2012) 770-785. \bibitem{19} P. Han; \emph{High-energy positive solutions for a critical growth Dirichlet problem in noncontractible domains}, Nonlinear Anal. 60 (2005) 369-387. \bibitem{20} P. Han; \emph{Multiple positive solutions of nonhomogeneous elliptic systems involving critical Sobolev exponents}, Nonlinear Anal. 64 (2006) 869-886. \bibitem{21} P. Han; \emph{The effect of the domain topology on the number of positive solutions of an elliptic system involving critical Sobolev exponents}, Houston J. Math. 32 (2006) 1241-1257. \bibitem{22} L. Jeanjean; \emph{Two positive solutions for a class of nonhomogeneous elliptic equations}, Diff. Int. Equations 10, 4 (1997) 609-624. \bibitem{22b} D. A. Kandilakis, M. Magiropoulos, N. B. Zographopoulos; \emph{The first eigenvalue of p-Laplacian systems with nonlinear boundary conditions}, Bound. Value Probl., 3 (2005), 307-321. \bibitem{23} S. E. Manouni; \emph{A study of nonlinear problems for the p-Laplacian in $\mathbb{R}^{n}$ via Ricceri’s principle}, Nonlinear Analysis 74 (2011) 4496-4502. \bibitem{24} K. Narukawa, Y. Takajo; \emph{Existence of nonnegative solutions for quasilinear elliptic equations with indefinite critical nonlinearities}, Nonlinear Analysis 74 (2011) 5793-5813. \bibitem{25} Z.-Q. Ou, C.-L. Tang; \emph{Existence and multiplicity of nontrivial solutions for quasilinear elliptic systems}, J. Math. Anal. Appl 383 (2011) 423-438. \bibitem{26} S. H. Rasouli, G. A. Afrouzi; \emph{The Nehari manifold for a class of concave-convex elliptic systems involving the p-Laplacian and nonlinear boundary condition}, Nonlinear Analysis 73 (2010) 3390-3401. \bibitem{27} Y. Shen, J. Zhang; \emph{Multiplicity of positive solutions for a navier boundary-value problem involving the p-biharmonic with critical exponent}, Electronic Journal of Differential Equations 2011, 47 (2011) 1-14. \bibitem{28} Y. Shen, J. Zhang; \emph{Multiplicity of positive solutions for a semilinear p-Laplacian system with Sobolev critical exponent}, Nonlinear Analysis 74 (2011) 1019-1030. \bibitem{29} M. Struwe; \emph{Variational methods Springer}, Berlin, 1990. \bibitem{30} M. Willem; \emph{Minimax Theorems}, Birkhauser, Boston, 1996. \bibitem{31} T.-F. Wu; \emph{The Nehari manifold for a semilinear elliptic system involving sign-changing weight functions}, Nonlinear Analysis 68 (2008) 1733-1745. \end{thebibliography} \end{document}