\documentclass[reqno]{amsart} \AtBeginDocument{{\noindent\small {\em Electronic Journal of Differential Equations}, Vol. 2005(2005), No. 20, pp. 1--10.\newline ISSN: 1072-6691. URL: http://ejde.math.txstate.edu or http://ejde.math.unt.edu \newline ftp ejde.math.txstate.edu (login: ftp)} \thanks{\copyright 2005 Texas State University - San Marcos.} \vspace{9mm}} \begin{document} \title[\hfilneg EJDE-2005/20\hfil Quasilinear elliptic systems] {Remarks on a class of quasilinear elliptic systems involving the (p,q)-Laplacian} \author[G. Zhang, X. Liu, S. Liu\hfil EJDE-2005/20\hfilneg] {Guoqing Zhang, Xiping Liu, Sanyang Liu} \address{Guoqing Zhang\hfill\break Department of Applied Mathematics, Xidian University, Xi'an Shaanxi, 710071, China \hfill\break College of Science, University of Shanghai for science and Technology, Shanghai, 200093, China} \email{zgqw2001@sina.com.cn} \address{Xiping Liu \hfill\break College of Science, University of Shanghai for science and Technology, Shanghai, 200093, China} \email{xipingliu@163.com} \address{Sanyang Liu \hfill\break Department of Applied Mathematics, Xidian University, Xi'an Shaanxi, 710071, China} \email{liusanyang@263.net} \date{} \thanks{Submitted July 16, 2004. Published February 8, 2005.} \subjclass[2000]{35B32, 35J20,35J50, 35P15} \keywords{Nehari manifold; (p,q)-Laplacian; variational methods} \begin{abstract} We study the Nehari manifold for a class of quasilinear elliptic systems involving a pair of (p,q)-Laplacian operators and a parameter. We prove the existence of a nonnegative nonsemitrivial solution for the systems by discussing properties of the Nehari manifold, and so global bifurcation results are obtained. Thanks to Picone's identity, we also prove a nonexistence result. \end{abstract} \maketitle \numberwithin{equation}{section} \newtheorem{theorem}{Theorem}[section] \newtheorem{lemma}[theorem]{Lemma} \newtheorem{definition}[theorem]{Definition} \section{Introduction} Consider the quasilinear elliptic boundary-value problem \begin{equation} \label{P} \begin{aligned} -\Delta_p u&=\lambda a(x)|u|^{p-2}u+\lambda b(x)|u|^{\alpha-1}|v|^{\beta+1}u\\ &\quad +\frac{\mu(x)}{(\alpha+1)(\delta+1)}|u|^{\gamma-1}|v|^{\delta+1}u \quad\mbox{in } \Omega\\ -\Delta_q v &=\lambda d(x)|v|^{q-2}v+\lambda b(x)|u|^{\alpha+1}|v|^{\beta-1}v\\ &\quad +\frac{\mu(x)}{(\beta+1)(\gamma+1)}|u|^{\gamma+1}|v|^{\delta-1}v \quad\mbox{in } \Omega\\ & u=0,\quad v=0 \quad\mbox{on } \partial\Omega\,, \end{aligned} \end{equation} where $\Omega$ is a bounded domain in $\mathbb{R}^N$ with smooth boundary $\partial \Omega$, $\lambda>0$ is a real parameter, and $\Delta_p u=\mathop{\rm div}(|\nabla u|^{p-2}\nabla u)$ is the p-Laplacian operator with $1
\lambda_1$ such that Problem \eqref{P} has two nonnegative nonsemitrival solutions wherever $\lambda \in (\lambda_1,\lambda^*)$. i.e. $\lambda=\lambda_1$ is a bifurcation point, and bifurcation is to the right when $ \lambda >\lambda_1$. In this paper, under the condition \begin{equation} \int_\Omega \mu(x)|u_1|^{\gamma+1}|v_1|^{\delta+1}\mathrm{d}x>0, \label{e1.3} \end{equation} we prove the existence of a nonnegative nonsemitrival solution for Problem \eqref{P} when $\lambda < \lambda_{1}$. i.e. the bifurcation is to the left. Combining this with the result of \cite{Dra}, we obtain global bifurcation results for Problem \eqref{P}, for which the corresponding bifurcation diagrams are shown in Fig 1. In addition, a nonexistence result is proved by using Picone's identity when $\lambda > \lambda_1$. \begin{figure}[th] \begin{center} \setlength{\unitlength}{1mm} \begin{picture}(100,30)(-16,-10) \put(-5,0){\vector(1,0){30}} \put(0,-5){\vector(0,1){25}} \put(-3,-4){0} \put(11,-4){$\lambda_1$} \put(23,1.5){$\lambda$} \put(-16,17){$\|(u,v)\|_X$} \put(-16,-10){(a) $\int_\Omega\mu(x)|u_1|^{\gamma+1}|v_1|^{\delta+1}\, \mathrm{d}x>0$} \qbezier(12,0)(11,8)(2,11) % \put(50,0){\vector(1,0){30}} \put(55,-5){\vector(0,1){25}} \put(39,17){$\|(u,v)\|_X$} \put(52,-4){0} \put(66,-4){$\lambda_1$} \put(73,-4){$\lambda^*$} \put(74,-1){\line(0,1){2}} \put(78,1.5){$\lambda$} \put(39,-10){b) $\int_\Omega\mu(x)|u_1|^{\gamma+1}|v_1|^{\delta+1}\,\mathrm{d}x<0$} \qbezier(67,0)(85,9)(57,15) \end{picture} \end{center} \caption{Bifurcation diagrams for Problem \eqref{P}} \end{figure} This paper is organized as follows. In section 2, we introduce notation, give some definitions, and state our basic assumptions. Section 3 is devoted to giving a detailed description of Figure 1 (a). In section 4, we prove a nonexistence result. \subsection{Remarks} (1) Figure 1 shows how the sign of $\int_\Omega \mu(x)|u_1|^{\gamma+1}|v_1|^{\delta+1}\mathrm{d}x$ determines the direction of bifurcation at the point $\lambda=\lambda_1$. \\ (2) This paper gives a complete bifurcation result for Problem \eqref{P} using the arguments developed in Allegretto and Huang \cite{All} and by Brown and Zhang \cite{Bro}. \section{Notation and hypotheses} Let $W_0^{1,p}(\Omega)$ denote the closure of the space $C_0 ^{\infty}(\Omega)$ with respect to the norm $\| u\|_p=(\int_{\Omega} |\nabla u|^p \mathrm{d}x)^{1/p}$. Let $X$ denote the product space $W_0^{1,p}(\Omega)\times W_0^{1,q}(\Omega)$ equipped with the norm $$ \|(u,v)\|_X=\|u\|_{p}+\|v\|_{q}. $$ Now, we state some assumptions used in this paper. \begin{itemize} \item[(H1)] Assume that $\alpha,\beta,\gamma,\delta$ satisfy \begin{gather*} \frac{\alpha+1}{p}+\frac{\beta+1}{q}=1 \,,\\ p<\gamma+1 \quad\mbox{or}\quad q<\delta+1 ,\quad \frac{\gamma+1}{p^*}+\frac{\delta+1}{q^*}<1\,, \\ \frac{1}{(\alpha+1)(\delta+1)}+\frac{1}{(\beta+1)(\gamma+1)}<1\,, \end{gather*} where $p^*=\frac{Np}{N-p}$, $q^*=\frac{Np}{N-p}$ are the well-known critical exponents. \item[(H2)] Assume $a(x),b(x),d(x)$ are nonnegative smooth functions such that $a(x)\in L^{\frac{N}{p}}(\Omega)\cap L^{\infty}(\Omega)$, $b(x)\in L^{\omega_1}(\Omega)\cap L^{\infty}(\Omega)$, $d(x)\in L^{\frac{N}{q}}(\Omega)\cap L^{\infty}(\Omega)$ and \begin{gather*} |\Omega_1^+|= |\{ x\in \Omega : a(x)>0 \} |>0\\ |\Omega_2^+|= |\{ x\in \Omega : d(x)>0 \} |>0\,, \end{gather*} where $b(x)\not\equiv 0$ and $\omega_1= p^*q^*/[p^*q^*-(\alpha+1)q^*-(\beta+1)p^*]$. \item[(H3)] $\mu(x)$ is a given smooth function which many change sign, and $\mu(x)\in L^{\omega_2}(\Omega)\cap L^\infty(\Omega)$, where $\omega_2= p^*q^*/[p^*q^*-(\gamma+1)q^*-(\delta+1)p^*]$. \end{itemize} \begin{lemma}[\cite{All, Wil}] There exists a number $\lambda_1>0$ such that \begin{enumerate} \item $$ \lambda_1=\inf \frac{(\frac{\alpha+1}{p}\int_\Omega |\nabla u|^p\mathrm{d}x+\frac{\beta+1}{q}\int_\Omega |\nabla v |^q\mathrm{d}x)} {({\frac{\alpha+1}{p}\int_\Omega a(x)|u|^p\mathrm{d}x+\frac{\beta+1}{q}\int_\Omega d(x)|v|^q\mathrm{d}x+\int_\Omega b(x)|u|^{\alpha+1}|v|^{\beta+1}\mathrm{d}x})}\,, $$ where the infimum is taken over $(u,v) \in X $ \item There exists a positive function $(u_1, v_1) \in X \cap L^{\infty}(\Omega)$, which is solution of the system \eqref{e1.2} \item The eigenvalue $\lambda_1$ is simple in the sense that the eigenfunctions associated with it are merely a constant multiple of each other \item $\lambda_1$ is isolated, that is, there exists $\delta>0$ such that in the interval $(\lambda _1, \lambda_1+\delta)$ there are no other eigenvalues of the system \eqref{e1.2}. \end{enumerate} \end{lemma} \begin{definition} \label{def2.2} We say that $(u,v)\in X $ is a weak solution of Problem \eqref{P} if for all $(\varphi,\xi)\in X$, \begin{align*} \int_\Omega|\nabla u|^{p-2}\nabla u\nabla\varphi\mathrm{d}x & = \lambda(\int_\Omega a(x)|u|^{p-2}u\varphi\mathrm{d}x+\int_\Omega b(x)|u|^{\alpha-1}|v|^{\beta+1}u\varphi\mathrm{d}x)\\ &\quad +\frac{1}{(\alpha+1)(\delta+1)}\int_\Omega \mu(x)|u|^{\gamma-1}|v|^{\delta+1}u\varphi\mathrm{d}x\, \\ \int_\Omega|\nabla v|^{q-2}\nabla v\nabla\xi\mathrm{d}x & = \lambda(\int_\Omega a(x)|v|^{p-2}v\xi\mathrm{d}x+\int_\Omega b(x)|u|^{\alpha+1}|v|^{\beta-1}v\xi\mathrm{d}x)\\ &\quad +\frac{1}{(\beta+1)(\gamma+1)}\int_\Omega \mu(x)|u|^{\gamma+1}|v|^{\delta-1}v\xi\mathrm{d}x\,. \end{align*} \end{definition} \section{The case $\lambda<\lambda_1$} It is well known that Problem \eqref{P} has a variational structure. i.e., weak solutions of Problem \eqref{P} are critical points of the functional $$ I(u,v)=J(u,v)-\lambda K(u,v)-\frac{1}{(\gamma+1)(\delta+1)}M(u,v) $$ where \begin{align*} J(u,v) & = \frac{\alpha+1}{p}\int_\Omega|\nabla u|^p\mathrm{d}x+\frac{\beta+1}{q}\int_\Omega|\nabla v|^q\mathrm{d}x,\\ K(u,v) & = \frac{\alpha+1}{p}\int_\Omega a(x)|u|^p\mathrm{d}x+\frac{\beta+1}{q}\int_\Omega d(x)|v|^q\mathrm{d}x+\int_\Omega b(x)|u|^{\alpha+1}|v|^{\beta+1}\mathrm{d}x,\\ M(u,v) & = \int_\Omega \mu(x)|u|^{\gamma+1}|v|^{\delta+1}\mathrm{d}x. \end{align*} Clearly, $I(u,v)\in C^1(X,R)$. Let $\Lambda_\lambda$ be the Nehari manifold associated with Problem \eqref{P}. i.e., \begin{equation} \Lambda_\lambda=\{(u,v)\in X:\langle I'(u,v),(u,v)\rangle =0\} \label{e3.1} \end{equation} It is clear that $\Lambda_\lambda$ is closed in $X$ and all critical points of $I(u,v)$ must lie on $\Lambda_\lambda$. So, $(u,v)\in\Lambda_\lambda$ if and only if \begin{equation} \begin{aligned} &\int_\Omega|\nabla u|^{p}\,\mathrm{d}x-\lambda\int_\Omega a(x)|u|^{p}\,\mathrm{d}x-\lambda\int_\Omega b(x)|u|^{\alpha+1}|v|^{\beta+1}\,\mathrm{d}x\\ &= \frac{1}{(\alpha+1)(\delta+1)}\int_\Omega \mu(x)|u|^{\gamma+1}|v|^{\delta+1}\,\mathrm{d}x\\ &\int_\Omega|\nabla v|^{q}\,\mathrm{d}x-\lambda\int_\Omega d(x)|v|^{q}\,\mathrm{d}x-\lambda\int_\Omega b(x)|u|^{\alpha+1}|v|^{\beta+1}\,\mathrm{d}x\\ &=\frac{1}{(\beta+1)(\gamma+1)}\int_\Omega \mu(x)|u|^{\gamma+1}|v|^{\delta+1}\,\mathrm{d}x \end{aligned}\label{e3.2} \end{equation} Hence, for $(u,v)\in\Lambda_\lambda$, using $\frac{\alpha+1}{p}+\frac{\beta+1}{q}=1$, we have \begin{equation} I(u,v)=(\frac{1}{p(\delta+1)}+\frac{1}{q(\gamma+1)} -\frac{1}{(\gamma+1)(\delta+1)})\int_\Omega \mu(x)|u|^{\gamma+1}|v|^{\delta+1}\,\mathrm{d}x \label{e3.3} \end{equation} Now, we define the following disjoint subsets of $\Lambda_\lambda$: \begin{align*} \Lambda_\lambda^+ &=\{(u,v)\in\Lambda_\lambda:\int_\Omega \mu(x)|u|^{\lambda+1}|v|^{\delta+1}\,\mathrm{d}x<0\}\\ \Lambda_\lambda^0 &=\{(u,v)\in\Lambda_\lambda:\int_\Omega \mu(x)|u|^{\lambda+1}|v|^{\delta+1}\,\mathrm{d}x=0\}\\ \Lambda_\lambda^- &=\{(u,v)\in\Lambda_\lambda:\int_\Omega \mu(x)|u|^{\lambda+1}|v|^{\delta+1}\,\mathrm{d}x>0\}\\ \end{align*} Let $0<\lambda<\lambda_1$, and consider the eigenvalue problem \begin{equation} \begin{gathered} -\Delta_pu-\lambda(a(x)|u|^{p}+b(x)|u|^{\alpha+1}|v|^{\beta+1}) = \mu_M|u|^{p-1}u \quad\Omega\\ -\Delta_qv-\lambda(d(x)|v|^{q}+b(x)|u|^{\alpha+1}|v|^{\beta+1}) = \mu_M|v|^{q-1}v \quad\Omega\,. \end{gathered}\label{e3.4} \end{equation} Then, there exists $\mu_M>0$ such that \begin{equation} \begin{gathered} \int_\Omega|\nabla u|^p\,\mathrm{d}x-\lambda\int_\Omega a(x)|u|^{p}\,\mathrm{d}x-\lambda\int_\Omega b(x)|u|^{\alpha+1}|v|^{\beta+1}\,\mathrm{d}x \geq \mu_M\int_\Omega|u|^{p}\,\mathrm{d}x\\ \int_\Omega|\nabla v|^q\,\mathrm{d}x-\lambda\int_\Omega d(x)|v|^{q}\,\mathrm{d}x-\lambda\int_\Omega b(x)|u|^{\alpha+1}|v|^{\beta+1}\,\mathrm{d}x \geq \mu_M\int_\Omega|v|^{q}\,\mathrm{d}x\\ \end{gathered}\label{e3.5} \end{equation} for every $(u,v)\in X$. Thus, $\Lambda_\lambda^+$ is empty, $\Lambda_\lambda^0=\{(0,0)\}$ and $\Lambda_\lambda=\Lambda_\lambda^-\cup\{(0,0)\}$. Clearly, $I(u,v)>0$ whenever $(u,v)\in \Lambda_\lambda^-$ and $I(u,v)$ is bounded below by on $\Lambda_\lambda^-$. i.e., $\inf_{(u,v)\in\Lambda_\lambda^-}I(u,v)\geq 0$. \begin{theorem} \label{thm3.1} Assume (H1)--(H3) and the condition \eqref{e1.3}. Then Problem \eqref{P} has a nonnegative nonsemitrivial solution for every $\lambda\in(0,\lambda_1)$. \end{theorem} \begin{proof} Let $\{(u_n,v_n)\}\subset\Lambda_\lambda^-$ be a minimizing sequence; i.e., $\lim_{n\to\infty}I(u_n,v_n)=\inf_{(u,v)\in\Lambda_\lambda^-}I(u,v)$. Since $$ I(u_n,v_n)=(\frac{1}{p(\delta+1)}+\frac{1}{q(\gamma+1)}-\frac{1}{(\gamma+1)(\delta+1)})\int_\Omega \mu(x)|u_n|^{\gamma+1}|v_n|^{\delta+1}\,\mathrm{d}x $$ using \eqref{e3.2} \eqref{e3.5} and $p<\gamma+1$ or $q<\delta+1$, we have $$ I(u_n,v_n)\geq\mu_M[(\frac{\alpha+1}{p}-\frac{\alpha+1}{\gamma+1}) \int_\Omega|u_n|^{p}\,\mathrm{d}x+(\frac{\beta+1}{q}-\frac{\beta+1}{\gamma+1})\int_\Omega|v_n|^{q}\,\mathrm{d}x]\,. $$ Then $\{(u_n,v_n)\}$ is bounded in $X$, and so we may assume $(u_n,v_n)\rightharpoonup(u_0,v_0) \in X$ and $u_n\to u_0$ in $L^{\gamma+1}(\Omega)$, $v_n\to v_0$ in $L^{\delta+1}(\Omega)$. First we claim that $\inf_{(u,v)\in\Lambda_\lambda^-}I(u,v)>0$. Indeed, suppose $\inf_{(u,v)\in\Lambda_\lambda^-}I(u,v)=0$. i.e. $\lim_{n\to\infty}I(u_n,v_n)=0$, we have $$ \int_\Omega\mu(x)|u_n|^{\gamma+1}|v_n|^{\delta+1}\,\mathrm{d}x\to 0 $$ and \begin{align} &\int_\Omega|\nabla u_n|^p-\lambda a(x)|u_n|^{p}-\lambda b(x)|u_n|^{\alpha+1}|v_n|^{\beta+1}\,\mathrm{d}x \nonumber\\ &=\frac{1}{(\alpha+1)(\delta+1)}\int_\Omega\mu(x)|u_n|^{\gamma+1}|v_n|^{\delta+1}\,\mathrm{d}x \to 0 \label{e3.6}\\ &\int_\Omega|\nabla v_n|^q-\lambda d(x)|v_n|^{q}-\lambda b(x)|u_n|^{\alpha+1}|v_n|^{\beta+1}\,\mathrm{d}x \nonumber\\ &=\frac{1}{(\beta+1)(\gamma+1)}\int_\Omega\mu(x)|u_n|^{\gamma+1}|v_n|^{\delta+1}\,\mathrm{d}x \to 0 \label{e3.7} \end{align} Moreover, by \cite[Lemma 2.1]{Dra} the compactness of the operators $K$ implies \begin{align*} &\int_\Omega|\nabla u_n|^p-\lambda a(x)|u_n|^{p}-\lambda b(x)|u_n|^{\alpha+1}|v_n|^{\beta+1}\,\mathrm{d}x\\ & \to \int_\Omega|\nabla u_0|^p-\lambda a(x)|u_0|^{p}-\lambda b(x)|u_0|^{\alpha+1}|v_0|^{\beta+1}\,\mathrm{d}x=0 \\ &\int_\Omega|\nabla v_n|^q-\lambda d(x)|v_n|^{q}-\lambda b(x)|u_n|^{\alpha+1}|v_n|^{\beta+1}\,\mathrm{d}x\\ & \to\int_\Omega|\nabla v_0|^q-\lambda d(x)|v_0|^{q}-\lambda b(x)|u_0|^{\alpha+1}|v_0|^{\beta+1}\,\mathrm{d}x=0 \end{align*} From $\lambda\in(0,\lambda_1)$ and the variational characterization of $\lambda_1$, we have $(u_n,v_n)\to(u_0,v_0)=(0,0)$. Let \begin{equation} \widetilde{u}_n=\frac{u_n}{(\|u_n\|_p^p +\|v_n\|_q^q)^{1/p}},\quad \widetilde{v}_n=\frac{v_n}{(\|u_n\|_p^p +\|v_n\|_q^q)^{\frac{1}{q}}} \label{e3.8} \end{equation} which are bounded sequences. Indeed, we have $$ \|\widetilde{u}_n\|_p^p+\|\widetilde{v}_n\|_q^q =1\quad \mbox{for every } n\in \mathbb{N} $$ Thus, we may assume $(\widetilde{u}_n,\widetilde{v}_n)\rightharpoonup(\widetilde{u}_0,\widetilde{v}_0)$. Using that $\frac{\alpha+1}{p}+\frac{\beta+1}{q}=1$, we have \[ \int_\Omega b(x)|\widetilde{u}_n|^{\alpha+1}|\widetilde{v}_n|^{\beta+1}\,\mathrm{d}x=\int_\Omega b(x)|u_n|^{\alpha+1}|v_n|^{\beta+1}\,\mathrm{d}x\Big/(\|u_n\|_p^p +\|v_n\|_q^q) \] Moreover the range of exponents implies \[ \frac{\int_\Omega\mu(x)|u_n|^{\gamma+1}|v_n|^{\delta+1}\,\mathrm{d}x}{\|u_n\|_p^p +\|v_n\|_q^q}\leq\frac{|\mu|_{\omega_2}|u_n|_{p^*}^{\gamma+1}|v_n|^{\delta+1}_{q^*}}{\|u_n\|_p^p +\|v_n\|_q^q} \to 0 \] Using \eqref{e3.6} and \eqref{e3.7}, we obtain \begin{gather*} \int_\Omega|\nabla\widetilde{u}_n|^{p}-\lambda a(x)|\widetilde{u}_n|^{p}-\lambda b(x)|\widetilde{u}_n|^{\alpha+1}|\widetilde{v}_n|^{\beta+1}\,\mathrm{d}x \to 0\,, \\ \int_\Omega|\nabla\widetilde{v}_n|^{q}-\lambda d(x)|\widetilde{v}_n|^{q}-\lambda b(x)|\widetilde{u}_n|^{\alpha+1}|\widetilde{v}_n|^{\beta+1}\,\mathrm{d}x \to 0\,. \end{gather*} Following the argument used on $\{(\widetilde{u}_n,\widetilde{v}_n)\}$ above, for $\{(u_n,v_n)\}$ we have \begin{align*} &\int_\Omega|\nabla\widetilde{u}_n|^{p}-\lambda a(x)|\widetilde{u}_n|^{p}-\lambda b(x)|\widetilde{u}_n|^{\alpha+1}|\widetilde{v}_n|^{\beta+1}\,\mathrm{d}x\\ &\to\int_\Omega|\nabla\widetilde{u}_0|^{p}-\lambda a(x)|\widetilde{u}_0|^{p}-\lambda b(x)|\widetilde{u}_0|^{\alpha+1}|\widetilde{v}_0|^{\beta+1}\,\mathrm{d}x=0\,, \\ &\int_\Omega|\nabla\widetilde{v}_n|^{q}-\lambda d(x)|\widetilde{v}_n|^{q}-\lambda b(x)|\widetilde{u}_n|^{\alpha+1}|\widetilde{v}_n|^{\beta+1}\,\mathrm{d}x\\ &\to\int_\Omega|\nabla\widetilde{v}_0|^{q}-\lambda d(x)|\widetilde{v}_0|^{q}-\lambda b(x)|\widetilde{u}_0|^{\alpha+1}|\widetilde{v}_0|^{\beta+1}\,\mathrm{d}x=0\,, \end{align*} and $(\widetilde{u}_n,\widetilde{v}_n)\to(\widetilde{u}_0,\widetilde{v}_0)=(0,0)$ in $X$, which contradict $\|(\widetilde{u}_n,\widetilde{v}_n)\|_X=1$, for every $n\in \mathbb{N}$. Now we show that $(u_n,v_n)\to(u_0,v_0)$ in $X$. Suppose otherwise, then $\|u_0|_p <\liminf_{n\to\infty}\|u_n\|_p$, $\|v_0\|_p<\liminf_{n\to\infty}\|v_n\|_q$, and \begin{align*} &\int_\Omega|\nabla u_0|^p-\lambda\int_\Omega a(x)|u_0|^{p}-\lambda\int_\Omega b(x)|u_0|^{\alpha+1}|v_0|^{\beta+1}\,\mathrm{d}x\\ & <\liminf_{n\to\infty}\int_\Omega|\nabla u_n|^p-\lambda a(x)|u_n|^{p}-\lambda b(x)|u_n|^{\alpha+1}|v_n|^{\beta+1}\,\mathrm{d}x=0\\ & \int_\Omega|\nabla v_0|^q-\lambda d(x)|v_0|^{q}-\lambda b(x)|u_0|^{\alpha+1}|v_0|^{\beta+1}\,\mathrm{d}x\\ & <\liminf_{n\to\infty}\int_\Omega|\nabla v_n|^q-\lambda d(x)|v_n|^{q}-\lambda b(x)|u_n|^{\alpha+1}|v_n|^{\beta+1}\,\mathrm{d}x=0\,. \end{align*} Since $\lambda\in(0,\lambda_1)$ and $(u_0,v_0)\not\equiv(0,0)$, we have \begin{gather*} \int_\Omega|\nabla u_0|^p-\lambda a(x)|u_0|^{p}-\lambda b(x)|u_0|^{\alpha+1}|v_0|^{\beta+1}\,\mathrm{d}x>0\,,\\ \int_\Omega|\nabla v_0|^q-\lambda d(x)|v_0|^{q}-\lambda b(x)|u_0|^{\alpha+1}|v_0|^{\beta+1}\,\mathrm{d}x>0 \end{gather*} which is a contradiction. Hence $(u_n,v_n)\to(u_0,v_0)$ in $X$. \end{proof} From \cite[Theorem 2.3]{Bro}, $(u_0,v_0)$ is a local minimizer on $\Lambda_\lambda^-$ and $(u_0,v_0)\not\in\Lambda_\lambda^0=\{(0,0)\}$, then $(u_0,v_0)$ is a critical point of $I(u,v)$. This solution is nonnegative due to the fact that $I(|u|,|v|)=I(u,v)$, and it is also nonsemitrivial by \cite[Lemma 2.5]{Dra}. \begin{theorem} \label{thm3.2} Assume (H1)--(H3) and the condition \eqref{e1.3}, if $\lambda_n\to\lambda_1^-$ and $(u_n,v_n)$ is a minimizer of $I(u_n,v_n)$ on $\Lambda_\lambda^-$, then $(u_n,v_n)\to(0,0)$. \end{theorem} \begin{proof} First we show that $\{(u_n,v_n)\}$ is bounded in $X$. Suppose not, then we may assume without loss of generality that $\|u_n\|_p\to\infty$, $\|v_n\|_q\to\infty$, as $n\to\infty$. Let $(\widetilde{u}_n,\widetilde{v}_n)$ are the sequence introduced by (3.8). The boundedness of $(\widetilde{u}_n,\widetilde{v}_n)$ implies $(\widetilde{u}_n,\widetilde{v}_n)\rightharpoonup (\widetilde{u}_0,\widetilde{v}_0)$ in $X$. Then \begin{align*} &\int_\Omega|\nabla\widetilde{u}_n|^{p}-\lambda_n a(x)|\widetilde{u}_n|^{p}-\lambda_n b(x)|\widetilde{u}_n|^{\alpha+1}|\widetilde{v}_n|^{\beta+1}\,\mathrm{d}x\\ &\to\int_\Omega|\nabla\widetilde{u}_0|^{p}-\lambda_1 a(x)|\widetilde{u}_0|^{p}-\lambda_1 b(x)|\widetilde{u}_0|^{\alpha+1}|\widetilde{v}_0|^{\beta+1}\,\mathrm{d}x=0\\ &\int_\Omega|\nabla\widetilde{v}_n|^{q}-\lambda_n d(x)|\widetilde{v}_n|^{q}-\lambda_n b(x)|\widetilde{u}_n|^{\alpha+1}|\widetilde{v}_n|^{\beta+1}\,\mathrm{d}x\\ &\to\int_\Omega|\nabla\widetilde{v}_0|^{q}-\lambda_1 d(x)|\widetilde{v}_0|^{q}-\lambda_1 b(x)|\widetilde{u}_0|^{\alpha+1}|\widetilde{v}_0|^{\beta+1}\,\mathrm{d}x=0\,. \end{align*} Since $(u_n,v_n)$ is a minimizer of $I(u_n,v_n)$ on $\Lambda_\lambda^-$, we have \[ I(u_n,v_n)=(\frac{1}{p(\delta+1)}+\frac{1}{q(\gamma+1)} -\frac{1}{(\gamma+1)(\delta+1)})\int_\Omega\mu(x)|u_n|^{\gamma+1}|v_n|^{\delta+1} \,\mathrm{d}x\to 0 \] Thus, we must have $(\widetilde{u}_n,\widetilde{v}_n)\to(\widetilde{u}_0,\widetilde{v}_0)\not\equiv(0,0)$ and $\widetilde{u}_0=k^p u_1, \widetilde{v}=k^q v_1$ for some positive constant $k$, it is easy to see \[ \lim_{n\to\infty}(\|u_n\|_p^p +\|v_n\|_q^q){(\frac{\gamma+1}{p} +\frac{\delta+1}{q}-1)} \int_\Omega\mu(x)|\widetilde{u}_n|^{\gamma+1}|\widetilde{v}_n|^{\delta+1}\,\mathrm{d}x =0\,. \] Hence $\lim_{n\to\infty}\int_\Omega\mu(x)|\widetilde{u}_n|^{\gamma+1}|\widetilde{v}_n|^{\delta+1}\,\mathrm{d}x =\int_\Omega\mu(x)|\widetilde{u}_0|^{\gamma+1}|\widetilde{v}_0|^{\delta+1}\,\mathrm{d}x$, it follows that $k=0$. But as $\|\widetilde{u}_0\|_p^p +\|\widetilde{v}_0\|_q^q =1$, that is impossible. Hence $\{(u_n,v_n)\}$ is bounded. Thus we may assume $(u_n,v_n)\rightharpoonup(u_0,v_0)$ in $X$. Then, using the same argument on $(u_n,v_n)$ as used on $(\widetilde{u}_n,\widetilde{v}_v)$. It follows that $(u_n,v_n)\to (0,0)$, and so the proof is complete. \end{proof} We remark that the two theorems above give a rather detailed description of the bifurcation diagram in Figure 1(a). \section{The case $\lambda>\lambda_1$} In this section, we prove a nonexistence result for Problem \eqref{P} by using the Picone identity. \begin{lemma}[Picone identity \cite{All}] Let $v>0$, $u\geq 0$ be differentiable, and let \begin{gather*} L(u,v)=|\nabla u|^p+(p-1)\frac{u^p}{v^p}|\nabla v|^p-p\frac{u^p}{v^{p-1}}\nabla u\nabla v|\nabla v|^{p-2}\\ R(u,v)=|\nabla u|^p-\nabla(\frac{u^p}{v^{p-1}})|\nabla v|^{p-2}\nabla v \end{gather*} Then $L(u,v)=R(u,v)\geq 0$. \end{lemma} Moreover, $L(u,v)=0$ a.e. on $\Omega$ if and only if $\nabla(\frac{u}{v})=0$ a.e. on $\Omega$. For the next theorem we will assume \begin{itemize} \item[(H3')] $u(x)$ is a nonnegative smooth function, and $\mu(x)\in L^{\omega_2}(\Omega)\cap L^\infty(\Omega)$, where $\omega_2=p^*q^*/[p^*q^*-(\gamma+1)q^*-(\delta+1)p^*]$ \end{itemize} \begin{theorem} \label{thm4.2} Assume (H1), (H2), (H3') and Condition \eqref{e1.3}. Then Problem \eqref{P} has no nonnegative nonsemitrivial solution, for every $\lambda>\lambda_1$. \end{theorem} \begin{proof} On the contrary, let $u_n\in C_0^\infty(\Omega), v_n\in C_0^\infty(\Omega)$. We apply Picone's identity to the functions $u_n, u$ and $v_n, v$, to obtain \begin{gather} 0\leq\int_\Omega|\nabla u_n|^p\,\mathrm{d}x+\int_\Omega\frac{u_n^p}{u^{p-1}}\Delta_pu\,\mathrm{d}x \label{e4.1}\\ 0\leq\int_\Omega|\nabla v_n|^q\,\mathrm{d}x+\int_\Omega\frac{v_n^q}{v^{q-1}}\Delta_qv\,\mathrm{d}x \label{e4.2} \end{gather} Using that $\frac{\alpha+1}{p}+\frac{\beta+1}{q}=1$, then multiplying \eqref{e4.1} by $\frac{\alpha+1}{p}$ and \eqref{e4.2} by $\frac{\beta+1}{q}$, and then adding, we obtain \begin{equation} \begin{aligned} &\frac{\alpha+1}{p}\int_\Omega|\nabla u_n|^p\,\mathrm{d}x+\frac{\beta+1}{q}\int_\Omega|\nabla v_n|^q\,\mathrm{d}x\\ &-\frac{\alpha+1}{p}\int_\Omega\lambda a(x)u_n^p\,\mathrm{d}x-\frac{\beta+1}{q}\int_\Omega\lambda d(x)v_n^q\,\mathrm{d}x\\ &\geq\frac{\alpha+1}{p}\int_\Omega\lambda b(x)u_n^p u^{\alpha+1-p}v^{\beta+1}\,\mathrm{d}x+\frac{\beta+1}{q}\int_\Omega\lambda b(x)v_n^q|u|^{\alpha+1}v^{\beta+1-q}\,\mathrm{d}x\\ &\quad +\frac{1}{p(\delta+1)}\int_\Omega\mu(x)u_n^pu^{\gamma+1-p} v^{\delta+1}\,\mathrm{d}x+\frac{1}{q(\gamma+1)}\int_\Omega\mu(x)v_n^qu^{\gamma+1}v^{\delta+1-q}\,\mathrm{d}x \end{aligned}\label{e4.3} \end{equation} Now, put $\theta_1=(\alpha+1)(\beta+1)/q$ and $\theta_2=(\alpha+1)(\beta+1)/p$, then \[ u_n^{\alpha+1}v_n^{\beta+1} = u_n^{\alpha+1}v_n^{\beta+1}\frac{v^{\theta_2}}{u^{\theta_1}} \frac{u^{\theta_1}}{v^{\theta_2}} \leq \frac{\alpha+1}{p}u_n^p u^{\alpha+1-p}v^{\beta+1}+\frac{\beta+1}{q}v_n^q u^{\alpha+1}v^{\beta+1-q} \] Since $\lambda>0$ and $b(x)\geq0$, we obtain \begin{equation} \begin{aligned} &\lambda\int_\Omega b(x)u_n^{\alpha+1}v_n^{\beta+1}\,\mathrm{d}x\\ &\leq\frac{\alpha+1}{p}\int_\Omega\lambda b(x)u_n^pu^{\alpha+1-p}v^{\beta+1}\,\mathrm{d}x+\frac{\beta+1}{q}\int_\Omega\lambda b(x)v_n^qu^{\alpha+1}v^{\beta+1-q}\,\mathrm{d}x \end{aligned} \label{e4.4} \end{equation} Using that $\frac{\alpha+1}{p}+\frac{\beta+1}{q}=1$ and $\frac{1}{(\alpha+1 )(\delta+1)}+\frac{1}{(\beta+1)(\gamma+1)}<1$, we obtain $$ \frac{\gamma+1}{p}+\frac{\delta+1}{q}>(\alpha+1 )(\beta+1)>1\,. $$ Then \[ u_n^{\gamma+1}v_n^{\delta+1}<\frac{\gamma+1}{p}u_n^{p}u^{\gamma+1-p}v^{\delta+1}+\frac{\delta+1}{q}v_n^q u^{\gamma+1}v^{\delta+1-q}\,. \] Since $\mu(x)\geq0$, we have \begin{equation} \begin{aligned} &\frac{1}{p(\delta+1)}\int_\Omega\mu(x)u_n^pu^{\gamma+1-p}v^{\delta+1}\,\mathrm{d}x +\frac{1}{\gamma+1}\int_\Omega\mu(x)v_n^qu^{\gamma+1}v^{\delta+1-q}\,\mathrm{d}x\\ &= \frac{1}{(\gamma+1)(\delta+1)} \Big[\frac{\gamma+1}{p}\int_\Omega\mu(x)u_n^pu^{\gamma+1-p}v^{\delta+1} \,\mathrm{d}x\\ &\quad +\frac{\delta+1}{q}\int_\Omega \mu(x) v_n^qu^{\gamma+1}v^{\delta+1-q} \,\mathrm{d}x\Big]\\ &\geq \frac{1}{(\gamma+1)(\delta+1)}\int_\Omega\mu(x)u_n^{\gamma+1}v_n^{\delta+1}\,\mathrm{d}x \end{aligned}\label{e4.5} \end{equation} Combining \eqref{e4.3}, \eqref{e4.4} and \eqref{e4.5}, we have \begin{equation} \begin{aligned} &\frac{\alpha+1 }{p}\int_\Omega|\nabla u_n|^p\,\mathrm{d}x+\frac{\beta+1}{q}\int_\Omega|\nabla v_n|^q\,\mathrm{d}x -\frac{\alpha+1 }{p}\int_\Omega\lambda a(x)u_n^p\,\mathrm{d}x\\ &-\frac{\beta+1}{q}\int_\Omega\lambda d(x)v_n^q\,\mathrm{d}x -\lambda\int_\Omega b(x)u_n^{\alpha+1 }v_n^{\beta+1}\,\mathrm{d}x\\ &>\frac{1}{(\gamma+1)(\delta+1)}\int_\Omega\mu(x)u_n^{\gamma+1}v_n^{\delta+1}\,\mathrm{d}x \end{aligned}\label{e4.6} \end{equation} Let $(u_n,v_n)$ converge to $(u_1,v_1)\in X$, then \begin{align*} &\frac{\alpha+1 }{p}\int_\Omega|\nabla u_n|^p\,\mathrm{d}x+\frac{\beta+1}{q}\int_\Omega|\nabla v_n|^q\,\mathrm{d}x -\frac{\alpha+1 }{p}\int_\Omega\lambda a(x)u_n^p\,\mathrm{d}x\\ &-\frac{\beta+1}{q}\int_\Omega\lambda d(x)v_n^q\,\mathrm{d}x -\lambda\int_\Omega b(x)u_n^{\alpha+1 }v_n^{\beta+1}\,\mathrm{d}x\\ &\to\frac{\alpha+1 }{p}\int_\Omega|\nabla u_1|^p\,\mathrm{d}x+\frac{\beta+1}{q} \int_\Omega|\nabla v_1|^q\,\mathrm{d}x -\frac{\alpha+1 }{p}\int_\Omega\lambda a(x)u_1^p\,\mathrm{d}x\\ &\quad -\frac{\beta+1}{q}\int_\Omega\lambda d(x)v_1^q\,\mathrm{d}x -\lambda\int_\Omega b(x)u_1^{\alpha+1 }v_1^{\beta+1}\,\mathrm{d}x\,,\\ &\frac{1}{(\gamma+1)(\delta+1)}\int_\Omega\mu(x) u_n^{\gamma+1}v_n^{\delta+1}\,\mathrm{d}x\to\frac{1}{(\gamma+1)(\delta+1)}\int_\Omega\mu(x) u_1^{\gamma+1}v_1^{\delta+1}\,\mathrm{d}x \end{align*} From the variational characterization of $\lambda_1$ and $\lambda>\lambda_1$, we have \begin{align*} &\frac{\alpha+1 }{p}\int_\Omega|\nabla u_1|^p\,\mathrm{d}x+\frac{\beta+1}{q} \int_\Omega|\nabla v_1|^q\,\mathrm{d}x -\frac{\alpha+1 }{p}\int_\Omega\lambda a(x)u_1^p\,\mathrm{d}x\\ &-\frac{\beta+1}{q}\int_\Omega\lambda d(x)v_1^q\,\mathrm{d}x -\lambda\int_\Omega b(x)u_1^{\alpha+1 }v_1^{\beta+1}\,\mathrm{d}x<0\,. \end{align*} Since $\int_\Omega\mu(x)u_1^{\gamma+1}v_1^{\delta+1}\,\mathrm{d}x>0$, we have $\frac{1}{(\gamma+1)(\delta+1)}\int_\Omega\mu(x)u_1^{\gamma+1}v_1^{\delta+1}\,\mathrm{d}x>0$, which is a contradiction that completes the proof. \end{proof} \begin{thebibliography}{99} \bibitem{All} W. Allegretto and Y. Huang; {\it A Picone's identity for the p-Laplacian and applications.} Nonlinear Analysis TMA, 32(1998), 819-830. \bibitem{Ber} H. Berestycki, I. Capuzzo Dolcetta and L. Nirenberg; {\it Variational methods for indefinite superlinear homogeneous elliptic problems}. Nonlinear Differential Equations Appl., 2(1995), 553-572. \bibitem{Bin} P. A. Binding, P. Drabek and Y. Huang; {\it On Neumann boundary value problems for some quasilinear elliptic equations.} Electron. J. Differential Equations, 1997(1997), No. 5, 1-11. \bibitem{Bro} K. J. Brown and Y. Zhang; {\it The Nehari manifold for a semilinear elliptic equation with a sign-changing weight function.} J. Differential Equations, 193(2003), 481-499. \bibitem{Dra} P. Drabek, N. M. Stavrakakis and N. B. Zographopoulos; {\it Multiple nonsemitrivial solutions for quasilinear elliptic systems.} Differential and Integral Equations, 16(2003), 1519-1531. \bibitem{Fle} J. Fleckinger, R. Mansevich, N. M. Stravrakakis and F. de Thelin; {\it Principal eigenvalues for some quasilinear elliptic equations on $R^N$.} Adv. Diff. Equ., 2(1997), 981-1003. \bibitem{Neh} Z. Nehari; {\it On a class of nonlinear second order differential equations.} Trans. Amer. Math. Soc., 95(1960), 101-123. \bibitem{Sta} N. M. Stavrakakis and N.B. Zographopoulos; {\it Existence results for some quasilinear elliptic systems in $R^N$.} Electron. J. Differential Equations, 1999 (1999), No. 39, 1-15. \bibitem{Sta2} N. M. Stavrakakis and N. B. Zographopoulos; {\it Global bifurcation results for some quasilinear elliptic systems on $R^N$}, Adv. Diff. Equ., 8(2003), 315-336. \bibitem{Wil} M. Willem, {\it Minimax Theorems.} Birkhauser. Boston, 1996. \bibitem{Zog} N. B. Zographopoulos, {\it On the isolation of the principal eigenvalue for a p-laplacian systems.} Appl. Math. Lett., to appear. \end{thebibliography} \end{document}