\documentclass[reqno]{amsart} \usepackage{hyperref} \AtBeginDocument{{\noindent\small \emph{Electronic Journal of Differential Equations}, Vol. 2008(2008), No. 81, pp. 1--11.\newline ISSN: 1072-6691. URL: http://ejde.math.txstate.edu or http://ejde.math.unt.edu \newline ftp ejde.math.txstate.edu (login: ftp)} \thanks{\copyright 2008 Texas State University - San Marcos.} \vspace{9mm}} \begin{document} \title[\hfilneg EJDE-2008/??\hfil Existence results] {Existence results for strongly indefinite elliptic systems} \author[J. Yang, Y. Ye, X. Yu\hfil EJDE-2008/??\hfilneg] {Jianfu Yang, Ying Ye, Xiaohui Yu} % in alphabetical order \address{Jianfu Yang \newline Department of Mathematics, Jiangxi Normal University, Nanchang, Jiangxi 330022, China} \email{jfyang\_2000@yahoo.com} \address{Ying Ye \newline Department of Mathematics, Jiangxi Normal University, Nanchang, Jiangxi 330022, China} \email{yeying19851985@163.com} \address{Xiaohui Yu \newline China Institute for Advanced Study, Central University of Finance and Economics, Beijing 100081, China} \email{yuxiao\_211@163.com} \thanks{Submitted April l7, 2008. Published May 28, 2008.} \thanks{Supported by grants 10571175 and 10631030 from the National Natural Sciences \hfill\break\indent Foundation of China} \subjclass[2000]{35J20,3 5J25} \keywords{Strongly indefinite elliptic system; existence} \begin{abstract} In this paper, we show the existence of solutions for the strongly indefinite elliptic system \begin{gather*} -\Delta u=\lambda u+f(x,v) \quad\text{in }\Omega, \\ -\Delta v=\lambda v+g(x,u) \quad\text{in }\Omega, \\ u=v=0, \quad\text{on }\partial\Omega, \end{gather*} where $\Omega$ is a bounded domain in $\mathbb{R}^N$ ($N\geq 3$) with smooth boundary, $\lambda_{k_0}<\lambda<\lambda_{k_0+1}$, where $\lambda_k$ is the $k$th eigenvalue of $-\Delta$ in $\Omega$ with zero Dirichlet boundary condition. Both cases when $f,g$ being superlinear and asymptotically linear at infinity are considered. \end{abstract} \maketitle \numberwithin{equation}{section} \newtheorem{theorem}{Theorem}[section] \newtheorem{lemma}[theorem]{Lemma} \section{Introduction} In this paper, we investigate the existence of solutions for the strongly indefinite elliptic system \begin{equation}\label{1.1} \begin{gathered} -\Delta u=\lambda u+f(x,v) \quad\text{in }\Omega, \\ -\Delta v=\lambda v+g(x,u) \quad\text{in }\Omega, \\ u=v=0, \quad\text{on }\partial\Omega,\\ \end{gathered} \end{equation} where $\Omega$ is a smooth bounded domain in $\mathbb{R}^N$, $N\geq 3$, $\lambda_{k_0}<\lambda<\lambda_{k_0+1}$, where $\lambda_k$ is the $k$th eigenvalue of $-\Delta$ in $\Omega$ with zero Dirichlet boundary condition. Problem \eqref{1.1} with $\lambda = 0$ was considered in \cite{FF,hv}, where the existence results for superlinear nonlinearities were established by finding critical points of the functional \begin{equation}\label{1.2} J(u,v)=\int_{\Omega} \nabla u \nabla v\,dx -\int_{\Omega} F(x,v)\,dx-\int_{\Omega} G(x,u)\,dx. \end{equation} A typical feature of the functional $J$ is that the quadratic part $$ Q(u,v)=\int_{\Omega} \nabla u \nabla v\,dx $$ is positive definite in an infinite dimensional subspace $E^+=\{(u,u): u\in H_0^1(\Omega)\}$ of $H_0^1(\Omega)\times H_0^1(\Omega)$ and negative definite in its infinite dimensional complimentary subspace $E^-=\{(u,-u): u\in H_0^1(\Omega)\}$, that is, $J$ is strongly indefinite. A linking theorem is then used in finding critical points of $J$. In the case that $\lambda$ lies in between higher eigenvalues, the parameter $\lambda$ affects the definiteness of the corresponding quadratic part $$ Q_\lambda(u,v)=\int_{\Omega} (\nabla u \nabla v-\lambda uv)\,dx $$ of the associated functional \begin{equation}\label{1.3} J_\lambda(u,v)=\int_{\Omega} (\nabla u \nabla v-\lambda uv)\,dx -\int_{\Omega} F(x,v)\,dx-\int_{\Omega} G(x,u)\,dx, \end{equation} of \eqref{1.1} defined on $H_0^1(\Omega)\times H_0^1(\Omega)$. A key ingredient in use of the linking theorem is to find a proper decomposition of $H_0^1(\Omega)\times H_0^1(\Omega)$ into a direct sum of two subspaces so that $Q_\lambda$ is definite in each subspace. Obviously, $Q_\lambda$ is neither positive definite in $E^+$ nor negative definite in $E^-$. So we need to find out a suitable decomposition of $H_0^1(\Omega)\times H_0^1(\Omega)$. We first consider the asymptotically linear case. Such a problem has been extensively studied for one equation, see for instance, \cite{lz,sz,z} and references therein. For asymptotically linear elliptic system, we refer readers to \cite{LY}. Particularly, in this case, the Ambrosetti-Rabinowtz condition is not satisfied, whence it is hard to show a Palais-Smale sequence is bounded. So one turns to using Cerami condition in critical point theory instead of the Palais-Smale condition, various existence results for asymptotically linear problems are then obtained. By a functional $I$ defined on $E$ satisfies Cerami condition we mean that for any sequence $\{u_n\}\subset E$ such that $|I(u_n)|\leq C$ and $(1+\|u_n\|)I'(u_n)\to 0$, there is a convergent subsequence of $\{u_n\}$. For the asymptotically linear system \eqref{1.1}, it is strongly indefinite and the nonlinearities do not fulfill the Ambrosetti-Rabinowitz condition. To handle the problem, we assume: \begin{itemize} \item[(A1)] $f,g\in C(\Omega\times \mathbb{R},\mathbb{R})$, $f(x,v)=o(|v|),g(x,u)=o(|u|)$ uniformly for $x\in \Omega$ as $|u|,|v|\to 0$ and $tf(x,t)\geq 0,tg(x,t)\geq 0$. \item[(A2)] There exist positive constants $l,m$, such that $\lim_{t\to \pm\infty}\frac {f(x,t)}{t}=l$ and $\lim_{t\to \pm\infty}\frac {g(x,t)}{t}=m$. \item[(A3)] $\lambda\pm\sqrt{ml}\neq \lambda_k$ for any $k\in \mathbb{N}$. \item[(A4)] There exists $u_0\in \mathop{\rm span}\{\varphi_{k_0+1},\varphi_{k_0+2}, \dots \}$ with $\int_\Omega|\nabla u_0|^2-\lambda(u_0)^2\,dx=\frac 12$ such that $$ \int_{\Omega} (|\nabla u_0|^2-\lambda u_0^2)\,dx-\min(l,m)\int_{\Omega} u_0^2\,dx<0. $$ \end{itemize} \begin{theorem}\label{thm1.1} Suppose {\rm (A1)-(A4)}, problem \eqref{1.1} has at least a nontrivial solution. \end{theorem} Condition (A4) holds, for example, if $\min(l,m)>\lambda_{k_0+1}-\lambda$, we choose $u_0=\alpha \varphi_{k+1}$ for some $\alpha>0$, then $\int_{\Omega} |\nabla u_0|^2-\lambda u_0^2\,dx-\min(l,m)\int_{\Omega} u_0^2\,dx=(\lambda_{k_0+1}-\lambda -\min(l,m))\int_{\Omega} u_0^2\,dx<0$. Theorem \ref{thm1.1} is proved by the following linking theorem with Cerami condition in \cite{LS}, which is a generalization of usual one in \cite{br}, \cite{R}. \begin{lemma}\label{lem1.2} Let $E$ be a real Hilbert space with $E=E_1\oplus E_2$. Suppose $I\in C^1(E,\mathbb{R})$, satisfies Cerami condition, and \begin{itemize} \item[(I1)] $I(u)=\frac 12(Lu,u)+b(u)$, where $Lu=L_1P_1u+L_2P_2u$ and $L_i:E_i\to E_i$ is bounded and selfadjoint, i=1,2. \item[(I2)] $b'$ is compact. \item[(I3)] There exists a subspace $\tilde{E}\subset E$ and sets $S\subset E,Q\subset \tilde E$ and constants $\alpha>\omega$ such that\\ {\rm (i)} $S\subset E_1$ and $I|_S\geq \alpha$, \\ {\rm (ii)} $Q$ is bounded and $I|_{\partial Q}\leq \omega$, \\ {\rm (iii)} $S $ and $Q$ link. \\ Then $I$ possesses a critical value $c\geq \alpha $. \end{itemize} \end{lemma} Next, we consider superlinear case. We assume that \begin{itemize} \item[(B1)] $f,g\in C(\Omega\times \mathbb{R},\mathbb{R})$, $f(x,v)=o(|v|),g(x,u)=o(|u|)$ uniformly for $x\in \Omega$ as $|u|,|v|\to 0$. \item[(B2)] There exists a constant $\gamma>2$ such that $$ 0<\gamma F(x,v)\leq vf(x,v), \quad 0<\gamma G(x,u)\leq ug(x,u), $$ where $F(x,v)=\int_0^v f(x,s)\,ds$ and $G(x,u)=\int_0^u g(x,u)\,ds$. \item[(B3)] There exist $p,q>1,\frac 1{p+1}+\frac 1{q+1}>\frac {N-2} N$, constants $a_1, a_2>0$, such that $|f(x,v)|\leq a_1+a_2|v|^q$, $|g(x,u)|\leq a_1+a_2|u|^p$. \end{itemize} \begin{theorem}\label{thm1.3} Assume {\rm (B1)-(B3)}, then \eqref{1.1} has at least one solution. \end{theorem} We remark that in \cite{hv}, it also considered the subcritical superlinear problem \begin{equation}\label{1.4} \begin{gathered} -\Delta u=\lambda v+f(v) \quad\text{in }\Omega, \\ -\Delta v=\mu u+g(u) \quad\text{in }\Omega, \\ u=v=0, \quad\text{on }\partial\Omega.\\ \end{gathered} \end{equation} The functional corresponding to \eqref{1.4} is no longer positive definite in $E^+$, but it is negative definite in $E^-$. It is different from our case. In section 2, we prove Theorem \ref{thm1.1}. While Theorem \ref{thm1.3} is showed in section 3. \section{Asymptotically linear case} Let $H:=H_0^1(\Omega)$, it can be decomposed as $H= H^1\oplus H^2$, where $H^1=\mathop{\rm span}\{\varphi_{k_0+1},\varphi_{k_0+2}\dots \}$, $H^2=\mathop{\rm span}\{\varphi_{1},\varphi_{2}\dots \varphi_{k_0}\}$ and $\varphi_{k}$ is the eigenfunction related to $\lambda_k$. Let $P_i$ be the projection of $H$ on the subspace $H^i, i=1,2$, then we define for $u\in H$ a new norm by $$ \|u\|^2=\int_{\Omega} |\nabla (P_1u)|^2-\lambda ( P_1 u)^2\,dx-\int_{\Omega} |\nabla (P_2u)|^2-\lambda ( P_2 u)^2\,dx, $$ it is equivalent to the usual norm of $H_0^1(\Omega)$. To find out the subspaces of $H\times H$ such that the quadratic part \[ Q_\lambda(u,v)= \int_{\Omega}(\nabla u\nabla v-\lambda uv)\,dx \] of the functional \[ J_\lambda(u,v)=\int_{\Omega}(\nabla u\nabla v-\lambda uv)\,dx -\int_{\Omega} F(x,v)\,dx-\int_{\Omega} G(x,u)\,dx \] is positive or negative definite on it, we denote \begin{gather*} E_{11}=\{(u,u):u\in H^1\}, \quad E_{12}=\{(u,-u):u\in H^1\}, \\ E_{21}=\{(u,u):u\in H^2\},\quad E_{22}=\{(u,-u):u\in H^2\}. \end{gather*} Therefore, $H\times H = E_{11}\oplus E_{12}\oplus E_{21}\oplus E_{22}$. We may write for any $(u,v)\in H\times H$ that \begin{equation}\label{2.1} (u,v)=(u_{11},u_{11})+(u_{12},-u_{12})+(u_{21},u_{21})+(u_{22},-u_{22}), \end{equation} where \begin{gather*} u_{11}=P_1(\frac {u+v} 2)\in H^1, \quad u_{21}=P_2(\frac {u+v} 2)\in H^2,\\ u_{12}=P_1(\frac {u-v} 2)\in H^1,\quad u_{22}=P_2(\frac {u-v} 2)\in H^2. \end{gather*} It is easy to check that $Q_\lambda$ is positive definite in $E_{11}\oplus E_{22}$ and negative definite in $E_{12}\oplus E_{21}$, so we denote $E_+=E_{11}\oplus E_{22}$ and $E_-=E_{12}\oplus E_{21}$ for convenience. Then \begin{equation}\label{2.2} J_\lambda(u,v)=\|u_{11}\|^2+\|u_{22}\|^2-\|u_{12}\|^2-\|u_{21}\|^2-\int_{\Omega} F(x,v)\,dx-\int_{\Omega} G(x,u)\,dx, \end{equation} it is $C^1$ on $H\times H$. \begin{lemma} \label{lem2.1} The functional $J_\lambda$ satisfies the Cerami condition. \end{lemma} \begin{proof} It is sufficient to show that any Cerami sequence is bounded, a standard argument then implies that the sequence has a convergent subsequence. We argue indirectly. Suppose it were not true, there would exist a Cerami sequence $z_n=\{(u_n,v_n)\}\subset H\times H$ of $J_\lambda$ such that $\|z_n\|\to \infty$. Let $$ w_n=\frac{z_n}{\|z_n\|}=(\frac{u_n}{\|z_n\|},\frac{v_n}{\|z_n\|}) =(w_n^1,w_n^2), $$ we may assume that \begin{gather*} (w_n^1, w_n^2)\rightharpoonup (w^1, w^2)\quad\text{in } H\times H, \quad (w_n^1, w_n^2)\to (w^1, w^2)\quad\text{in } L^2(\Omega)\times L^2(\Omega),\\ w_n^1\to w^1,w_n^2\to w^2\quad\text{a.e. in } \Omega. \end{gather*} We write as the decomposition \eqref{2.1} that $u_n =\sum_{i,j=1}^2 u_{ij}^n$ and correspondingly, $w_n^1 =\sum_{i,j=1}^2 w_{ij}^n$. We claim that $(w^1,w^2)\neq (0,0)$. Otherwise, there would hold \begin{equation}\label{2.3} |\langle J_\lambda'(u_n,v_n),(u_{11}^n,u_{11}^n)\rangle|\leq \|J_\lambda'(u_n,v_n)\|\cdot\|(u_{11}^n,u_{11}^n)\|\leq \|J_\lambda'(u_n,v_n)\|\cdot \|(u_n,v_n)\|\to 0; \end{equation} that is, \begin{equation}\label{2.4} \|u_{11}^n\|^2-\int_{\Omega} f(x,v_n)u_{11}^n\,dx-\int_{\Omega} g(x,u_n)u_{11}^n\,dx \to 0 \end{equation} implying \begin{equation}\label{2.5} \|w_{11}^n\|^2-\int_{\Omega} \frac {f(x,v_n)}{v_n}\frac {v_n}{\|z_n\|}\frac {u_{11}^n}{\|z_n\|}\,dx-\int_{\Omega}\frac {g(x,u_n)}{u_n}\frac {u_n}{\|z_n\|}\frac {u_{11}^n}{\|z_n\|}\,dx\to 0. \end{equation} Therefore, \begin{equation}\label{2.6} \|w_{11}^n\|^2\leq C\int_{\Omega}[ (w^1_n)^2+(w^2_n)^2]\,dx+o(1), \end{equation} which yields $\|w_{11}^n\|\to 0$. Similarly, $\|w_{12}^n\|\to 0$, $\|w_{21}^n\|\to 0$ and $\|w_{22}^n\|\to 0$ as $n\to\infty$. Consequently, $w_n\to 0$. This contradicts to $\|w_n\|=1$. Hence, there are three possibilities: (i) $w^1\neq 0,w^2\neq 0$; (ii) $w^1\neq 0,w^2=0$; (iii) $w^1= 0,w^2\neq 0$. We show next that all these cases will lead to a contradiction. Hence, $\|z_n\|$ is bounded. In case (i), we claim that $(w^1,w^2)$ satisfies \begin{equation}\label{2.7} \begin{gathered} -\Delta w^1=\lambda w^1+lw^2, \quad\text{in }\Omega, \\ -\Delta w^2=\lambda w^2+mw^1, \quad\text{in }\Omega, \\ w^1=w^2=0, \quad\text{on }\partial\Omega. \end{gathered} \end{equation} Indeed, let \begin{equation}\label{2.8} p_n(x)= \begin{cases} \frac{f(x,v_n(x))}{v_n(x)} & \text{if } v_n(x)\neq 0, \\ 0 &\text{if } v_n(x)= 0, \\ \end{cases} \end{equation} and \begin{equation}\label{2.9} q_n(x)= \begin{cases} \frac{g(x,u_n(x))}{u_n(x)} &\text{if } u_n(x)\neq 0, \\ 0 &\text{if } u_n(x)= 0. \end{cases} \end{equation} Since $0\leq p_n,q_n\leq M$ for some $M>0$, we may suppose that $p_n\rightharpoonup \varphi$, $q_n\rightharpoonup \psi$ in $L^2(\Omega)$ and $p_n\to \varphi$, $q_n\to \psi$ $a.e$ in $\Omega$. The fact $w^1(x)\neq 0$ implies $u_n(x)\to \infty$ and consequently, $q_n(x)\to m$. Similarly, $w^2(x)\neq 0$ yields $v_n(x)\to \infty$ and $p_n(x)\to l$. Hence, $\varphi (x)=l$ if $w^2(x)\neq 0$ and $\psi (x)=m$ if $w^1(x)\neq 0$. Since $J_\lambda'(u_n,v_n)\to 0$, for any $(\eta_1,\eta_2)\in H\times H$, we have \begin{gather}\label{2.10} \int_{\Omega} \nabla v_n \nabla \eta_1-\lambda v_n \eta_1\,dx-\int_{\Omega} g(x,u_n)\eta_1\,dx\to 0,\\ \label{2.11} \int_{\Omega} \nabla u_n \nabla \eta_2-\lambda u_n \eta_2\,dx-\int_{\Omega} f(x,v_n)\eta_2\,dx\to 0. \end{gather} It follows from $\|z_n\|\to \infty$ that \begin{gather}\label{2.12} \int_{\Omega} \nabla w_n^1 \nabla \eta_2-\lambda w_n^1 \eta_2\,dx-\int_{\Omega} p_n(x)w_n^2\eta_2\,dx\to 0,\\ \label{2.13} \int_{\Omega} \nabla w_n^2 \nabla \eta_1-\lambda w_n^2 \eta_1\,dx-\int_{\Omega} q_n(x)w_n^1\eta_1\,dx\to 0. \end{gather} Noting $p_nw_n^2, q_nw_n^1$ are bounded in $L^2(\Omega)$, we may assume $p_nw_n^2\rightharpoonup \xi(x), q_nw_n^1\rightharpoonup \zeta (x)$ in $L^2(\Omega)$ and $p_nw_n^2\to \xi(x)$, $q_nw_n^1\to \zeta (x)$ a.e. in $\Omega$. We deduce from the fact $w_n^2\to w^2$, $w_n^1\to w^1$, $p_n\to \varphi$ and $q_n\to \psi$ a.e. in $\Omega$ that $\xi=\varphi w^2=lw^2$ and $\zeta=\psi w^1=mw^1$. Let $n\to \infty$ in \eqref{2.12} and \eqref{2.13} we see that $(w^1,w^2)$ solves \eqref{2.7}. Let $\tilde{w}^2=\sqrt{\frac lm}w^2$, then $(w^1,\tilde{w}^2)$ solves \begin{equation}\label{2.14} \begin{gathered} -\Delta w^1=\lambda w^1+\sqrt{ml}w^2 \quad\text{in }\Omega, \\ -\Delta \tilde w^2=\lambda \tilde w^2+\sqrt{ml}w^1 \quad\text{in }\Omega, \\ w^1=\tilde w^2=0, \quad\text{on }\partial\Omega,\\ \end{gathered} \end{equation} which implies \begin{equation}\label{2.15} \begin{gathered} -\Delta (w^1+\tilde w^2)=(\lambda +\sqrt{ml})(w^1+\tilde w^2) \quad\text{in }\Omega, \\ w^1+\tilde w^2=0 \quad\text{on }\partial\Omega. \end{gathered} \end{equation} If $w^1+\tilde w^2\neq 0 $, this contradicts to (A3). If $w^1+\tilde w^2=0$, then \begin{equation}\label{2.16} \begin{gathered} -\Delta w^1=(\lambda -\sqrt{ml})w^1 \quad\text{in }\Omega, \\ w^1=0 \quad\text{on }\partial\Omega. \end{gathered} \end{equation} This again contradicts to (A3). For case (ii), we derive from \eqref{2.12} that $ \int_{\Omega} p_n(x)w_n^2\eta_2\,dx\to 0$ and then $w^1$ solves \begin{equation}\label{2.17} \begin{gathered} -\Delta w^1=\lambda w^1 \quad\text{in }\Omega, \\ w^1=0 \quad\text{on }\partial\Omega, \end{gathered} \end{equation} which is a contradiction to the assumption that $\lambda_{k_0}<\lambda<\lambda_{k_0+1}$. Similarly, we may rule out case (iii). The proof is complete. \end{proof} Next, we show that $J_\lambda$ has the linking structure. Denote $z_0=(u_0,u_0)$, where $u_0$ is given by assumption $(A_4)$, then $\|z_0\|^2=1$. Let $[0,s_1 z_0]=\{sz_0: 0\leq s\leq s_1\}$, $M_R=\{z=z^-+\rho z_0: \|z\|\leq R, \rho\geq 0\}$, $\tilde H=\mathop{\rm span}\{z_0\}\oplus E_-$, $S=\partial B_\rho\cap E_+$. \begin{lemma}\label{lem2.2} There exist constants $\alpha>0$ and $\rho>0$, such that $J_\lambda(u,v)\geq \alpha $ for $(u,v)\in S$. \end{lemma} \begin{proof} By (A1) and (A2), for any $\varepsilon>0$ there is $C_\varepsilon>0$ such that $$ |F(x,t)|\leq \varepsilon |t|^2+C_\varepsilon |t|^p,\quad |G(x,t)|\leq \varepsilon |t|^2+C_\varepsilon |t|^p $$ for some $2\rho$ such that $J_\lambda (u,v)\leq 0$ for $(u,v)\in \partial M_R$. \end{lemma} \begin{proof} For $z\in \partial M_R$, we write $z=z^-+r z_0$ with $\|z\|=R$, $r>0$ or $\|z\|0$. We argue by contradiction. Suppose the assertion is not true, we would have a sequence $\{z_n\}\in \partial M_R,z_n=\rho_nz_0+z_n^-,\rho_n>0,\|z_n\|=n$ such that $J_\lambda(z_n)>0$. We write $z_n=(u_n,v_n)=(\rho_nu_0+\phi_n,\rho_nu_0+\psi_n)$, then \begin{equation}\label{2.20} J_\lambda(z_n)=\frac 12 \rho_n^2-\frac 12\|z_n^-\|^2-\int_{\Omega} F(x,v_n)+G(x,u_n)\,dx>0, \end{equation} that is \begin{equation}\label{2.21} \frac {J_\lambda(z_n)}{\|z_n\|^2}=\frac 12 (\frac {\rho_n^2}{\|z_n\|^2}-\frac {\|z_n^-\|^2}{\|z_n\|^2})-\int_{\Omega} \frac {F(x,v_n)+G(x,u_n)}{\|z_n\|^2}\,dx>0. \end{equation} Since $F,G\geq 0$, then we have $\rho_n\geq \|z_n^-\|$. The fact $\frac{\rho_n^2+\|z_n^-\|^2}{\|z_n\|^2}=1$ implies $\frac 12\leq \frac{\rho_n^2}{\|z_n\|^2}\leq 1$. Assume $\frac{\rho_n^2}{\|z_n\|^2}\to \rho_0^2>0$, hence $\rho_n\to +\infty$. We may also assume $\frac {\phi_n}{\|z_n\|}\rightharpoonup \xi_1,\frac {\psi_n}{\|z_n\|}\rightharpoonup \xi_2$ in $H$ and $\frac {\phi_n}{\|z_n\|}\to \xi_1,\frac {\psi_n}{\|z_n\|}\to \xi_2$ a.e. in $\Omega$. If $x\in \Omega$ such that $\rho_0 u_0(x)+\xi_1(x)\neq 0$, then $u_n(x)=\rho_nu_0(x)+\phi_n(x)\to\infty$. Similarly, if $x\in \Omega$ such that $\rho_0 u_0(x)+\xi_2(x)\neq 0$, we have $v_n(x)=\rho_nu_0(x)+\psi_n(x)\to\infty$. It follows from (\ref{2.21}) that \begin{equation}\label{2.22} \begin{split} 0&<\frac 12 \frac {\rho_n^2}{\|z_n\|^2}-\frac 12 \frac {\|z_n^-\|^2}{\|z_n\|^2}-\int_{\Omega} [\frac {F(x,v_n)}{v_n^2}(\frac {v_n}{\|z_n\|})^2+\frac {G(x,u_n)}{u_n^2}(\frac {u_n}{\|z_n\|})^2]\,dx\\ &\leq \frac 12 \frac {\rho_n^2}{\|z_n\|^2} -\frac 12 \frac {\|z_n^-\|^2}{\|z_n\|^2} -\int_{\{\rho_0 u_0+\xi_2\neq 0\}} \frac {F(x,v_n)}{v_n^2}(\frac {v_n}{\|z_n\|})^2\,dx \\ &\quad +\int_{\{\rho_0 u_0+\xi_1\neq 0\}}\frac {G(x,u_n)}{u_n^2} (\frac {u_n}{\|z_n\|})^2\,dx \end{split} \end{equation} Let $z=\rho_0 z_0+\xi^-$ with $\xi^-=(\xi_1,\xi_2)$ and take limit in \eqref{2.22}, we get \begin{equation}\label{2.23} \begin{split} &\frac 12 (\rho_0^2\|z_0\|^2-\|\xi^-\|^2)-\frac l2 \int_{\{\rho_0 u_0+\xi_2\neq 0\}}(\rho_0 u_0+\xi_2)^2\,dx\\ &-\frac m2 \int_{\{\rho_0 u_0+\xi_1\neq 0\}}(\rho_0 u_0+\xi_1)^2\,dx\geq 0. \end{split} \end{equation} There are two cases: either $\xi^-=(\xi_1,\xi_2)\in E_{12}$, that is, $\xi_1=-\xi_2\in H^1$ or $\xi^-=(\xi_1,\xi_2)\in E_{21}$, that is, $\xi_1=\xi_2\in H^2$. In both cases we have $\int_{\Omega} (u_0\xi_1+u_0\xi_2)\,dx=0$. By (\ref{2.23}), we obtain \begin{equation}\label{2.24} \begin{split} 0&\leq \frac 12 (\rho_0^2\|z_0\|^2-\|\xi^-\|^2)-\min (l,m)\int_{\Omega} (\rho_0^2u_0^2+\xi_1^2)\,dx\\ &\leq \rho_0^2(\int_{\Omega} |\nabla u_0|^2-\lambda u_0^2\,dx-\min(l,m)\int_{\Omega} u_0^2\,dx)-\frac 12\|\xi^-\|^2-\min(l,m)\int_{\Omega} \xi_1^2\,dx\\ &<0, \end{split} \end{equation} a contradiction. \end{proof} \begin{proof}[Proof of Theorem \ref{thm1.1}] Let $L(u,v)=(v,u)$, we may check that $L$ is a bounded selfadjoint operator on $H\times H$ and that $E_{11},E_{12},E_{21}.E_{22}$ are invariant subspace of $L$, so both $E_+$ and $E_-$ are invariant subspace of $L$. (I1) of Lemma \ref{lem1.2} then holds. (I2) follows from the Sobolev compact imbeddings; (i) and (ii) in (I3) are consequences of Lemma \ref{lem2.2} and Lemma \ref{lem2.3}. The proof of (iii) in (I3) can be found in \cite{br} and \cite{R}. The proof of Theorem \ref{thm1.1} is complete. \end{proof} \section{Superlinear case} Let $\phi_1,\phi_2,\phi_3,\dots $ be the eigenfunctions of $-\Delta$ in $\Omega$ with Dirichlet boundary condition, which consist of the orthogonal basis of $L^2(\Omega)$. We assume that the eigenfunctions are normalized in $L^2(\Omega)$; i.e, $\int_\Omega \phi_i\phi_j\,dx=\delta_{ij}$. Thus, $$ L^2(\Omega)=\big\{u=\sum_{k=1}^\infty \xi_k \phi_k:\sum_{k=1}^\infty \xi_k^2<\infty\big\}, $$ and $$ (u,v)_{L^2}=\sum_{k=1}^\infty \xi_k\eta_k, $$ with $u=\sum_{k=1}^\infty \xi_k \phi_k$, $v=\sum_{k=1}^\infty \eta_k \phi_k$. For $u\in L^2(\Omega)$, we define operator $(-\Delta)^{r/2}$ by $$ (-\Delta)^{r/2}u=\sum_{k=1}^\infty \lambda_k^{r/2} \xi_k \phi_k $$ with domain $$ D((-\Delta)^{r/2})=\Theta^r(\Omega)=\big\{\sum_{k=1}^\infty \xi_k \phi_k:\sum_{k=1}^\infty \lambda_k^r\xi_k^2<\infty\big\} $$ for $r\geq 0$. It is proved in \cite{LM} that $\Theta^r(\Omega)=H_0^r(\Omega)=H^r(\Omega)$ if $00$ such that $20$ if $k=1,\dots ,k_0$; while $\mu_k^+>0$ and $\mu_k^-<0$ if $k=k_0+1,\dots $. Furthermore, $$ \mu_k^\pm\to \pm1\quad\text{as } k\to \infty. $$ Let $H^+(H^-)$ be the subspace spanned by eigenvectors corresponding to positive (negative) eigenvalues of $L_k$, then $$ E^r(\Omega)=H^+\oplus H^-. $$ Both $H^+$ and $H^-$ are infinite dimensional. Now we introduce an equivalent norm $\|\cdot\|_*$ on $E^r(\Omega)$ by $$ \frac 12\|z\|^2_*=(L{z^+},{z^+})-(L{z^-},{z^-}), $$ where $z^\pm\in H^\pm$. Then the functional corresponding to \eqref{1.1} is $$ I(z)=\frac 12 (Lz,z)_{E^r(\Omega)}-\Gamma(z) $$ for $z=(u,v)\in E^r(\Omega)$, where $$ \Gamma (z)=\int_\Omega F(x,v)\,dx+\int_\Omega G(x,u)\,dx. $$ \begin{lemma} \label{lem3.1} The functional $I$ satisfies the (PS) condition. \end{lemma} \begin{proof} Let $\{z_n\}$ be a (PS) sequence of $I$ in $E^r(\Omega)$, we need only to show that $\{z_n\}$ is bounded. Since \begin{equation}\label{3.2} \begin{split} M+\varepsilon\|z_n\| &\geq I(z_n)-\frac 12 \langle I'(z_n),z _n\rangle\\ &\geq (\frac 12-\frac 1\gamma)(\int_\Omega |u_n||g(x,u_n)|\,dx+\int_\Omega|v_n||f(x,v_n)|\,dx)-C,\\ \end{split} \end{equation} we have \begin{equation}\label{3.3} \int_\Omega |u_n||g(x,u_n)|\,dx+\int_\Omega |v_n||f(x,v_n)|\,dx\leq C+\varepsilon \|z_n\|. \end{equation} We write $z_n^\pm=(u_n^\pm,v_n^\pm)$, then \begin{equation}\label{3.4} \begin{split} \|z_n^\pm\|^2-\varepsilon \|z_n^\pm\| &\leq |\langle Lz_n,z_n^\pm \rangle-I'(z_n)z^\pm|\\ &=|\langle \Gamma'(z_n),z_n^\pm\rangle|\\ &=|\int_\Omega g(x,u_n)u_n^\pm\,dx+\int_\Omega f(x,v_n)v_n^\pm\,dx|\\ &\leq \{\int_\Omega |g(x,u_n)|^\frac {p+1}p\}^{\frac p{p+1}}\|u_n^\pm\|_{L^{p+1}}+\{\int_\Omega |f(x,v_n)|^\frac {q+1}q\}^{\frac q{q+1}}\|v_n^\pm\|_{L^{q+1}}\\ &\leq C\{1+\{\int_\Omega |g(x,u_n)||u_n|\}^{\frac p {p+1}}+\{\int_\Omega |f(x,v_n)||v_n|\}^{\frac q {q+1}}\}\|z_n^\pm\|_{E^r} \end{split} \end{equation} Dividing (\ref{3.3}) by $\|z_n^\pm\|_{E^r}$, we obtain \begin{equation}\label{3.5} \|z_n^\pm\|_{E^r}\leq C\{1+\{\int_\Omega |g(x,u_n)||u_n|\}^{\frac p {p+1}}+\{\int_\Omega |f(x,v_n)||v_n|\}^{\frac q {q+1}}\}. \end{equation} It follows from (\ref{3.3}) and (\ref{3.5}) that \begin{equation}\label{3.6} \|z_n^\pm\|_{E^r}\leq C\{1+\{C+\varepsilon \|z_n\|_{E^r}\}^{\frac p {p+1}}+\{C+\varepsilon \|z_n^\pm\|_{E^r}\}^{\frac q {q+1}}\}, \end{equation} which implies that $\|z_n\|_{E^r}$ is bounded. The proof is complete. \end{proof} \begin{proof}[Proof of Theorem \ref{thm1.3}] The proof will be completed by verifying the conditions in Lemma \ref{lem1.2}. We denote $E^1=H^+$ and $E^2=H^-$, $b(z)=\Gamma(z)$ and $L$ is defined by (\ref{3.1}). Apparently, (I1) and (I2) of Lemma \ref{lem1.2} hold. Now, we verify (I3). For $\rho>0$, let $s_1>\rho$ and $s_2$ be positive constants to be specified later. Let $e^\pm$ be the eigenvectors corresponding to the positive eigenvalue and negative eigenvalue of $L^1$ respectively and set $[0,s_1e^+]=\{se^+:0\leq s\leq s_1\}$, $Q=[0,s_1e^+]\oplus (\bar B_{s_2}\cap H^-)$, $\tilde H=\mathop{\rm span}\{e^+\}\oplus H^-$, $S=\partial B_\rho\cap H^+$. By assumption (B3), for any $\varepsilon>0$ there exists $C_\varepsilon>0$ such that $$ G(x,u)\leq \varepsilon u^2+C(\varepsilon)|u|^{p+1}, f(x,v)\leq \varepsilon v^2+C(\varepsilon)|v|^{q+1},\forall u,v\in \mathbb{R}, $$ which implies $$ I(z^+)\geq (\frac 12-\varepsilon)\|z^+\|^2-C(\varepsilon)\|z^+\|^{p+1}-C(\varepsilon)\|z^+\|^{q+1} $$ for $z^+\in E^+$. Thus, we may fix $\rho>0$ and $\alpha>0$ such that $I(z)\geq \alpha$ on $S$. This proves (i) of (I3) in Lemma \ref{lem1.2}. Next we show that for suitable choices of $s_1$ and $s_2$, $I(z)\leq 0$ on $\partial Q$. Note that the boundary of $Q$ in $\tilde H$ consists of three parts, i.e, $\partial Q=\ \{Q\cap \{s=0\}\}\cup \{Q\cap \{s=s_1\}\}\cup \{[0,s_1e^+]\oplus (\partial B_{s_2}\cap H^-)\}$. It is obvious that $I(z)\leq 0$ on $Q\cap \{s=0\}$ since $I(z)\leq 0$ for $(u,v)\leq H^-$ and $\Gamma(z)$ is nonnegative. For the remaining parts of $\partial Q$, we write $z=z^-+s e^+\in \tilde H$, then \begin{equation}\label{3.7} I(z)=\frac 12 s^2-\frac 12 \|z^-\|^2-\Gamma(z^-+s e^+). \end{equation} We may show as in \cite{hv} that \begin{equation}\label{3.8} \Gamma(z^-+s e^+)\geq Cs^\beta-C_1, \end{equation} where $\beta=\min\{p+1,q+1\}$. Therefore, \begin{equation}\label{3.9} I(z^-+s e^+)\leq \frac 12 s^2-Cs^\beta+C_1-\frac 12 \|z^-\|^2. \end{equation} Choose $s_1$ sufficient large such that $$ \psi(s)=\frac 12s^2-Cs^\beta +C_1\leq 0\ \forall s\geq s_1, $$ and then choose $s_2$ large such that $s_2^2>2\max_{s\geq 0}\psi(s)$, then we get $I(z)\leq 0$ on $\partial Q$. This proves (ii) of (I3) in Lemma \ref{lem1.2}. Since $S$ and $\partial Q$ are link. The proof is complete. \end{proof} \begin{thebibliography}{00} \bibitem{bbf} P. Bartolo, V. Benci and D. Fortunato, \emph{Abstract Critical Point Theorems and Applications to Some Nonlinear Problem with "Strong" resonance at infinity}, Nonlin. Anal. \textbf{7}(1983), 981-1012. \bibitem{br} V. Benci and P.H. Rabinowitz, \emph{Critical Point Theorem for Indefinite Functionals}, Invent. Math. \textbf{52}(1979), 241-273. \bibitem{LS} G.B. Li, A. Suzukin,\emph{An asymptotically periodic Schro\"inger equation with indefinite linear part}, Comm. Contemp. Math. \textbf{4} (2002),763-776. \bibitem{lz} G.B. Li, H.S. Zhou,\emph{The Existence of a Positive Solution to Asymptotically Linear Scalar Field Equations}, Prc. R. Soc. Edinb. A \textbf{130} (2000),81-105. \bibitem{FF} D G de Figueiredo, P.L.Felmer,\emph{On superquadratic elliptic systems}, Trans. Amer. Math. Soc\textbf{343}(1994), 99-116. \bibitem{hv} J. Hulshof and R. van der Vorst, \emph{Differential Systems with Strongly Indefinite Variational Struture}, J. Funct. Anal. \textbf{114} (1993),32-58. \bibitem{LM} J.L. Lions and E. Magenes, \emph{Non-homogeneous Boundary Value Problems and Applications I}, Springer Verlag. 1972. \bibitem{LY} G.B. Li and J.F. Yang,\emph{Asymptotically Linear Elliptic Systems}, Comm. P.D.E.\textbf{29}(2004),925-954. \bibitem{R} P.H. Rabinowitz, \emph{Minimax Theorems and Applications to Partial Differential Equations}, AMS Memoirs \textbf{65}(1986). \bibitem{sz} C.A. Stuart, H.S. Zhou,\emph{Applying the Mountain Pass Theorem to Asymptotically linear Elliptic Equation On $\mathbb{R}^N$}, Comm. P.D.E. \textbf{24} (1999),1731-1758. \bibitem{z} H.S. Zhou,\emph{An Application of a Mountain Pass Theorem}, Acta. Mathematica Sinica. \textbf{18} (2002), 27-36. \end{thebibliography} \end{document}