\documentclass[reqno]{amsart}
\usepackage{hyperref}

\AtBeginDocument{{\noindent\small
\emph{Electronic Journal of Differential Equations},
Vol. 2013 (2013), No. 276, pp. 1--10.\newline
ISSN: 1072-6691. URL: http://ejde.math.txstate.edu or http://ejde.math.unt.edu
\newline ftp ejde.math.txstate.edu}
\thanks{\copyright 2013 Texas State University - San Marcos.}
\vspace{9mm}}

\begin{document}
\title[\hfilneg EJDE-2013/276\hfil Nontrivial solutions for elliptic systems]
{Existence of nontrivial solutions for a class of elliptic systems}

\author[C. Li, Z.-Q. Ou, C.-L. Tang \hfil EJDE-2013/276\hfilneg]
{Chun Li, Zeng-Qi Ou,  Chun-Lei Tang}  % in alphabetical order

\address{Chun Li \newline
School of Mathematics and Statistics,
Southwest University, Chongqing 400715, China}
\email{Lch1999@swu.edu.cn}

\address{Zeng-Qi Ou \newline
School of Mathematics and Statistics,
Southwest University, Chongqing 400715, China}
\email{ouzengq707@sina.com}

\address{Chun-Lei Tang \newline
School of Mathematics and Statistics,
Southwest University, Chongqing 400715, China}
\email{tangcl@swu.edu.cn}

\thanks{Submitted January 28, 2013. Published December 23, 2013.}
\subjclass[2000]{35J50, 35A15, 35B38}
\keywords{Elliptic systems; generalized mountain pass theorem; critical point}

\begin{abstract}
 Using a version of the generalized mountain pass theorem, we
 obtain the existence of nontrivial solutions for a class
 of superquadratic elliptic systems.
\end{abstract}

\maketitle
\numberwithin{equation}{section}
\newtheorem{theorem}{Theorem}[section]
\newtheorem{definition}[theorem]{Definition}
\newtheorem{remark}[theorem]{Remark}
\allowdisplaybreaks


\section{Introduction and statement of results}

Consider the  elliptic system
\begin{equation}\label{HS}
\begin{gathered}
 -\Delta  u=H_v(u,v,x),  \quad  \text{in }\Omega,\\
 -\Delta v=H_u(u,v,x), \quad \text{in }\Omega,\\
u =0,\quad v=0, \quad    \text{on } \partial\Omega,
\end{gathered}
\end{equation}
where $\Omega$ is a bounded open subset of $\mathbb{R}^N$, with smooth boundary $\partial\Omega$, and   $H_u$ denotes the partial derivative
of $H$ with respect to $u$.

The system \eqref{HS} has been already studied  in the recent works \cite{Benci&Rabinowitz,Bonheure,Clement,Mitidieri,Figueiredo,Ding,
Figueiredo&Felmer,Felmer&Wang,Hulshof}  and
the reference therein.
Using  the generalized mountain pass theorem in its infinite dimensional setting,   Benci and Rabinowitz \cite{Benci&Rabinowitz}  studied a special case of  the system
\begin{equation}\label{BR}
\begin{gathered}
-\Delta  w =    H_w(w,z,x),\\
 \Delta z =    H_z(w,z,x),
\end{gathered}
\end{equation}
which is equivalent to  system \eqref{HS}.


In
 Cl\'{e}ment,  De Figueiredo and  Mitidieri \cite{Clement}
 discussed the existence of a positive solution for the system below subjected to Dirichlet boundary conditions:
\begin{equation}\label{S2}
-\Delta u=f(v),\quad -\Delta v=g(u),\quad \text{in } \Omega.
\end{equation}
In this case, the Hamiltonian is $H(u,v) = F(v) + G(u)$, where
$F(t) = \int_0^tf (s)ds$, and similarly $G$ is a primitive of $g$.
The approach in \cite{Clement} for system \eqref{S2}
was via a Topological argument, using a theorem of Krasnoselski on
Fixed Point Index for compact mappings in cones in Banach spaces.

Using a variational approach through
a version of the generalized mountain pass theorem,
De Figueiredo and  Felmer \cite{Figueiredo&Felmer} obtained the
existence of nontrivial solutions for  system \eqref{HS},
which extends the results in \cite{Benci&Rabinowitz} and \cite{Clement}.
 Felmer and Wang \cite{Felmer&Wang} proved
the existence of infinitely many strong solutions  for  the elliptic
system \eqref{HS}.
De Figueiredo and Ding \cite{Ding} studied the existence and multiplicity
of solutions of the elliptic system \eqref{BR}.
 For more details on semilinear elliptic systems of the  Hamiltonian types,
we refer the reader to \cite{Figueiredo} and the references therein.

%\begin{definition} \label{def1.1} \rm
   We say that $(u ,v)$ is a strong solution of \eqref{HS} if
\[
u\in W^{2,p/(p-1)}(\Omega)\cap W_0^{1,p/(p-1)}(\Omega),\quad
v\in W^{2,q/(q-1)}(\Omega)\cap W_0^{1,q/(q-1)}(\Omega)
\]
and $(u,v)$  satisfies $-\Delta  u=H_v(u,v,x)$ and $-\Delta v=H_u(u,v,x)$ a.e.
in $\Omega$.
%\end{definition}

In this article, motivated by \cite{Figueiredo&Felmer}, we study
the existence of strong  solutions for the elliptic system \eqref{HS}.
This kind of Hamiltonian was studied recently by  Chen and Tang \cite{Chen}
in the context of Hamiltonian systems.

 Here and in the sequel, we assume that $p\geq\alpha> p-1>0$ and
$q\geq\beta>q-1>0$ such that
 \begin{itemize}
 \item [(i)]
$\frac{1}{\alpha}+\frac{1}{\beta}<1$,

\item [(ii)]
$\{2-(\frac{1}{p}+\frac{1}{q})\}\max\{\frac{p}{\alpha},\frac{q}{\beta}\}
<1+\frac{2}{N}$,

\item [(iii)]
\[
\frac{p-1}{p}\frac{q}{\beta}<1,\quad
 \frac{q-1}{q}\frac{p}{\alpha}<1.
\]
\end{itemize}
We will always assume $N \geq 3$. If $N = 2$ or $N = 1$, we need less
restrictive assumptions. Furthermore, in the case $N \geq 5$, we also impose
\begin{itemize}
\item [(iv)]
\[
(1-\frac{1}{p})\max\{\frac{p}{\alpha},\frac{q}{\beta}\}
<\frac{N+4}{2N},\quad
(1-\frac{1}{q})\max\{\frac{p}{\alpha},\frac{q}{\beta}\}<\frac{N+4}{2N}.
\]
\end{itemize}


Our main results are the following theorems.

\begin{theorem}\label{Th1}
  Suppose that   $H $ satisfies:
\begin{itemize}
 \item[(H0)] $H: \mathbb{R}^2\times
\overline{\Omega}\to \mathbb{R} $  is of class $C^1$;

\item[(H1)]   $  H(u,v,x)\geq 0 $ for all
$(u,v,x)\in \mathbb{R}^2\times\overline{\Omega}$;

\item[(H2)] There exists $c_0>0$ such that
\[
\frac{1}{\alpha} H_u(u,v,x)\cdot u
  +\frac{1}{\beta}H_v(u,v,x)\cdot v\geq H(u,v,x)>0
\]
for all $(u,v)\in \mathbb{R}^2,\,|(u,v)|\geq c_0$ and $x\in \overline{\Omega}$;

\item[(H3)]
\[
 \lim_{|(u,v)|\to 0}\frac{H(u,v,x)}{|u|^{1+ \alpha/\beta}+|v|^{1+\beta/\alpha}}=0
\]
uniformly for $x\in \Omega$;

\item [(H4)] There exists $c_1> 0$ such that
 \begin{gather*}
   |H_u(u,v,x)|\leq  c_1(|u|^{p-1}+|v|^{(p-1)q/p}+1),\\
   |H_v(u,v,x)|\leq  c_1(|v|^{q-1}+|u|^{(q-1)p/q}+1)
  \end{gather*}
for all $(u,v)\in \mathbb{R}^2 $ and $x\in \overline{\Omega}$.
\end{itemize}
 Then problem \eqref{HS} possesses at least one
nontrivial strong solution.
\end{theorem}

\begin{remark}\label{rmk1.3} \rm
For Hamiltonian systems, the corresponding superquadratic  condition (H2)
is due to Felmer \cite{Felmer}.
The hypothesis (H3) was introduced in \cite{Chen}.
\end{remark}


\begin{theorem}\label{Th2}
Suppose that $H$ satisfies
{\rm (H1)--(H4)} and
\begin{itemize}
\item[(H0')]  $H: \mathbb{R}^2\times \overline{\Omega}\to \mathbb{R} $ is
of class $C^{1,\varepsilon}$;

\item[(H5)] $H_u(u,v,x)\geq0,\,H_v(u,v,x)\geq0 $ for all
$(u,v)\in \mathbb{R}^2$,  $u\geq0$, $v\geq0$, $x\in\overline{\Omega}$;

\item [(H6)] $H_u(u,v,x)=0 $ when $u = 0$,
$H_v(u,v,x)=0 $ when $v =0$.
\end{itemize}
Then \eqref{HS} possesses at least one positive solution $(u, v)$
with $u(x)> 0$, $v(x) > 0$ if $x\in \Omega$.
\end{theorem}

\begin{remark} \label{rmk1.5} \rm
It is easy to show that our Theorems \ref{Th1} and \ref{Th2}
generalize  Theorems 0.1 and  0.3 in \cite{Figueiredo&Felmer}.
There are functions $H$ satisfying our Theorems and not satisfying the
corresponding results in \cite{Figueiredo&Felmer}.
In fact, for $\alpha>1, \beta>1$ satisfying $1/\alpha+1/\beta<1$,
let
$$
H(u,v,x)=a_1( |u|^{1+\alpha/\beta}+|v|^{1+\beta/\alpha})^{\gamma_1}
+a_2( |u|^{1+\alpha/\beta}+|v|^{1+\beta/\alpha})^{\gamma_2},
$$
where $a_1>0$, $a_2>0$, $1<\gamma_1<\alpha\beta/(\alpha+\beta)<\gamma_2$.
Choose $\gamma_2=(\alpha\beta+1)/(\alpha+\beta)$,
$p=\alpha+1/\beta$, $q=\beta+1/\alpha$, then
$H$ satisfies  our Theorems  and does not satisfy the corresponding
results in \cite{Figueiredo&Felmer}.
\end{remark}

\begin{remark}\label{rmk1.6} \rm
If $H(u, v) =  |u|^p/p  +  |v|^q /q$ then one could use a fourth-order
approach and then assumption (iv) would not be necessary
(see \cite{Bonheure,Mitidieri}). We do not know if (iv) can be avoided
for general Hamiltonians.
\end{remark}


\section{Proof of main results}

To set up our problem variationally, we shall have to utilize
fractional Sobolev spaces.
For more details and references we cite \cite{Figueiredo&Felmer}.
Consider the spaces $E^s$,
which are obtained as the domains of fractional
powers of the operator
\[
-\Delta:H^2(\Omega)\cap H^1_0(\Omega)\subset L^2(\Omega)\to L^2(\Omega),
\]
where  $\Delta$ denotes the Laplacian and $H^2(\Omega)$, $H^1_0(\Omega)$
 are the usual Sobolev spaces.
Namely $E^s = D((-\Delta)^{s/2})$ for $0 \leq s \leq 2$,
and the corresponding operator is denoted by
\[
A^s:E^s\to L^2(\Omega).
\]
The spaces $E^s$ are Hilbert spaces with inner product
\[
(u,v)_{E^s}=\int_\Omega A^suA^sv\,dx.
\]
Its associated norm is denoted by $\|u\|_{E^s}$. In $E^s$,
we find the Poincar\'{e}'s
inequality for the operator $A^s$
\[
 \|A^su\|_{L^2(\Omega)}\geq \lambda_1^{s/2}\|u\|_{L^2(\Omega)}\quad
\text{for all } u\in E^s,
\]
where $\lambda_1$ is the first eigenvalue of $-\Delta$.

Next, we define the spaces on which we set up the problem. For numbers
$s > 0$ and $t > 0$ with $s+t = 2$, we define the Hilbert space
$E = E^s \times E^t$ and the
bilinear form $B : E \times E \to \mathbb{R}$ by the formula
\[
B((u,v),(\phi,\psi))=\int_\Omega (A^suA^t\psi+A^s\phi A^tv)dx.
\]
The bilinear form $B$ is continuous and symmetric.
There exists a selfadjoint bounded
linear operator $L : E \to E$ such that
\[
 B(z, \eta) = (Lz, \eta)_E
\]
for all $z,\,  \eta\in E$. Here $(\cdot,\cdot)_E$ denotes the natural
inner product in $E$ induced by
$E^s$ and $E^t$. We can also define the quadratic form
$\mathcal{Q} : E \to \mathbb{R}$ associated to $B$ and $L$ as
\begin{equation}\label{Q-z}
\mathcal{Q}(z)=\frac{1}{2}(Lz, z)_E=\int_\Omega A^su A^tv\,dx
\end{equation}
for all $z = (u, v) \in E$.
The operator $L$ defined above can be written as
\cite[Proposition 1.1]{Figueiredo&Felmer}
\begin{equation}\label{Luv}
 L(u, v) = ((A^s)^{-1}A^tv, (A^t)^{-1}A^su).
\end{equation}

We  define the subspaces
\begin{equation}\label{subspace}
 E^+=\{(u,A^{-t}A^su)|u\in E^s\},\quad
 E^-=\{(u,-A^{-t}A^su)|u\in E^s\},
\end{equation}
which give a natural splitting $E = E^+ \oplus E^-$.
The spaces $E^+$ and $E^-$ are the
positive and negative eigenspaces of $L$, they are consequently
orthogonal with respect to the bilinear form $B$; that is,
\[
B(z^+,z^-)=0,\quad \forall z^+\in E^+,\; \forall z^-\in E^-.
\]
We also find that
\begin{equation}\label{norm-Q}
 \frac{1}{2}\|z\|_E^2=\mathcal{Q}(z^+)-\mathcal{Q}(z^-),
\end{equation}
where $z=z^++z^-,z^{\pm}\in E^{\pm}$.

Now we will choose the numbers $s$ and $t$ defining the orders of the
Sobolev spaces involved.
From inequality (ii), we see the existence of
 $s,t \in \mathbb{R}$, $s+t = 2$
such that
 \begin{equation}\label{1-p}
 (1- \frac{1}{p})\max\{\frac{p}{\alpha},\frac{q}{\beta}\}
<\frac{1}{2}+\frac{s}{N}
 \end{equation}
and
\begin{equation}\label{1-q}
(1- \frac{1}{q})\max\{\frac{p}{\alpha},\frac{q}{\beta}\}<\frac{1}{2}+\frac{t}{N}.
 \end{equation}

By (iii) and (iv), if $N\geq5$, we can choose $s > 0$ and $t > 0$. Since
$p/\alpha\geq 1$ and $q/\beta \geq 1$, we obtain  from \eqref{1-p}
and \eqref{1-q} that
\begin{equation}\label{imbed}
 \frac{1}{p}>\frac{1}{2}-\frac{s}{N},\quad
\frac{1}{q}>\frac{1}{2}-\frac{t}{N}.
\end{equation}
These last inequalities and Sobolev Embedding Theorem give the compact
inclusions (see \cite[Theorem 1.1]{Figueiredo&Felmer})
\[
E^s \hookrightarrow L^p(\Omega),\quad
E^t \hookrightarrow L^q(\Omega).
\]
Now we can define a functional $\Phi : E\to \mathbb{R}$ as
\begin{equation}\label{functional}
\Phi(z)=\mathcal{Q}(z)-\mathcal{ H}(z)
= \int_\Omega A^su A^tv\,dx-\int_\Omega H(u,v,x)dx
\end{equation}
for $z = (u, v)\in E$. The functional $\Phi$ is of class $C^1$.
The functional
\[
\mathcal{H}(u,v)=\int_\Omega H(u(x),v(x),x)dx
\]
is of class $C^1$ and its derivative is given by
\[
\mathcal{H'}(u,v)(\phi,\psi)=\int_\Omega H_u(u,v,x)\phi
+H_v(u,v,x)\psi dx
\]
for all $(u, v), (\phi,\psi)\in E$. Moreover $\mathcal{H'} : E \to E$ is a
compact operator (see  \cite{Figueiredo&Felmer}).

For details and proof of the aspects
discussed so far, we refer the reader to \cite{Figueiredo&Felmer}.
 In particular, see in \cite{Figueiredo&Felmer}
 that critical points of $\Phi$ correspond
to the strong solutions of \eqref{HS}.

For  our proofs, we introduce the following abstract critical point
theorem due to  Felmer \cite{Felmer}.
We consider a Hilbert space $E$ with inner product $\langle\cdot,\cdot\rangle$
 and norm $\|\cdot\|$. We assume that
$E$ has a splitting $E = X \oplus Y $, where the subspaces  $X$ and $Y$
are not necessarily orthogonal
and both of them can be infinite dimensional.
Let $ \Phi :E \to \mathbb{R}$ be a functional having the
structure
\[
\Phi (z) = \frac{1}{2}\langle Lz, z\rangle +   \mathcal{H } (z).
\]
\begin{itemize}
  \item[(I1)] $L:E \to E$ is a linear, bounded, selfadjoint operator.

\item[(I2)] $\mathcal{H }'$ is compact.

\item[(I3)] There are two linear bounded, invertible operators
$B_1,B_2: E \to E$ satisfying:
 If $\omega \in \mathbb{R}_0^+$, the linear operator
\[ 
\widehat{B }(\omega) = P_XB_1^{-1} \exp(\omega L)B_2 :X\to X
\]
is invertible.
\end{itemize}
Here $P_X$ denotes the projection of $E$ onto $X$ induced by the splitting
$E = X \oplus Y$, and $\mathbb{R}_0^+$ is a set of nonnegative real numbers.

Let $\rho >0$ and define
\begin{equation}\label{S}
 S=\{B_1z:\|z\|=\rho,z\in Y\}.
\end{equation}
For $z_+\in Y$, $z_+\neq0$, $\sigma>\rho/\|B_1^{-1}B_2z_+\|$ and $M>\rho$,
we define
\begin{equation}\label{Q}
 Q=\{B_2(\tau z_++z):0\leq\tau\leq\sigma,\|z\|\leq M,z\in X\}.
\end{equation}
We define $\partial Q$ as the boundary of $Q$ relative to the subspace
\[
\{B_2(\tau z_+ +z) | \tau \in \mathbb{R}, z \in X\}.
\]
Let us consider the class of functions
\[
\Gamma = \{h\in C(E\times[0,1],E):
h\text{ satisfies the following three conditions}\}
\]
\begin{enumerate}
\item $h(z,t) = \exp(\omega(z, t)L)z +  K (z, t)$, where
$\omega :E \times[0, 1] \to \mathbb{R}_0^+ $ is continuous and
transforms bounded sets into bounded sets, and
$K :E \times[0, 1]\to E$  is compact.

\item  $h(z,t) =z$ for all $z\in\partial Q$ and all $t\in[0,1]$.

\item  $h(z,0) =z$ for all $z\in Q$.
\end{enumerate}

\begin{theorem}[\cite{Felmer}]\label{ThA}
 Let $\Phi:E \to \mathbb{R}$ be a $C^1$ functional satisfying the
Palais-Smale condition and  {\rm (I1)--(I3)}.
 Furthermore assume that there is a constant $ \delta>0$ such that
\begin{itemize}

\item[(IS)] $\Phi(z)\geq\delta$ for all $z\in S$,

\item[(IQ)] $ \Phi(z)\leq0$ for all $z\in \partial Q$.
\end{itemize}
Then $\Phi$ possesses a critical point with critical value
$d\geq \delta$ characterized by
\[
d=\inf_{h\in \Gamma}\sup_{z\in Q}\Phi(h(z,1)).
\]
\end{theorem}
Here, we define the operators $B_1$, $B_2$ and the splitting
 $E=E^s\times E^t=E^-\oplus E^+$. Let $X=E^-$ and $Y=E^+$.
We define $B_1 :E\to E$ by
\begin{equation}\label{B1}
B_1(u,v)=(\rho^{\beta-1}u, \rho^{\alpha-1}v)
\end{equation}
and $B_2 : E\to E$ by
\begin{equation}\label{B2}
B_2(u,v)=(\sigma^{\beta-1}u, \sigma^{\alpha-1}v).
\end{equation}
Certainly $B_1$ and $B_2$ are bounded linear operators and both of them
are invertible.
From \eqref{S} and \eqref{B1}, we obtain
\begin{equation}\label{IS}
S = \{(\rho^{\beta-1}u, \rho^{\alpha-1}v) : \|(u,v)\|
= \rho, (u, v) \in E^+\}.
\end{equation}
By \eqref{Q} and \eqref{B2}, we have
\begin{equation} \label{IQ}
\begin{aligned}
 Q=\big\{&\tau(\sigma^{\beta-1}u_+,\sigma^{\alpha-1}v_+)
+(\sigma^{\beta-1}u ,\sigma^{\alpha-1}v ):
 0\leq\tau\leq\sigma, \\
& 0\leq \|(u,v)\|\leq M,\,(u,v)\in E^-\big\},
\end{aligned}
\end{equation}
where $z_+ = (u_+, v_+)\in E^+$ with $u_+$ some fixed eigenvector of $-\Delta$.
In what follows, we note that $z_+$ is an eigenvector of $L$ associated to a
positive eigenvalue  (i.e. to 1).
We assume $\|z_+\|_E=1$. We denote by $\partial Q$ the boundary of $Q$
relative to the subspace
\[
 \{\tau(\sigma^{\beta-1}u_+,\sigma^{\alpha-1}v_+)+(\sigma^{\beta-1}u ,
\sigma^{\alpha-1}v ):  \tau\in \mathbb{R},\,  (u,v) \in E^-\}.
\]

Now, we can give the proof of our Theorems.

\begin{proof}[Proof of Theorem \ref{Th1}] 
The proof is divided into several steps.

\textbf{Step 1:}  $\Phi$ satisfies the Palais-Smale condition. 
See \cite[Proposition 2.1]{Figueiredo&Felmer}.


\textbf{Step 2:} We claim that $\Phi$ satisfies (I1)--(I3).
From \eqref{Q-z} and \eqref{functional}, we have
\begin{align*}
\Phi(z)&= \mathcal{Q}(z)-\int_\Omega H(u,v,x)dx \\
 &= \frac{1}{2}(Lz, z)_E-\int_\Omega H(u,v,x)dx.
\end{align*}
Taking $\mathcal{H}(z)=\int_\Omega H(z,x)dx$, we obtain
\[
(\Phi'(z),\eta)= \langle Lz, \eta\rangle- \langle\mathcal{H'}(z),\eta\rangle,
\]
where $z=(u,v)$ and $\eta=(\phi,\psi)$. So, $\Phi'=L-\mathcal{H'}$, 
where $L$ is a linear bounded selfadjoint operator. 
And, from the growth hypothesis (H4),  $\mathcal{H'}$
is a compact operator. Thus, $\Phi$ satisfies (I1) and (I2).
From  \eqref{Luv}, one has
\[
L(u, v) = ((A^s)^{-1}A^tv, (A^t)^{-1}A^su).
\]
It is well known that
\begin{gather*}
\exp(\omega L)=1+\omega L+\frac{1}{2!}\omega^2 L^2
+ \frac{1}{3!}\omega^3 L^3+\frac{1}{4!} \omega^4 L^4+\dots,\\
\cosh(\omega L)=1+\frac{1}{2!}\omega^2 L^2+ \frac{1}{4!}\omega^4 L^4+\dots,\\
\sinh(\omega L)=\omega L+\frac{1}{3!}\omega^3 L^3+ \frac{1}{5!}\omega^5 L^5+\dots.
\end{gather*}
Hence, for $\omega\in \mathbb{R}$, the operator $\exp(\omega L) : E\to E$ 
is given by
\begin{equation}\label{exp}
\exp(\omega L)(u, v) = \cosh(\omega)(u, v) 
+ \sinh(\omega)(A^{-s}  A^tv, A^{-t}  A^s u).
\end{equation}
We can give an explicit formula for  $\widehat{B}(u,v)$. 
For $z\in E^-$, one has $z = (u, -A^{-t}  A^s u)$
 with $u \in E^s$.  From \eqref{B1}, \eqref{B2}  and  \eqref{exp}, one sees
\[
B^{-1}_1 exp(\omega L)B_2z = (\xi u,  \eta A^{-t}A^su),
\]
where
\[
\xi  =  \frac{ \cosh(\omega)\sigma^{\beta-1} 
- \sinh(\omega)\sigma^{\alpha-1} }{\rho^{\beta-1}},\quad 
 \eta =  \frac{-\cosh(\omega)\sigma^{\alpha-1} 
+\sinh(\omega)\sigma^{\beta-1}}{\rho^{\alpha-1} }.
\]
Since the orthogonal projections $P^{\pm}: E \to E^\pm$ are given by the formula
(see \cite{Figueiredo&Felmer})
\[
P^{\pm}(u, v) =  \frac{1}{2}(u\pm A^{-s}A^tv, v \pm A^{-t}A^su).
\]
Using the formula for the projection into $E^-$,
we obtain
 \begin{align*}
 \widehat{B}(\omega)z &=P^-(\xi u,  \eta A^{-t}A^su)\\
 &=\frac{1}{2}(\xi u-\eta u,\eta A^{-t}A^su-\xi A^{-t}A^su)\\
 &=\frac{1}{2}((\xi-\eta)u, -(\xi-\eta) A^{-t}A^su )\\
 &= \frac{\theta}{2}( u, -A^{-t}A^su ),
  \end{align*}
where
\[
\theta=\big\{ \big(\frac{\sigma^{\beta-1}}{\rho^{\beta-1}}
 +\frac{\sigma^{\alpha-1}}{\rho^{\alpha-1}}\big)\cosh(\omega)
 - \big(\frac{\sigma^{\alpha-1}}{\rho^{\beta-1}}
 +\frac{\sigma^{\beta-1}}{\rho^{\alpha-1}}\big)\sinh(\omega)\big\}.
\]
If we assume  $\sigma> 1$ and $\rho < 1$,  it is easy to see that  
$\theta$ is positive. In fact
\[
 \big(\frac{\sigma^{\beta-1}}{\rho^{\beta-1}}
 +\frac{\sigma^{\alpha-1}}{\rho^{\alpha-1}}\big)
 -\big(\frac{\sigma^{\alpha-1}}{\rho^{\beta-1}}
 +\frac{\sigma^{\beta-1}}{\rho^{\alpha-1}}\big)
 =\frac{(\rho^{\beta-1}-\rho^{\alpha-1})
 (\sigma^{\alpha-1}-\sigma^{\beta-1})}{\rho^{\alpha+\beta-2}}
\]
is positive so that $\theta > 0$ independently of the value of
 $\omega\in \mathbb{R}$. It
implies that $\widehat{B}(\omega)$ is invertible.


\textbf{Step 3:} We claim that (IS) is satisfied, that is, there exist 
$\rho>0$ and $\delta> 0$
such that $\Phi(z)\geq\delta$, $\forall z\in S$, where $S$ is defined 
by $\eqref{IS}$.


From hypothesis (H3) and (H4),  for each $\varepsilon>0$, we have
\begin{equation}\label{H-u-v-x}
H(u,v,x)\leq \varepsilon\big(|u|^{1+ \alpha/\beta}+|v|^{1+\beta/\alpha}\big)
+c_2\big(|u|^p+|v|^q\big),
\end{equation}
where $c_2=c_2(\varepsilon)>0$. Let $\tilde{z}=(u,v)\in E^+$ and take 
$z=(\rho^{\beta-1}u,\rho^{\alpha-1}v)$
for some $\rho>0$. Then, by \eqref{H-u-v-x},  one has
\begin{equation} \label{H-leq}
\begin{aligned}
&\int_\Omega H(u,v,x)dx\\
&\leq \varepsilon\Big(\rho^{(\beta-1)(1+\alpha/\beta)}
 \int_\Omega|u|^{1+\alpha/\beta}dx
+\rho^{(\alpha-1)(1+\beta/\alpha)}
\int_\Omega|v|^{1+\beta/\alpha}dx\Big) \\
&\quad +c_2\Big(\rho^{(\beta-1)p}\int_\Omega|u|^pdx+
 \rho^{(\alpha-1)q}\int_\Omega|v|^qdx\Big).
 \end{aligned}
\end{equation}
 Since $\alpha\leq p$, $\beta\leq q$, by (i) and \eqref{imbed}, one sees that
\[
\frac{1}{1+\alpha/\beta}=\frac{\beta}{\alpha+\beta}
>\frac{1}{p}>\frac{1}{2}-\frac{s}{N}
\]
 and
\[
\frac{1}{1+\beta/\alpha}=\frac{\alpha}{\alpha+\beta}
>\frac{1}{q}>\frac{1}{2}-\frac{t}{N}.
\]
Hence, Sobolev Embedding Theorem gives  the compact inclusions
(see \cite[Theorem 1.1]{Figueiredo&Felmer})
\[
E^s \hookrightarrow L^{1+ \alpha/\beta}(\Omega),\quad
E^t \hookrightarrow L^{1+\beta/\alpha}(\Omega).
\]
By
\eqref{H-leq}, there exist two positive constants $c_3$ and $c_4$  such that
\begin{equation} \label{int-H-leq}
\begin{aligned}
 \int_\Omega H(u,v,x)dx
&\leq \varepsilon c_3\Big(\rho^{(\beta-1)(1+\alpha/\beta)}
 \|\tilde{z}\|_E^{1+\alpha/\beta}
+\rho^{(\alpha-1)(1+\beta/\alpha)}\|\tilde{z}\|_E^{1+\beta/\alpha} \big) \\
&\quad +c_4\Big(\rho^{(\beta-1)p}\|\tilde{z}\|_E^p
+\rho^{(\alpha-1)q} \|\tilde{z}\|_E^q\Big).
 \end{aligned}
\end{equation}
As $(u,v)\in E^+$, then $v =A^{-t}A^su$ and   $u = A^{-s}A^tv$. We obtain
\begin{equation}\label{Q-z-u-v}
 \mathcal{Q}(z)=\int_\Omega\rho^{\beta-1}A^su\rho^{\alpha-1}A^tv\,dx
=\rho^{\alpha+\beta-2}\int_\Omega A^suA^tv\,dx.
\end{equation}
It follows from  \eqref{norm-Q} and \eqref{Q-z-u-v} that
\begin{equation}\label{Q-rho}
 \mathcal{ Q}(z)=\frac{1}{2}\rho^{\alpha+\beta-2}\|\tilde{z}\|_E^2.
\end{equation}
If we consider $\rho=\|\tilde{z}\|_E$, from \eqref{int-H-leq}
and \eqref{Q-rho}, we obtain
\begin{equation} \label{Phi-rho}
\begin{aligned}
  \Phi(z)&\geq\frac{1}{2}\rho^{\alpha+\beta }- \varepsilon c_3(\rho^{ \beta +\alpha }
+\rho^{ \alpha + \beta })-c_4(\rho^{ \beta p} +\rho^{ \alpha  q})\\
&=\big(\frac{1}{2} -2\varepsilon c_3 \big)\rho^{ \beta +\alpha }
 - c_4(\rho^{ \beta p} +\rho^{ \alpha  q}).
\end{aligned}
\end{equation}
Since   $1/\alpha + 1/\beta<1,\,\alpha\leq p$ and $\beta\leq q$, one has
$\beta +\alpha<\alpha  q$ and $\beta +\alpha<\beta p$.
Taking $\varepsilon=1/(8c_3)$,  if  $\rho$ is small enough, by \eqref{Phi-rho},
there exists $\delta>0$ such that
\[
\Phi(z)\geq\delta>0,\quad \text{if } \|\tilde{z}\|_E=\rho
\]
and this inequality holds for $z \in S$, according to the definition of $S$.

\textbf{Step 4.} We claim that (IQ) is satisfied, that is,  
there are constants $\sigma> 0$ and $M > 0$ such that  
$\Phi(z)\leq 0$ for all $z \in \partial Q$, where $Q$ is defined 
by \eqref{IQ}.
For $\tau\in \mathbb{R}^+,\,(u,v)\in E^-$, we take
 \begin{equation}\label{ztau}
 z=\tau(\sigma^{\beta-1}u_+,\sigma^{\alpha-1}v_+)
+(\sigma^{\beta-1}u ,\sigma^{\alpha-1}v ).
 \end{equation}
From \eqref{subspace}, by the definitions of $E^+$ and $E^-$, one has
\begin{equation}\label{v-A}
 v_+=A^{-t}A^su_+,\quad v=-A^{-t}A^su.
\end{equation}
Then,  from \eqref{ztau} and \eqref{v-A} we obtain
\begin{equation} \label{Qz}
\begin{aligned}
\mathcal{Q}(z)
&=\int_\Omega (\tau \sigma^{\beta-1}A^su_++\sigma^{\beta-1}A^su)
(\tau \sigma^{\alpha-1}A^su_+-\sigma^{\alpha-1}A^su)dx \\
&= \sigma^{\alpha+\beta-2}\int_\Omega (\tau A^su_++ A^su)
(\tau  A^su_+- A^su)dx \\
&= \frac{1}{2}\sigma^{\alpha+\beta-2} (\tau^2- \|(u,v)\|_E^2).
\end{aligned}
\end{equation}
By hypothesis (H1), we see that for $\tau= 0$,
\begin{equation}\label{phiz-1}
 \Phi(z)\leq0.
\end{equation}
It follows from (H2) that there are constants $c_5 > 0$ and $c_6> 0$
such that
\[
 H(u,v,x)\geq c_5(|u|^\alpha+|v|^\beta)-c_6.
\]
So, we have
\begin{equation}\label{H-c5-c6}
 \int_\Omega H(z,x)dx\geq c_5\int_\Omega(\sigma^{\alpha(\beta-1)}|\tau u_+
 + u |^\alpha
 +\sigma^{\beta(\alpha-1)}| \tau v_++ v  |^\beta)dx-c_6|\Omega|.
 \end{equation}
Now, every  $u$ can be decomposed  as $u =\gamma u_+ + \hat{u}$,
where $ \hat{u}$ is orthogonal to $u^+$ in
$L^2(\Omega)$, and $\gamma\in \mathbb{R}$. We obtain  from H\"{o}lder's 
inequality that
\[
 (\tau+\gamma)\int_\Omega|u^+|^2dx
=\int_\Omega(\tau u^++u)u^+dx
\leq\|\tau u^++u\|_{L^\alpha(\Omega)}\|u^+\|_{L^{\alpha'}(\Omega)}.
\]
Hence, for some constant $c_7>0$, we get
\begin{equation}\label{tau+gamma}
\tau+\gamma\leq c_7\|\tau u^++u\|_{L^\alpha(\Omega)}.
\end{equation}
Similarly, we obtain
\begin{equation}\label{tau-gamma}
\tau-\gamma\leq c_7\|\tau v^++v\|_{L^\beta(\Omega)}.
\end{equation}
If $\gamma\geq 0$, we get from \eqref{Qz}, \eqref{H-c5-c6} and \eqref{tau+gamma}
that
\begin{equation}\label{tau-alpha}
\Phi(z)\leq\frac{1}{2}\sigma^{\alpha+\beta-2}  \tau^2
-c_8\tau^\alpha\sigma^{\alpha(\beta-1)}+c_6|\Omega|
\end{equation}
for some positive constant $c_8$.
And, if $\gamma\leq 0$, we conclude from  \eqref{Qz}, \eqref{H-c5-c6} and
\eqref{tau-gamma} that
\begin{equation}\label{tau-beta}
\Phi(z)\leq\frac{1}{2}\sigma^{\alpha+\beta-2}  \tau^2
-c_8\tau^\beta\sigma^{\beta(\alpha-1)}+c_6|\Omega|.
\end{equation}
Choosing $ \tau= \sigma$, and taking $\sigma$ large enough it follows
from $1/\alpha + 1/\beta<1$, \eqref{tau-alpha} and \eqref{tau-beta} that
\begin{equation}\label{phiz-2}
\Phi(z)\leq0.
\end{equation}
Finally,  we choose $M$. Given $ \tau\in(0,\sigma)$, we deduce
from \eqref{Qz} and \eqref{H-c5-c6} that
\[
\Phi(z)\leq \frac{1}{2}\sigma^{\alpha+\beta}
- \frac{1}{2}\sigma^{\alpha+\beta-2} \|(u,v)\|_E^2+c_6|\Omega|.
\]
So that if $M$ is enough large and $\|(u,v)\|_E^2=M$, one has
\begin{equation}\label{phiz-3}
\Phi(z)\leq0.
\end{equation}
Thus, from \eqref{phiz-1}, \eqref{phiz-2} and \eqref{phiz-3}, we have
\[
\Phi(z)\leq0,\quad \forall z\in \partial Q.
\]

Hence, the hypothesis of Theorem \ref{ThA} is satisfied. 
Thus, there exists $z\in E$ such that $\Phi'(z) = 0$;
i.e., $z$ is an $(s, t)$-weak solution of \eqref{HS}. 
Next,  \cite[Theorem 1.2]{Figueiredo&Felmer}  gives that $z = (u,v)$ is
such that $u\in W^{2,p/(p-1)}(\Omega)\bigcap W_0^{1,p/(p-1)}(\Omega)$
and $v\in W^{2,q/(q-1)}(\Omega)\bigcap W_0^{1,q/(q-1)}(\Omega)$. That
is, $(u, v)$ is a strong solution of \eqref{HS}.

Moreover, $(0,0)$ is a solution of \eqref{HS}. 
Since $\Phi(z)\geq\delta> 0$ and $\Phi(0, 0)  = 0$, it implies  
that $(u, v)$  is not trivial.
\end{proof}

\begin{proof}[Proof of Theorem \ref{Th2}] 
Here, we define the functional
$\widehat{\Phi}:E\to \mathbb{R}$ as
\[
\widehat{\Phi}(z)=\mathcal{Q}(z)-\int_\Omega \widehat{H}(z,x)dx,
\]
where
\[
\widehat{H}(u,v,x)=\begin{cases}
       H(u,v,x), & \text{if } u\geq0,v\geq0, \\
       H(0,v,x), & \text{if } u\leq0,v\geq0,\\
        H(u,0,x), & \text{if } u\geq0,v\leq0,\\
        0, & \text{if }  u\leq0,v\leq0.
     \end{cases}
\]
From (H6), $\widehat{H}$ is of class $C^{1,\varepsilon}$. And,
$\widehat{H}$ satisfies (H1), (H3) and (H4). Moreover,
(H2) is satisfied in a restricted form. Obviously, the critical
points of $\widehat{\Phi}$ correspond to the strong solutions of
\begin{gather*}
 -\Delta  u=\widehat{H}_v(u,v,x),  \quad  \text{in }\Omega,\\
 -\Delta v=\widehat{H}_u(u,v,x), \quad \text{in } \Omega,\\
u =0,\quad v=0,  \quad    \text{on } \partial\Omega.
\end{gather*}
Since $\widehat{H}_u(u,v,x)\geq0$ and $\widehat{H}_v(u,v,x)\geq 0$,
by the maximum principle, we obtain that 
$u > 0$ and $v > 0$ in $\Omega$. As the proof of Theorem \ref{Th1}, 
we can get that hypotheses of Theorem \ref{ThA} still hold. 
Hence,  \eqref{HS} possesses at least one positive solution $(u, v)$ 
with $u(x)> 0, v(x) > 0$ if $x\in \Omega$.
\end{proof}

\subsection*{Acknowledgements}
The authors would like to thank the anonymous referees for their
valuable suggestions.

This research was supported by the
National Natural Science Foundation of China (No. 11071198) and the
Fundamental Research Funds for the Central Universities (No.
XDJK2010C055).

\begin{thebibliography}{00}
\bibitem{Benci&Rabinowitz} V. Benci, P. Rabinowitz;
\emph{Critical point theorems for indefinite
functionals}, Invent. Math. 52 (1979), 241-273.

\bibitem{Bonheure}  D. Bonheure,  E. Moreira dos Santos,  M. Ramos;
\emph{Ground state and non-ground state solutions of some strongly 
coupled elliptic systems}, Trans. Amer. Math. Soc. 364 (2012), no. 1, 447-491.

\bibitem{Chen} S.-J. Chen, C.-L. Tang;
\emph{Periodic and subharmonic solutions of a class
of superquadratic Hamiltonian systems}, J. Math. Anal. Appl. 297 (2004) 267-284.



\bibitem{Clement} Ph. Cl\'{e}ment, D. G. De Figueiredo, E. Mitidieri;
\emph{Positive solutions of semilinear elliptic systems}, 
Comm. Partial Differential Equations 17 (1992), 923-940.

\bibitem{Mitidieri} Ph. Cl\'{e}ment, P. Felmer, E. Mitidieri;
\emph{Homoclinic orbits for a class of infinite-dimensional Hamiltonian systems}, 
Ann. Scuola Norm. Sup. Pisa Cl. Sci. (4) 24 (1997), no. 2, 367-393.

\bibitem{Figueiredo} D. G. De Figueiredo;
\emph{Semilinear elliptic systems: existence, multiplicity, symmetry of solutions},
 Handbook of differential equations: stationary partial differential equations. 
Vol. V, 1-48, Handb. Differ. Equ., Elsevier/North-Holland, Amsterdam, 2008.

\bibitem{Ding} D. G. De Figueiredo, Y. H. Ding;
\emph{Strongly indefinite functionals and multiple
 solutions of elliptic systems},
Trans. Amer. Math. Soc. 355 (2003), 2973-2989.



\bibitem{Figueiredo&Felmer} D. G. De Figueiredo, P. L. Felmer;
\emph{On Superquadratic elliptic systems},
 Trans. Amer. Math. Soc. 343 (1994), 99-116.

 \bibitem{Felmer} P. L. Felmer;
\emph{Periodic solutions of  superquadratic Hamiltonian systems},
 J. Differential Equations 102 (1993) 188-207.

\bibitem{Felmer&Wang} P. L. Felmer, Z. Q. Wang;
\emph{Multiplicity for symmetric indefinite functionals:
 Application to Hamiltonian and elliptic systems}, 
Topol. Methods Nonlinear Anal. 12 (2) (1998), 207-226.

\bibitem{Hulshof}  J. Hulshof, R. C. A. M. van der Vorst;
\emph{Differential systems with strongly
indefinite variational structure}, J. Funct. Anal. 114 (1993), 32-58.

\end{thebibliography}

\end{document}