\documentclass[reqno]{amsart}
\usepackage{hyperref}
\AtBeginDocument{{\noindent\small
\emph{Electronic Journal of Differential Equations},
Vol. 2020 (2020), No. 52, pp. 1--21.\newline
ISSN: 1072-6691. URL: http://ejde.math.txstate.edu or http://ejde.math.unt.edu}
\thanks{\copyright 2020 Texas State University.}
\vspace{8mm}}
\begin{document}
\title[\hfilneg EJDE-2020/52\hfil Weak solutions to superlinear elliptic systems]
{Existence of weak solutions to superlinear elliptic systems without
the Ambrosetti-Rabinowitz condition}
\author[X. Wang, P. Zhao \hfil EJDE-2020/52\hfilneg]
{Xiaohui Wang, Peihao Zhao}
\address{Xiaohui Wang \newline
School of Mathematics and Statistics,
Lanzhou University, Lanzhou 730000, China}
\email{xiaohuiwang1@126.com}
\address{Peihao zhao \newline
School of Mathematics and Statistics,
Lanzhou University, Lanzhou 730000, China}
\email{zhaoph@lzu.edu.cn}
\thanks{Submitted May 8, 2018. Published May 27, 2020.}
\subjclass[2010]{35J20, 35J47, 35J15, 35A15}
\keywords{Superlinear elliptic system; weak solution;
\hfill\break\indent mountain pass lemma; linking theorem}
\begin{abstract}
In this article, we study the existence of the weak solution for
superlinear elliptic equations and systems without the Ambrosetti-Rabinowitz condition.
The Ambrosetti-Rabinowitz condition guarantees the boundedness of the $PS$ sequence
of the functional $I$ for the corresponding problem.
We establish the existence of the weak solution for the superlinear elliptic equation
by using $(PS)_c$ form of the Mountain pass lemma, and the existence of the weak solution
for the superlinear elliptic system by using $(PS)_c^{*}$ form of the Linking theorem.
\end{abstract}
\maketitle
\numberwithin{equation}{section}
\newtheorem{theorem}{Theorem}[section]
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{remark}[theorem]{Remark}
\allowdisplaybreaks
\section{Introduction and statement of main results}
In this article, we investigate the existence of the nontrivial weak solution for
the superlinear elliptic problems. We first consider the $p$-Laplacian equation
\begin{equation}\label{a1}
\begin{gathered}
-\Delta_{p} u= \lambda f(x,u)\quad \text{in }\Omega, \\
u=0 \quad \text{on }\partial \Omega,
\end{gathered}
\end{equation}
where $p>1$, $\lambda>0$, $\Omega\subset \mathbb{R}^n$ is a bounded domain,
$f:\overline{\Omega}\times \mathbb{R}\to \mathbb{R}$ is a continuous function,
and for $1
p$ and $r>0$ such that
$$
0<\theta F(x,t)\leq f(x,t)t\quad\text{for all } |t|\geq r\ \text{and}\ x\in\Omega,
$$
where
$$
F(x,t)= \int_0^t f(x,s)ds.
$$
Since 1973 when Ambrosetti and Rabinowitz~\cite{AR} established the Mountain pass lemma
under the AR condition, many researchers have studied the superlinear elliptic
problems under the AR condition. The AR condition guarantees the boundedness of
the $PS$ sequence of the functional $I$ given by the corresponding problem, which
plays a key role in the application of the critical point theory.
Although the AR condition is convenient, it is very restrictive and excludes a lot
of nonlinear problems. Therefore, many researchers have been studied various problems
without the AR condition.
In 2004, Schechter and Zou \cite{SZ} established the existence of
nontrivial weak solution for the problem \eqref{a1} without the AR condition when $p=2$.
In this paper, for a general $p$ $(1
p$ and $r>0$ such that
$$
\mu F(x,t)-tf(x,t)\leq C(|t|^p+1)\quad \text{for all }|t|\geq r \text{ and }x\in\Omega.
$$
\end{itemize}
\begin{theorem}\label{thm1.1}
If $f$ satisfies {\rm (H1)--(H4)}, then
for each $\lambda >0$, problem \eqref{a1} has at least one nontrivial solution.
\end{theorem}
Secondly, we consider the non-cooperative elliptic system
\begin{equation}\label{a2}
\begin{gathered}
-\triangle u=H_u(x,u,v)\quad x \in \Omega,\\
-\triangle v=-H_v(x,u,v)\quad x \in \Omega,\\
u(x)=v(x)=0\quad x \in \partial\Omega,\\
\end{gathered}
\end{equation}
where $\Omega\subset \mathbb{R}^n$ $(n\geq 3)$ is a smooth bounded domain,
$H:\overline{\Omega}\times{\mathbb{R}}^2\to\mathbb{R}$ is a $C^1$ function, $H_u$
denotes the partial derivative of $H$ with respect to the variable $u$.
We write $z:=(u,v)$, we suppose $H(x,0)\equiv 0$ and $H_z(x,0)\equiv 0$,
then $z=0$ is a trivial solution for this system. We will also establish the
existence of the nontrivial solution for the elliptic system \eqref{a2} in this paper.
Roughly speaking, we are mainly interested in the class of Hamiltonians $H$ such that
$$
H(x,u,v)\sim |u|^p+|v|^q+R(x,u,v)\quad \text{with }
\lim_{|z|\to{\infty}}\frac{R(x,u,v)}{|u|^p+|v|^q}=0,
$$
where $1
1$.
For elliptic system, we shall say $H$ satisfies the AR condition, if there exist
$\mu>2,~\nu>1$ and $R\geq 0$ such that
$$
\frac{1}{\mu}H_u(x,z)u+\frac{1}{\nu}H_v(x,z)v\geq H(x,z)\quad\text{whenever}~|z|\geq R,
$$
with the provision that $\nu=\mu$ if $q>2$.
In 1995, by using variational method, Costa and Magalhaes \cite{CM} established the
existence of the nontrivial weak solution for the subcritical non-cooperative elliptic
system without the AR condition.
In 2004, Lam and Lu \cite{LL} obtained the existence of the nontrivial weak solution
for the critical and subcritical superlinear cooperative elliptic system without the AR
condition.
In 2003, De Figueiredo and Ding \cite{DD} obtained the existence of the nontrivial
weak solution for the supercritical superlinear non-cooperative elliptic system
when $2
2$ and $R_1>0$ such that
$$
\mu H(x,z)-zH_z(x,z)\leq C(|z|^p+1)\quad\text{whenever } |z|\geq R_1;
$$
\item[(H8)] For $p$ and $q$ as above,
$$
H(x,z)\geq \gamma_1(|u|^p+|v|^q)-\gamma_2\quad\text{for all }(x,z);
$$
\item[(H9)] $H(x,0,v)\geq 0$ and
$H_u(x,u,0)=o(|u|)$ uniformly with respect to $x$, as $u\to 0$.
\end{itemize}
\begin{theorem}\label{thm1.2}
Suppose $H$ satisfies {\rm (H5)--(H8)}.
Then the superlinear elliptic system \eqref{a2} has at least one nontrivial weak solution.
\end{theorem}
The rest of this paper is organized as follows.
In section 2, we will discuss the superlinear elliptic equation \eqref{a1} by a variational
method, and establish the existence of the nontrivial weak solution for this
superlinear elliptic equation. Furthermore, we will investigate the superlinear
non-cooperative elliptic system \eqref{a2} by variational method in section 3,
and establish the existence of the nontrivial weak solution for this superlinear
non-cooperative elliptic system.
\section{Superlinear elliptic equation}
In this section, we establish the existence of the nontrivial weak solution for the
superlinear elliptic boundary value problem \eqref{a1} of $p$-Laplacian type.
\subsection{Preliminaries}
Throughout this section, let $\Omega$ be a bounded domain in $\mathbb{R}^n$.
For $1
0$, there exists $\delta(\epsilon_1)>0$
such that
$$
A\Big(\frac{x+y}{2}\Big)\leq\frac{1}{2}A(x)+\frac{1}{2}A(y)-\delta(\epsilon_1),
$$
for $x,y\in S$ with $\|x-y\|>\epsilon_1$. If $A$ is uniformly convex on every ball of $X$,
we shall say that $A$ is locally uniformly convex, i.e., if for any $\epsilon_2>0$,
there exists $\delta(\epsilon_2)>0$ such that $x,~y\in X,~|A(x)|\leq 1,~|A(y)|\leq 1$
and $|A(x-y)|>\epsilon_2$, then
$$
\big|A\big(\frac{x+y}{2}\big)\big|<1-\delta(\epsilon_2).
$$
\end{definition}
\begin{remark}[\cite{NM}] \label{rm2.2} \rm
$X$ is uniformly convex if and only if its norm is locally uniformly convex.
\end{remark}
\begin{remark}[\cite{NM}]\label{rm2.3} \rm
The Banach space $W_0^{1,p}(\Omega)$ with norm
$\|u\|=\big(\int_{\Omega}|\nabla u|^pdx\big)^{1/p}$ is uniformly convex.
\end{remark}
\begin{remark}[\cite{B}]\label{rm2.4} \rm
Every uniformly convex Banach space is reflexive. That is, the Banach space
$W_0^{1,p}(\Omega)$ is reflexive.
\end{remark}
\begin{remark}\cite{ZFC}\label{rm2.5} \rm
Let $X$ be a reflexive Banach space, $\{x_n\}$ is a bounded sequence in $X$.
Then $\{x_n\}$ has weak convergent subsequence. That is, the bounded sequence in
reflexive Banach space $W_0^{1,p}(\Omega)$ has weak convergent subsequence.
\end{remark}
\begin{definition}\label{def2.6} \rm
Let $I$ be a functional defined in Banach space $X$. We say that $I$ is weakly
lower semicontinuous, if for any sequence $\{x_{n}\}$ such that
$x_{n}\rightharpoonup x$ weakly, then we have
$$
\liminf_{n\to \infty}I(x_{n})\geq I(x).
$$
\end{definition}
\begin{definition}\label{def2.7} \rm
An operator $I':X\to X^*$ satisfies the $(S_+)$ condition, if for every sequence
$\{x_n\}\subset X$ such that $x_n\rightharpoonup x$ and
$$
\limsup_{n\to+\infty}\langle I'(x_n),x_n-x\rangle\leq 0,
$$
we have $x_n\to x$ strongly.
\end{definition}
We would like to mentioned that the $(S_+)$ condition is used to prove that the
weak convergent sequence obtained is actually strongly convergent.
Next, we verify that the relevant functional satisfies the $(S_+)$ condition.
\begin{proposition}\label{prop2.8}
Let $X$ be a Banach space. We denote $I(x)=\|x\|^p$, where $p\geq 1$, $x\in X$, then
$I:X\to \mathbb{R}$ is $C^1$ and $I':X\to X^{*}$ satisfies the $(S_+)$ condition.
\end{proposition}
\begin{proof}
It is easy to verify that $I:X\to \mathbb{R}$ is a $C^1$ functional.
Let $\{x_n\}$ be a sequence in $X$ such that $x_n\rightharpoonup x$ and
$$
\limsup_{n\to+\infty}\langle I'(x_n),x_n-x\rangle\leq 0.
$$
\noindent\textbf{Claim:} $x_n\to x$ in $X$.
Indeed, since $\{x_n\}$ is weakly convergent, it is bounded. That is, there is a large
enough $R>0$ such that
$\|x_n\|0$ and a subsequence $\{x_{n}\}$ that verifies $\|x-x_{n}\|\geq\epsilon$.
Using the uniform convexity of $I$ over ball $B(0,R)$, we obtain that there exists a
$\delta(\epsilon)>0$ such that
$$
\frac{1}{2}I(x)+\frac{1}{2}I(x_{n})-I\big(\frac{x+x_{n}}{2}\big)\geq\delta(\epsilon).
$$
Taking $n\to+\infty$, we have
$$
\limsup I\big(\frac{x+x_{n}}{2}\big)\leq c-\delta(\epsilon),
$$
which contradicts \eqref{b1}. Then the desired conclusion follows from the claim.
\end{proof}
\begin{definition}\label{def2.9} \rm
Let $(X,\|\cdot\|_X)$ be a real Banach space with dual space $(X^*,\|\cdot\|_{X^*})$,
and $I\in C^1(X,\mathbb{R})$. For $c\in \mathbb{R}$, we shall say $I$ satisfies
the $(PS)_c$ condition, if for any sequence $\{x_n\}\subset X$ such that
$I(x_n)\to c$ and $I'(x_n)\to 0$, we have that $\{x_n\}$ is strongly convergent in $X$.
\end{definition}
\begin{theorem}[Mountain pass lemma \cite{ZFC}] \label{thm2.10}
Let $X$ be a real Banach space, $I\in C^{1}(X,\mathbb{R})$ satisfies
\begin{itemize}
\item[(1)] $I(0)\leq 0$;
\item[(2)] There exist constants $\rho,~\alpha>0$ such that
$I(u)\geq\alpha$, when $\|u\|=\rho$;
\item[(3)] There exists an $e\in E\setminus {B_{\rho}}$ such that $I(e)<0$.
\end{itemize}
Denote
$c=\inf_{\gamma\in\Gamma}\max_{0\leq t\leq 1}I(\gamma(t))$,
where
$$
\Gamma=\{\gamma\in C([0,1];X):\gamma(0)=0,\gamma(1)=e\}.
$$
Then $c>0$ and there is a sequence $\{x_n\}\subset X$ such that
$$
I(x_n)\to c,\quad I'(x_n)\to 0.
$$
Furthermore, if $f$ satisfies the $(PS)_c$ condition, then $c$ is the critical value of $I$.
\end{theorem}
\subsection{Existence of a nontrivial weak solution to the elliptic equation}
In this subsection, we establish the existence of the nontrivial weak solution for the
elliptic equation. We firstly introduce the energy functional corresponding to the
elliptic equation \eqref{a1}.
If $\Omega\subset \mathbb{R}^n$ is a bounded domain and $f$ satisfies $(H1)$ and $(H2)$,
then we define functional in $W_0^{1,p}(\Omega)$,
\begin{equation}\label{b2}
I_{\lambda}(u)=\frac{1}{p}\int_{\Omega}|\nabla u|^pdx-\lambda\int_{\Omega}F(x,u)dx.
\end{equation}
For any $\lambda\in \mathbb{R}^1$, a straightforward computation yields that
$I_{\lambda}\in C^1(W_0^{1,p}(\Omega),\mathbb{R})$, and
\begin{equation}\label{b3}
\langle I_{\lambda}'(u),v\rangle
=\int_{\Omega}|\nabla u|^{p-2}\nabla u\nabla v\,dx-\lambda\int_{\Omega}f(x,u)v\,dx,
\end{equation}
for any $u\in W_0^{1,p}(\Omega)$.
Next, we prove that the functional $I_{\lambda}$ satisfies the mountain pass geometry
as follows.
\begin{lemma}\label{lem2.11}
If $\lambda>0$ and f satisfies {\rm (H1)--(H3)}, then
\begin{itemize}
\item[(1)] $I_{\lambda}(u)$ is unbounded from below in $W_0^{1,p}(\Omega)$;
\item[(2)] $u=0$ is a strictly local minimum for $I_{\lambda}(u)$.
\end{itemize}
\end{lemma}
\begin{proof}
For any $M>0$, it follows from (H3) that there is a $C_M>0$ such that
\begin{equation}\label{b4}
F(x,t)\geq Mt^p-C_M\quad\text{for all }t\geq 0 \text{ and all } x\in\Omega.
\end{equation}
Indeed, for any $M>0$, there is a $s_0>0$ such that
$$
\frac{F(x,t)}{t^p}\geq M\quad\text{whenever }t>s_0.
$$
That is,
$F(x,t)\geq Mt^p$ whenever $t>s_0$.
Furthermore, thanks to $F$ being continuous on $\overline{\Omega}\times[0,s_0]$, we have
$$
\max_{{x\in\overline{\Omega}},{0\leq t \leq s_0}}\{F(x,t)-Mt^p\}\leq C_M.
$$
Also since
$$
F(x,t)-Mt^p+\max_{{x\in\overline{\Omega}},{0\leq t \leq s_0}}\{F(x,t)-Mt^p\}\geq 0,
$$
we obtain
$F(x,t)\geq Mt^p-C_M$,
for any $x\in\overline{\Omega}$ and $0\leq t \leq s_0$.
To sum up, we obtain \eqref{b4}.
Taking $\phi\in W_0^{1,p}(\Omega)$ with $\phi>0$, and $t\geq 0$.
Then for any $\lambda>0$, we have
\begin{align*}
I_\lambda(t\phi)
& = \frac{1}{p}t^p\int_{\Omega} |\nabla\phi|^pdx -\lambda\int_\Omega F(x,t\phi)dx \\
&\leq \frac{1}{p}t^p\|\phi\|^p-\lambda t^pM\int_{\Omega}\phi^pdx+\lambda C_M|\Omega|\\
&=t^p\Big(\frac{1}{p}\|\phi\|^p-\lambda M\int_\Omega \phi^pdx\Big)+\lambda C_M|\Omega|.
\end{align*}
If $M$ is large enough such that
$$
\frac{1}{p}\|\phi\|^p-\lambda M\int_\Omega\phi^p dx<0,
$$
then
$ \lim_{t\to +\infty}I_\lambda (t\phi)=-\infty$,
which is equivalent to $(1)$.
On the other hand, for any $\epsilon>0$, by using (H1) and (H2),
it is easy to see that there exists a $C_\epsilon>0$ such that
$$
|f(x,t)|\leq\epsilon|t|^{p-1}+C_{\epsilon}|t|^{q-1}\quad \text{for all }
(x,t)\in \overline{\Omega}\times \mathbb{R}.
$$
That is,
\begin{equation}\label{b5}
|F(x,t)|\leq\epsilon|t|^p+C_\epsilon|t|^q\quad \text{for all }
(x,t)\in \overline{\Omega}\times \mathbb{R},
\end{equation}
where $q\in(p,p^*)$. Indeed, in view of $(H1)$, for any $\epsilon>0$, there is a
$\delta>0$ such that
$$
\frac{|f(x,t)|}{|t|^{p-1}}<\epsilon\quad\text{for any }|t|<\delta.
$$
That is,
$|f(x,t)|<\epsilon|t|^{p-1}$ for any $|t|<\delta$.
Furthermore, from (H2), we have
\begin{gather*}
|f(x,t)|\leq a+b|t|^{q-1}\leq a|t|^{q-1}+b|t|^{q-1}=(a+b)|t|^{q-1}\quad\text{for }|t|>1, \\
|f(x,t)|\leq a+b|t|^{q-1}=(a|t|^{1-q}+b)|t|^{q-1}\leq(a|\delta|^{1-q}+b)|t|^{q-1}
\quad \text{for }\delta\leq|t|\leq 1.
\end{gather*}
Therefore, for any $\epsilon>0$, there is a $C_\epsilon>0$ such that
\begin{equation}\label{b6}
|f(x,t)|\leq \epsilon|t|^{p-1}+C_\epsilon|t|^{q-1}\quad
\text{for all }(x,t)\in \overline{\Omega}\times \mathbb{R},
\end{equation}
where $C_\epsilon=\max\{(a+b),a|\delta|^{1-q}+b\}$.
It follows from \eqref{b5} and the Poincar\'e inequality that
$$
\|u\|_p^p\leq\frac{1}{\lambda_1}\|u\|^p,
$$
where
$$
0<\lambda_1=\inf_{u\in W_0^{1,p}(\Omega),u\neq0}\frac{\|u\|^p}{\|u\|_p^p}.
$$
And therefore, for any $\lambda>0$ and $\epsilon>0$ small enough such that
$\frac{1}{p}-\frac{\lambda\epsilon}{\lambda_1}>0$, the H\"older inequality implies
\begin{equation}\label{b7}
\begin{aligned}
I_\lambda(u)
&=\frac{1}{p}\|u\|^p-\lambda\int_\Omega F(x,u)dx\\
&\geq\frac{1}{p}\|u\|^p-\lambda\epsilon\int_\Omega |u|^pdx-\lambda C_\epsilon\int_\Omega|u|^qdx\\
&\geq\frac{1}{p}\|u\|^p-\lambda\epsilon\int_{\Omega} |u|^pdx
-\lambda C_\epsilon |\Omega|^\frac{p-q}{p}\Big(\int_\Omega|u|^pdx\Big)^{q/p}\\
&\geq\Big(\frac{1}{p}-\frac{\lambda\epsilon}{\lambda_1}\Big)
\|u\|^p-\lambda C_\epsilon|\Omega|^\frac{p-q}{p}
\Big(\frac{1}{\lambda_1}\|u\|^p\Big)^{q/p}\\
&=\Big(\frac{1}{p}-\frac{\lambda\epsilon}{\lambda_1}\Big)
\|u\|^p-\frac{\lambda C_\epsilon}{\lambda_1^{q/p}}|\Omega|^{\frac{p-q}{p}}\|u\|^q\\
&=\Big(\frac{1}{p}-\frac{\lambda\epsilon}{\lambda_1}
-\frac{\lambda C_\epsilon}{\lambda_1^{q/p}}|\Omega|^{\frac{p-q}{p}}\|u\|^{q-p}\Big)\|u\|^p\\
&\geq\frac{1}{2}\Big(\frac{1}{p}-\frac{\lambda\epsilon}{\lambda_1}\Big)\|u\|^p,\\
\end{aligned}
\end{equation}
provided $\|u\|=\rho$ is sufficiently small such that
$$
\frac{\lambda C_\epsilon}{\lambda_1^{q/p}}|\Omega|^{\frac{p-q}{p}}\|u\|^{q-p}
<\frac{1}{2}\Big(\frac{1}{p}-\frac{\lambda\epsilon}{\lambda_1}\Big),
$$
when $q\in(p,p^*)$. Therefore $u=0$ is a strictly local minimum for $I_\lambda(u)$.
\end{proof}
\begin{lemma}\label{lem2.12}
Assume $f$ satisfies {\rm (H1)--(H3)} and $0<\lambda_0<\mu_0$.
Then $I_\lambda(u)$ possesses uniform mountain pass geometric structure around $u=0$
for $\lambda\in [\lambda_0,\mu_0]$, i.e., there is an $e\in W_0^{1,p}(\Omega)$
such that $I_\lambda(e)<0$ for any $\lambda\in [\lambda_0,\mu_0]$, and there exist
constants $\rho,~\alpha>0$ such that $I_{\lambda}(u)\geq\alpha$ for any
$\lambda\in [\lambda_0,\mu_0]$, and $u\in W_0^{1,p}(\Omega)$ with $\|u\|=\rho$.
\end{lemma}
\begin{proof}
Fix $\epsilon>0$ small enough, in view of \eqref{b7}, we have
$$
I_\lambda(u)\geq\frac{1}{2}\Big(\frac{1}{p}-\frac{\mu_0\epsilon}{\lambda_1}\Big)\|u\|^p,
$$
for any $\lambda\in[\lambda_0,\mu_0]$ and $u\in W_0^{1,p}(\Omega)$.
Thus there is a $\rho=\rho(\mu_0,\epsilon)>0$, taking
$$
\alpha=\frac{1}{2}\Big(\frac{1}{p}-\frac{\mu_0\epsilon}{\lambda_1}\Big)\|\rho\|^p,
$$
we have
$I_\lambda(u)\geq\alpha$,
for any $\lambda\in[\lambda_0,\mu_0]$ and $u\in W_0^{1,p}(\Omega)$ with $\|u\|=\rho$.
Let us take $\phi\in W_0^{1,p}(\Omega)$ with $\phi>0$, and $M>0$ large enough such that
$$
\frac{1}{p}||\phi||^p-\lambda_0M\int_\Omega\phi^pdx<0.
$$
As a consequence of \eqref{b4}, for any $t>0$, we have
%\label{eq}
\begin{align*}
I_{\lambda_0}(t\phi)
&\leq\frac{1}{p}t^p\|\phi\|^p-\lambda_0t^pM\int_\Omega\phi^pdx+\lambda_0C_M|\Omega|\\
&\leq t^p\Big(\frac{1}{p}\|\phi\|^p-\lambda_0M\int_\Omega\phi^pdx\Big)
+\lambda_0C_M|\Omega|.
\end{align*}
Furthermore, taking $e=t_0\phi$ with $t_0$ large enough such that $I_{\lambda_0}(e)<0$,
for any $0<\lambda_0<\lambda$, we have
$$
I_\lambda(e)0$ such that
\begin{equation}\label{b11}
\frac{F(x,u_n)}{|u_n|^p}>1,
\end{equation}
for any $x\in\Omega$ and $|u_n|\geq N_0$. Since $F$ is continuous on
$\overline{\Omega}\times[-N_0,N_0]$, there is a $M>0$ such that
\begin{equation}\label{b12}
|F(x,u_n)|\leq M\quad\text{for all } (x,u_n)\in\overline{\Omega}\times[-N_0,N_0].
\end{equation}
Combining \eqref{b11} with \eqref{b12}, we deduce that there is a constant $C$ such that
$F(x,u_n)\geq C$ for all $(x,u_n)\in\overline{\Omega}\times \mathbb{R}$,
which shows that
\begin{equation}\label{b13}
\frac{F(x,u_n)-C}{\|u_n\|^p}\geq 0.
\end{equation}
Thanks to \eqref{b8}, we have
$$
c=I_\lambda(u_n)+o(1)=\frac{1}{p}\|u_n\|^p-\lambda\int_\Omega F(x,u_n)dx+o(1).
$$
So we obtain
\begin{equation}\label{b14}
\|u_n\|^p=pc+p\lambda\int_\Omega F(x,u_n)dx+o(1).
\end{equation}
In accordance with \eqref{b8} and \eqref{b14}, we obtain
\begin{equation}\label{b15}
\int_\Omega F(x,u_n)dx\to +\infty.
\end{equation}
Next, we claim that $|\Omega_0|=0$.
In fact, if $|\Omega_0|\neq 0$, then by using \eqref{b10}, \eqref{b14} and the
Fatou's lemma, we have
\begin{equation}\label{b16}
\begin{aligned}
\quad \quad +\infty
&=\int_{\Omega_0}\liminf_{n\to +\infty}\frac{F(x,u_n)}{|u_n|^p}|w_n|^pdx-\int_{\Omega_0}\limsup_{n\to +\infty}\frac{C}{\|u_n\|^p}\\
&=\int_{\Omega_0}\liminf_{n\to +\infty}
\Big(\frac{F(x,u_n)}{|u_n|^p}|w_n|^p -\frac{C}{\|u_n\|^p}\Big)dx\\
&\leq \liminf_{n\to +\infty}\int_{\Omega_0}
\Big(\frac{F(x,u_n)}{|u_n|^p}|w_n|^p-\frac{C}{\|u_n\|^p}\Big)dx\\
&\leq \liminf_{n\to +\infty}\int_\Omega
\Big(\frac{F(x,u_n)}{|u_n|^p}|w_n|^p-\frac{C}{\|u_n\|^p}\Big)dx\\
&=\liminf_{n\to +\infty}\int_\Omega\frac{F(x,u_n)}{|u_n|^p}|w_n|^pdx
-\limsup_{n\to +\infty}\int_\Omega\frac{C}{\|u_n\|^p}dx\\
&=\liminf_{n\to +\infty}\int_\Omega\frac{F(x,u_n)}{|u_n|^p}|w_n|^pdx
-\limsup_{n\to +\infty}\frac{C|\Omega|}{\|u_n\|^p}\\
&=\liminf_{n\to +\infty}\int_\Omega\frac{F(x,u_n)}{|u_n|^p}|w_n|^pdx\\
&=\liminf_{n\to +\infty}\frac{\int_\Omega F(x,u_n)dx}{pc+p\lambda \int F(x,u_n)dx+o(1)}.
\end{aligned}
\end{equation}
Therefore, it follows from \eqref{b15} and \eqref{b16} that
$+\infty\leq\frac{1}{p\lambda}$.
This is a contradiction, which implies that $|\Omega_0|=0$. Hence we obtain that
$w(x)=0$ a.e.\ in $\Omega$.
From \eqref{b8}, we have
$$
I_{\lambda}(u_n)=\frac{1}{p}\|u\|^p-\lambda\int_\Omega F(x,u_n)dx\to c.
$$
Then
$$
\frac{I_{\lambda}(u_n)}{\|u_n\|^p}
=\frac{1}{p}-\lambda\int_{\Omega}\frac{F(x,u_n)}{|u_n|^p}|w_n|^pdx,
$$
that is,
$$
\int_{\Omega}\frac{F(x,u_n)}{|u_n|^p}|w_n|^pdx\to\frac{1}{p\lambda},
$$
Again by \eqref{b8}, we have
$$
\langle I_{\lambda}'(u_n),u_n\rangle=\|u_n\|^p-\lambda\int_{\Omega}f(x,u_n)u_ndx=o(1),
$$
where $o(1)\to 0$, as $n\to\infty$. Then
%\label{eq}
\begin{align*}
1-\lambda\int_\Omega\frac{u_nf(x,u_n)}{|u_n|^p}|w_n|^pdx
=\frac{\langle I_{\lambda}'(u_n),u_n\rangle}{\|u_n\|^p}
\leq \frac{\|I_{\lambda}'(u_n)\|\cdot\|u_n\|}{\|u_n\|^p}
=\frac{\|I_{\lambda}'(u_n)\|}{\|u_n\|^{p-1}}\to 0,
\end{align*}
that is,
\[
\int_\Omega\frac{u_nf(x,u_n)}{|u_n|^p}|w_n|^pdx\to \frac{1}{\lambda}.
\]
Therefore,
\[
\quad \quad \int_\Omega\frac{\mu F(x,u_n)-u_nf(x,u_n)}{|u_n|^p}|w_n|^pdx\to \frac{\mu}{p\lambda}-\frac{1}{\lambda}.
\]
However, the hypothesis (H4) implies
\[
\quad \quad \limsup\frac{\mu F(x,u_n)-u_nf(x,u_n)}{|u_n|^p}|w_n|^p\leq\limsup C\frac{|u_n|^p+1}{|u_n|^p}|w_n|^p=0.
\]
Therefore,
$$
\frac{\mu}{p\lambda}-\frac{1}{\lambda}\leq 0,
$$
which leads to a contradiction. Hence $\{u_n\}$ is bounded, i.e., there is a $C>0$ such that
$\|u_n\|\leq C<+\infty$.
\end{proof}
\begin{lemma}\label{lem2.14}
Assume~$f$ satisfies {\rm (H2)}. Then the $(PS)_c$ sequence
$\{u_n\}\subset W_0^{1,p}(\Omega)$ for the functional $I_{\lambda}$ defined in
\eqref{b2} has a convergent subsequence.
\end{lemma}
\begin{proof}
Let $\{u_n\}\subset W_0^{1,p}(\Omega)$ be a $(PS)_c$ sequence for the functional
$I_{\lambda}$. Using Lemma \ref{lem2.13}, we deduce that $\{u_n\}$ is bounded.
Therefore, there exists a $u\in W_0^{1,p}(\Omega)$ such that
\begin{equation}\label{b17}
u_n\rightharpoonup u\quad \text{in}~ W_0^{1,p}(\Omega).
\end{equation}
Furthermore, the Sobolev's compact imbedding implies
$ u_n\to u\quad \text{in}~L^q(\Omega)$.
Denote $\epsilon_n=\|I_{\lambda}'(u_n)\|_{*}$. It is easy to check that $\epsilon_n\to 0$
and
\begin{equation}\label{b18}
|\langle I_{\lambda}'(u_n),v\rangle|
=\Big|\int_\Omega|\nabla u_n|^{p-2}\nabla u_n\nabla v\,dx
-\lambda\int_{\Omega}f(x,u_n)v\,dx\Big| \leq \epsilon_n\|v\|,
\end{equation}
for any $v\in W_0^{1,p}(\Omega)$. Thanks to (H2), we have
\begin{equation}\label{b19}
\int_\Omega(f(x,u_n)-f(x,u))(u_n-u)dx\to 0.
\end{equation}
In fact, it follows from the H\"older inequality that
\begin{align*}
&\big|\int_\Omega(f(x,u_n)-f(x,u))(u_n-u)dx\big|\\
&\leq \Big(\int_\Omega |f(x,u_n)-f(x,u)|^pdx\Big)^{1/p}
\Big(\int_\Omega |u_n-u|^qdx\Big)^{1/q}.
\end{align*}
Since $\|u_n\|\leq C$ (see Lemma \ref{lem2.13}) and $f$ is continuous on
$\overline{\Omega}\times[-C,C]$, there is a $M>0$ such that
$$
|f(x,u_n)|\leq M\quad\text{for all } (x,u_n)\in \overline{\Omega}\times[-C,C].
$$
Therefore,
\begin{gather*}
\Big(\int_\Omega |f(x,u_n)-f(x,u)|^pdx\Big)^{1/p}
\leq\Big(\int_\Omega(2M)^pdx\Big)^{1/p} =2M|\Omega|^{1/p},\\
\Big(\int_\Omega|u_n-u|^qdx\Big)^{1/q}\to 0,
\end{gather*}
since $u_n\to u$ in $L^q(\Omega)$. Hence
$$
\int_\Omega(f(x,u_n)-f(x,u))(u_n-u)dx\leq 2M|\Omega|^{1/p}
\Big(\int_\Omega|u_n-u|^qdx\Big)^{1/q}\to 0,
$$
as $n\to +\infty$. Taking $v=u_n-u$ in \eqref{b18}, and it follows
from \eqref{b19} that
\begin{align*}
\langle I'(u_n),u_n-u\rangle
&=\int_\Omega |\nabla u_n|^{p-2}\nabla u_n\nabla(u_n-u)dx\\
&=\langle I'(u_n),u_n-u\rangle+\int_\Omega f(x,u_n)(u_n-u)dx\\
&\leq\epsilon_n\|u_n-u\|+\int_\Omega f(x,u_n)(u_n-u)dx
\to 0.
\end{align*}
By using the $(S_{+})$ property of $I_{\lambda}'$, we conclude that
$u_n\to u$ in $W_0^{1,p}(\Omega)$.
\end{proof}
\begin{proof}[roof of Theorem \ref{thm1.1}]
Firstly, in view of Lemmas \ref{lem2.11}, \ref{lem2.12} and the Mountain pass lemma
(Theorem \ref{thm2.10}), there is a $(PS)_c$ sequence $\{u_n\}\subset W_0^{1,p}(\Omega)$
that satisfies
$I(u_n)\to c$ and $I'(u_n)\to 0$.
Secondly, in accordance to Lemma \ref{lem2.14}, we deduce that $\{u_n\}$ converges
strongly to some function $u\in W_0^{1,P}(\Omega)$.
Clearly, $u$ is a weak solution for the problem \eqref{a1}.
This completes the proof.
\end{proof}
\section{Existence of the nontrivial weak solution for the superlinear elliptic system}
In this section, we establish the existence of the nontrivial solution for the
supercritical superlinear (i.e., $p\in(2,2^*)$, $q\in(2^*,+\infty)$) elliptic system
\eqref{a2} without the AR condition.
\subsection{Preliminaries}
The key point is to show the boundedness of the $(PS)_c^*$ sequence of the energy functional.
We denote by $|\cdot|_t$ the usual $L^t(\Omega)$ norm for all $t\in[1,\infty]$.
For $q>2^*$, let $V_q=H_0^1(\Omega)\cap L^q(\Omega)$ and the Banach space $V_q$ equipped
with the norm $\|v\|_{V_q}=(|\nabla v|_2^2+|v|_q^2)^{\frac{1}{2}}$.
Let $E_q$ be the product space $H_0^1(\Omega)\times V_q$ with elements denoted by
$z=(u,v)$ and the norm in $E_q$ by $\|z\|_q=(|\nabla u|_2^2+\|v\|_{V_q}^2)^{\frac{1}{2}}$.
We also denote $|z|=|u|+|v|$. $E_q$ has the direct sum decomposition
$$
E_q=E_q^-\oplus E^+,\quad z=z^-+z^+,
$$
where $E_q^-={\{0\}}\times V_q$ and $E^+=H_0^1(\Omega)\times {\{0\}}$.
For simplicity, write $z^+=u$, $z^-=v$.
If $\Omega\subset \mathbb{R}^n$ is a smooth bounded domain and~$H$ satisfies $(H5)$,
then we define the functional on $E_q$ as
\begin{equation}\label{c1}
I(z):=\frac{1}{2}\int_\Omega\left(|\nabla u|^2-|\nabla v|^2\right)dx
-\int_\Omega H(x,z)dx.
\end{equation}
By a straightforward computation, we obtain that $I$ is a $C^1$ functional, and
\begin{equation}\label{c2}
\langle I'(z),w\rangle=\int_\Omega\nabla u\nabla\varphi dx
-\int_\Omega H_u(x,z)\varphi dx+\int_\Omega H_v(x,z)\psi dx.
\end{equation}
It is not difficult to verify that the critical point of $I$ is the solution of the
elliptic system \eqref{a2}.
Next, we show that the Frechet derivative of the functional $I$ is weakly sequence
continuous
.
\begin{lemma}\label{lem3.1}
Assume {\rm (H5)} holds. Then $I'$ is weakly sequence continuous, that is,
$I'(z_n)\rightharpoonup I'(z)$,
as $z_n\rightharpoonup z$.
\end{lemma}
\begin{proof}
Suppose $z_n\rightharpoonup z$ in $E_q$. We claim that $I'(z_n)\rightharpoonup I'(z)$,
that is,
$$
\langle I'(z_n),w\rangle\to\langle I'(z),w\rangle,
$$
for any $w=(\varphi,\psi)\in E_q$.
Since $z_n\rightharpoonup z$, we have $u_n\rightharpoonup u$ in $H_0^1$, and
$v_n\rightharpoonup v$ in $V_q$. Thus $(u_n,\varphi)\to(u,\varphi)$, that is,
$$
\int_{\Omega}\nabla u_n\nabla\varphi dx\to\int_{\Omega}\nabla u\nabla\varphi dx.
$$
Similarly, we have
$$
\int_{\Omega}\nabla v_n\nabla\psi dx\to\int_{\Omega}\nabla v\nabla\psi dx.
$$
Therefore,
$$
\int_{\Omega}\left(\nabla u_n\nabla\varphi-\nabla v_n\nabla\psi\right)dx\to\int_{\Omega}\left(\nabla u\nabla\varphi-\nabla v\nabla\psi\right)dx.
$$
Next, we verify the following two equalities
\begin{gather}\label{c3}
\lim_{n\to\infty}\int_{\Omega}H_u(x,z_n)\varphi dx
=\int_{\Omega}H_u(x,z)\varphi dx,\quad \text{for any }\varphi\in H_0^1(\Omega),\\
\label{c4}
\lim_{n\to\infty}\int_{\Omega}H_v(x,z_n)\psi dx
=\int_{\Omega}H_v(x,z)\psi dx,\quad \text{for any }\psi\in V_q.
\end{gather}
It follows from the Sobolev's compact imbedding theorem and the Interpolation
theorem that
\begin{gather*}
u_n\to u\quad \text{in $L^t$ for any $t\in[1,2^*)$}, \\
v_n\to v\quad \text{in $L^t$ for any $t\in[1,q)$}.
\end{gather*}
By $(H5)$, we have
$$
|H_u(x,z_n)\varphi|\leq\gamma_0\left(|\varphi|+|u_n|^{p-1}|\varphi|+|v_n|^{\frac{q}{2}-1}
|\varphi|\right)
$$
and
\begin{align*}
&\int_{\Omega}\Big(|\varphi|+|u_n|^{p-1}|\varphi|+|v_n|^{\frac{q}{2}-1}|\varphi|\Big)dx\\
&\leq \int_{\Omega}|\varphi|dx+\Big(\int_{\Omega}|u_n|^{(p-1)\frac{p}{p-1}}dx
\Big)^{\frac{p-1}{p}}
\Big(\int_{\Omega}|\varphi|^pdx\Big)^{1/p} \\
&\quad +\Big(\int_{\Omega}|v_n|^{(\frac{q}{2}-1)\cdot2_{*}}dx\Big)^{1/2_*}
\Big(\int_{\Omega}|\varphi|^{2^{*}}dx\Big)^{1/2^*} \\
&=|\varphi|_1+|u_n|_p^{p-1}|\varphi|_p+|v_n|_{2_{*}(\frac{q}{2}-1)}^{\frac{q}{2}-1}
|\varphi|_{2^{*}}.
\end{align*}
Thanks to
$\varphi\in H_0^1(\Omega)\hookrightarrow L^{2^{*}}$,
and
$$
\big(\frac{q}{2}-1\big)2_{*}
=\big(\frac{q}{2}-1\big)\frac{2^{*}}{2^{*}-1}<\big(\frac{q}{2}-1\big)2\alpha$ such that
$$
\sup_{x\in Q_0}I(x)\leq\alpha,\quad\inf_{x\in S}I(x)\geq\beta.
$$
\end{itemize}
Then there is a sequence $\{x_n\}\subset E$ such that
$$
I(x_n)\to c,\quad I_n'(x_n)\to 0,
$$
where $c$ is defined in \eqref{c5}.
\end{theorem}
\subsection{Existence of a nontrivial weak solution for the elliptic system}
Now we set $E^{'}=E_q^{-},~E^{2}=E^{+}$ and $e_n^1=e_n^-,~e_n^2=e_n^+$
for all $n\in N$, and therefore, $E_q=E^1\oplus E^2$.
We will show that the functional $I$ defined in \eqref{c1} satisfies the linking
geometry.
\begin{lemma}\label{lem3.5}
Suppose $H$ satisfies {\rm (H5)} and {\rm (H9)}.
Then there exist constants $r$ and $\rho>0$ such that
$$
\inf I(\partial B_r^{+})\geq\rho,
$$
where $B_r^{+}=B_r(0)\cap E^{+}$.
\end{lemma}
\begin{proof}
Recalling (H5) and (H9), for any $\epsilon>0$, there is $C_{\epsilon}>0$ such that
$$
H(x,u,0)\leq\epsilon|u|^2+C_{\epsilon}|u|^{2^{*}}.
$$
In fact, it follows from $(H5)$ that
\begin{equation}\label{c6}
|H_u(x,u,0)|\leq\gamma_0\left(1+|u|^{p-1}\right).
\end{equation}
Furthermore, in the light of (H9), we have $H_u(x,u,0)=o(|u|)$ as $u\to 0$.
Then for any $\epsilon>0$, there is a constant $\overline{c}>0$ such that
$$
|H_u(x,u,0)|\leq\epsilon|u|\quad\text{whenever } |u|<\overline{c}.
$$
And there exists $C>0$ such that
$$
|H_u(x,u,0)|\leq\gamma_0\left(1+|u|^{p-1}\right)
\leq C|u|^{p-1}\quad\text{whenever } |u|>\overline{c}.
$$
To sum up, we have
$$
|H_u(x,u,0)|\leq\epsilon|u|+C_{\epsilon}|u|^{p-1}.
$$
That is,
$$
|H(x,u,0)|\leq\epsilon|u|^2+C_{\epsilon}|u|^p<\epsilon|u|^2+C_{\epsilon}|u|^{2^{*}}.
$$
Therefore,
$$
I(u):=\frac{1}{2}\int_{\Omega}|\nabla u|^2dx-\int_{\Omega}H(x,u,0)dx
\geq\frac{1}{2}\int_{\Omega}|\nabla u|^2dx-\epsilon|u|_2^2-C_{\epsilon}|u|_{2^{*}}^{2^{*}}.
$$
Then we obtain the conclusion.
\end{proof}
Assume $e\in E^{+}$ with $|\nabla e|_2^2=1$, and let
$$
Q=\{(se,v):0\leq s\leq r_1,~\|v\|_q\leq r_2\}.
$$
\begin{lemma}\label{lem3.6}
Suppose~$H$ satisfies {\rm (H8)} and {\rm (H9)}.
Then there are constants $r_1,r_2>0$ with $r_1>r$ such that
$$
I(z)\leq 0\quad\text{for all } z\in\partial Q.
$$
\end{lemma}
\begin{proof}
In view of (H9) and $H(x,0,v)\geq 0$, we have
$$
I(z):=-\frac{1}{2}\int_{\Omega}|\nabla v|^2dx-\int_{\Omega}H(x,0,v)dx\leq0,
$$
when $z\in E_q^-$.
By (H8), we obtain
\begin{align*}
I((se,v))
&=\frac{s^2}{2}\int_{\Omega}|\nabla e|^2dx-\frac{1}{2}\int_{\Omega}|\nabla v|^2dx
-\int_{\Omega}H(x,se,v)dx\\
&\leq\frac{s^2}{2}-\frac{1}{2}|\nabla v|_2^2
-\int_{\Omega}\left(\gamma_1(|se|^p+|v|^q)-\gamma_2\right)dx\\
&\leq\frac{s^2}{2}-\frac{1}{2}|\nabla v|_2^2
-c_1\int_{\Omega}\left(|se|^p+|v|^q\right)dx+c_2.\\
\end{align*}
Since $p>2$, we obtain the conclusion.
\end{proof}
Next, we establish the boundedness of the $(PS)_c^{*}$ sequence, which plays
an important role in the existence theory of the nontrivial weak solution.
\begin{lemma}\label{lem3.7}
Assume $H$ satisfies {\rm (H6)} and {\rm (H7)}. Then the $(PS)_c^{*}$ sequence
$\{z_n\}\subset E_q$ is bounded, where $z_n=(u_n,v_n)$.
\end{lemma}
\begin{proof}
Without loss of generality, suppose $\|z_n\|_q\to+\infty$.
By
$$
\|z_n\|_q^2=|\nabla u_n|_2^2+|\nabla v_n|_2^2+|v_n|_q^2,
$$
we assume that
$$
|\nabla u_n|_2\to+\infty,\quad\frac{|\nabla v_n|}{|\nabla u_n|_2}\to a<1.
$$
Setting $Y_n=\frac{z_n}{\|z_n\|_q}$, then $Y_n\in E_q$ with $\|Y_n\|_q=1$.
Therefore, there is $Y\in E_q$ such that $Y_n\rightharpoonup Y$ in $E_q$.
Then we have
$Y_n(x)\to Y(x)$ a.e.\ in $\Omega$.
Denote
$$
\Omega_0=\{x\in \Omega,\ Y(x)\neq 0\}.
$$
Then we have
\begin{equation}\label{c7}
\lim_{n\to+\infty}\frac{z_n}{\|z_n\|_q}=\lim_{n\to+\infty}Y_n=Y\neq 0\quad
\text{a.e.\ in } \Omega_0,
\end{equation}
which implies
$|z_n|\to+\infty$ a.e.\ in $\Omega_0$.
It follows from (H6) that
$$
\lim_{n\to+\infty}\frac{H(x,z_n)}{|z_n|^2}|Y_n|^2=+\infty\quad
\text{a.e. in }\Omega_0.
$$
Again by using (H6), there is $N_0>0$ such that, for any $x\in\Omega$, we have
\begin{equation}\label{c8}
\frac{H(x,z_n)}{|z_n|^2}>1\quad\text{whenever } |z_n|\geq N_0.
\end{equation}
Since $H$ is continuous on $\overline{\Omega}\times[-N_0,N_0]\times[-N_0,N_0]$,
there exists an $M>0$ such that
\begin{equation}\label{c9}
|H(x,z_n)|\leq M,
\end{equation}
for any $(x,z_n)\in\overline{\Omega}\times[-N_0,N_0]\times[-N_0,N_0]$.
From \eqref{c8} and \eqref{c9}, we deduce that there exists constant $C$ such that
$$
H(x,z_n)\geq C\quad\text{for all } (x,z_n)\in \overline{\Omega}\times \mathbb{R}
\times \mathbb{R}.
$$
This implies that
$$
\frac{H(x,z_n)-C}{\|z_n\|_q^2}\geq 0.
$$
Since
$$
I(z_n)=\frac{1}{2}\int_{\Omega}\left(|\nabla u_n|^2-|\nabla v_n|^2\right)dx
-\int_{\Omega}H(x,z_n)dx=c+o(1),
$$
we have
$$
\int_{\Omega}\left(|\nabla u_n|^2-|\nabla v_n|^2\right)dx
=2c+2\int_{\Omega}H(x,z_n)dx+o(1).
$$
In view of $|\nabla u_n|_2\to+\infty$, we have
$\int_{\Omega}H(x,z_n)dx\to+\infty$.
Therefore,
\begin{align*}
+\infty&=\int_{\Omega_0}\liminf_{n\to{+\infty}}\frac{H(x,z_n)}{|z_n|^2}|Y_n|^2dx
-\int_{\Omega_0}\limsup_{n\to{+\infty}}\frac{C}{\|z_n\|_q^2}dx\\
&=\int_{\Omega_0}\liminf_{n\to{+\infty}}
\Big(\frac{H(x,z_n)}{|z_n|^2}|Y_n|^2-\frac{C}{\|z_n\|_q^2}\Big)dx\\
&\leq\liminf_{n\to{+\infty}}\int_{\Omega_0}
\Big(\frac{H(x,z_n)}{|z_n|^2}|Y_n|^2-\frac{C}{\|z_n\|_q^2}\Big)dx\\
&\leq\liminf_{n\to{+\infty}}\int_{\Omega}
\Big(\frac{H(x,z_n)}{|z_n|^2}|Y_n|^2-\frac{C}{\|z_n\|_q^2}\Big)dx\\
&=\liminf_{n\to{+\infty}}\int_{\Omega}\frac{H(x,z_n)}{\|z_n\|_q^2}dx-\limsup_{n\to{+\infty}}
\int_{\Omega}\frac{C}{\|z_n\|_q^2}dx\\
&=\liminf_{n\to{+\infty}}\int_{\Omega}\frac{H(x,z_n)}{\|z_n\|_q^2}dx-\limsup_{n\to{+\infty}}
\frac{C|\Omega|}{\|z_n\|_q^2}\\
&=\liminf_{n\to{+\infty}}\int_{\Omega}\frac{H(x,z_n)}{\|z_n\|_q^2}dx\\
&=\liminf_{n\to{+\infty}}\frac{\int_{\Omega}H(x,z_n)dx}{|\nabla u|_2^2
+|\nabla v|_2^2+|v|_q^2}\\
&=\liminf_{n\to{+\infty}}\frac{\int_{\Omega}H(x,z_n)dx}{2c+2\int_{\Omega}H(x,z_n)dx
+2|\nabla v|_2^2+|v|_q^2+o(1)}
=\frac{1}{2},
\end{align*}
which leads to a contradiction.
Then $|\Omega_0|=0,$ and therefore,
$Y(x)=0$ a.e.\ in $\Omega_0$.
Since
$$
I(z_n)=\frac{1}{2}\int_{\Omega}\left(|\nabla u_n|^2-|\nabla v_n|^2\right)dx
-\int_{\Omega}H(x,z_n)dx,
$$
we have
$$
\frac{I(z_n)}{\|z_n\|_q^2}=\frac{1}{2}-\int_{\Omega}\frac{H(x,z_n)}{|z_n|^2}|Y_n|^2dx,
$$
that is,
$$
\int_{\Omega}\frac{H(x,z_n)}{|z_n|^2}|Y_n|^2dx\to\frac{1}{2}.
$$
Moreover, thanks to
$$
\langle I'(z_n),z_n\rangle=\int_{\Omega}|\nabla u_n|dx
-\int_{\Omega}|\nabla v_n|^2dx-\int_{\Omega}H_z(x,z_n)z_n\,dx.
$$
we have
$$
1-\int_{\Omega}\frac{H_z(x,z_n)z_n}{\|z_n\|_q^2}dx
=\frac{\langle I'(z_n),z_n\rangle}{\|z_n\|_q^2}\
leq\frac{\|I'(z_n)\|\cdot\|z_n\|_q}{\|z_n\|_q^2}=\frac{\|I'(z_n)\|}{\|z_n\|_q^2}\to 0,
$$
that is,
$$
\int_\Omega\frac{z_nH_z(x,z_n)}{|z_n|} |Y_n|^2dx\to 1.
$$
Therefore,
$$
\int_{\Omega}\frac{\mu H(x,z_n)-z_nH_z(x,z_n)}{|z_n|^2}|Y_n|dx\to \frac{\mu}{2}-1.
$$
Then it follows from (H7) that
$$
\limsup\frac{\mu H(x,z_n)-z_nH_z(x,z_n)}{|z_n|^2} |Y_n|^2
\leq\limsup C\frac{|z_n|^2+1}{|z_n|^2} |Y_n|^2=0;
$$
that is,
$\frac{\mu}{2}-1\leq 0$.
Hence, $\mu\leq 2$. This is a contradiction.
Therefore, $\{z_n\}$ is bounded in $E_q$.
\end{proof}
\begin{lemma}\label{lem3.8}
Let $\{z_n\}\subset X_n$ be a $(PS)_c^{*}$ sequence. Then there exists $z\in E_q$
such that along a subsequence, $z_n\rightharpoonup z$ with $I'(z)=0$ and $I(z)\geq c$.
\end{lemma}
\begin{proof}
Since $\{z_n\}\subset X_n$ is a $(PS)_c^{*}$ subsequence, it follows from
Lemma \ref{lem3.7} that the $(PS)_c^*$ sequence is bounded, that is, $\{z_n\}$
is bounded. Thus, $\{z_n\}$ has weakly convergent subsequence, might as well suppose
$z_n\rightharpoonup z$ in $E_q$. Then for any $1\leq s<2^{*}$, the imbedding theorem
implies that
$z_n\to z$ in $(L^s(\Omega))^2$.
As a result,
$$
z_n(x)\to z(x)\quad \text{a.e. in }\Omega.
$$
Moreover, by using Lemma \ref{lem3.1}, we know $I'$ is weakly sequence continuous.
Hence, we obtain
$I'(z)=0$.
Let $w=(\varphi,\psi)=(u_n-u,0)$ in \eqref{c2} and by $I_n'(z_n)\to 0$, we have
\begin{align*}
(\nabla u_n,\nabla u_n-\nabla u)_{L^2}
&=I_n'(z_n)(u_n-u,0)+\int_{\Omega}H_u(x,z_n)(u_n-u)dx\\
&=o(1)+\int_{\Omega}H_u(x,z_n)(u_n-u)dx.
\end{align*}
By using (H5), H\"older's inequality and
$\frac{2q}{q+2}<2<2^{*}$,
we have
\begin{align*}
&\big|\int_{\Omega}H_u(x,z_n)(u_n-u)dx\big|\\
&\leq\int_{\Omega}\gamma_0\left(1+|u_n|^{p-1}+|v_n|^{\frac{q}{2}-1}\right)|u_n-u|dx\\
&\leq\gamma_0\left(|u_n-u|_1+|u_n|_p^{p-1}|u_n-u|_p+|v_n|_q^{\frac{q}{2}-1}
|u_n-u|_{\frac{2q}{q+2}}\right)
=o(1).
\end{align*}
Therefore,
$$
\left(\nabla u_n,\nabla u_n-\nabla u\right)_{L^2}=o(1),
$$
that is,
$$
|\nabla u_n|_2^2\to|\nabla u|_2^2.
$$
Then
$u_n\to u\quad\text{in}~H_0^1(\Omega)$.
Let $p_n:E_q\to X_n$ denote the projection. Observe that $P_nz\to z$ in $E_q$ for all
$z\in E_q$. Moreover, using again $(H5)$ and H\"older's inequality, we deduce
\begin{align*}
&\big|\int_{\Omega}H_v(x,z_n)(v-P_nv)dx\big| \\
&\leq C\left(|v-P_nv|_1+|u_n|_p^{p-1}|v-P_nv|_p+|v_n|_q^{q-1}|v-P_nv|_q\right)
\to 0.
\end{align*}
On the other hand, lettin $w=(\varphi,\psi)=(0,v_n-P_nv)$ in \eqref{c2}, and by
$I_n'(z_n)\to 0$, we obtain
\begin{align*}
&I_n'(z_n)(0,v_n-P_nv)\\
&=-\int_{\Omega}\nabla v_n\nabla(v_n-P_nv)dx+\int_{\Omega}H_v(x,z_n)(v_n-P_nv)dx\\
&=-\int_{\Omega}\nabla v_n\nabla(v_n-v+v-P_nv)dx+\int_{\Omega}H_v(x,z_n)(v_n-v+v-P_nv)dx\\
&=-(\nabla v_n \nabla v_n-\nabla v)_{L^2}
-\int_{\Omega}\nabla v_n\nabla(v-p_n)dx
+\int_{\Omega}H_v(x,z_n)(v_n-v)dx \\
&\quad +\int_{\Omega}H_v(x,z_n)(v-P_nv)dx\\
&=-(\nabla v_n \nabla v_n-\nabla v)_{L^2}+\int_{\Omega}H_v(x,z_n)(v_n-v)dx+o(1).\\
\end{align*}
Then
\begin{align*}
(\nabla v_n \nabla v_n-\nabla v)_{L^2}
&=\int_{\Omega}H_v(x,z_n)(v_n-v)dx+o(1)\\
&=\int_{\Omega}H_z(x,z_n)(z_n-z)dx+\int_{\Omega}H_u(x,z_n)(u_n-u)dx+o(1)\\
&=\int_{\Omega}H_z(x,z_n)z_ndx-\int_{\Omega}H_z(x,z_n)zdx+o(1).\\
\end{align*}
It follows from the Lebesgue's theorem and the weak sequential continuity of $H_z$ that
\begin{align*}
|\nabla v|_2^2-\limsup_{n\to\infty}|\nabla v_n|_2^2
&=\liminf_{n\to\infty}\Big(\int_{\Omega}H_z(x,z_n)z_ndx
-\int_{\Omega}H_z(x,z_n)zdx\Big)\\
&\geq\int_{\Omega}\liminf_{n\to\infty}\left(H_z(x,z_n)z_n-H_z(x,z_n)z\right)dx
=0,
\end{align*}
that is,
$$
|\nabla v|_2^2\geq\limsup_{n\to\infty}|\nabla v_n|_2^2,
$$
which together with the weak lower semicontinuity of the norm implies that
$$
|\nabla v|_2\leq\limsup_{n\to\infty}|\nabla v_n|_2.
$$
So
$|\nabla v_n|_2\to|\nabla v|_2$,
that is, $v_n\to v$ in $H_0^1(\Omega)$.
Observe that
\begin{align*}
I(z)-I(z_n)&=\frac{1}{2}\left(|\nabla u|_2^2-|\nabla u_n|_2^2\right)
-\frac{1}{2}\left(|\nabla v|_2^2-|\nabla v_n|_2^2\right) \\
&\quad +\int_{\Omega}H(x,z_n)dx-\int_{\Omega}H(x,z)dx.
\end{align*}
The Lebesgue's theorem then yields
\begin{align*}
I(z)-C
&=\liminf_{n\to\infty}\int_{\Omega}H(x,z_n)dx-\int_{\Omega}H(x,z)dx\\
&\geq\int_{\Omega}\liminf_{n\to\infty}H(x,z_n)dx-\int_{\Omega}H(x,z)dx
=0.
\end{align*}
Then we have
$I(z)\geq C$.
\end{proof}
\begin{proof}[Proof of Theorem \ref{thm1.2}]
From the above discussion, it follows from Lemmas \ref{lem3.5} and \ref{lem3.6} that
$I$ has the linking geometry. Let $Q_n:=Q\cap X_n$ and define
$$
c_n:=\inf_{\gamma\in\Gamma_n}\sup_{x\in\gamma(Q_n)}I(x),
$$
where $\Gamma_n:=\{\gamma\in C(Q_n,X_n):\gamma\mid_{\partial Q_n}=id\}$. Then
$$
\rho\leq c_n\leq k:=\sup I(\gamma(Q)).
$$
Therefore, by the Linking theorem, there is $z_n\in X_n$ such that
$$
|I(z_n)-c_n|\leq\frac{1}{n}\quad\text{and}\quad \|I_n'(z_n)\|\leq\frac{1}{n}.
$$
So we obtain a $(PS)_c^{*}$ sequence $\{z_n\}\subset E_q$ with $c\in[\rho,k]$.
Lemma \ref{lem3.8} implies $z_n\rightharpoonup z$ with $I'(z)=0$ and $I(z)\geq c$.
As a result, the Theorem \ref{thm1.2} is obtained.
\end{proof}
\subsection*{Acknowledgements}
This research was supported by the National Science Foundation of China (No.11471147).
The authors would like to thank the referees for helpful suggestions.
\begin{thebibliography}{00}
\bibitem{AF} R. Adams, J. Fournier;
\emph{Sobolev Spaces}, Academic Press, New York (2003).
\bibitem{AR} A. Ambrosetti, P. H. Rabinowitz;
\emph{Dual variational methods in critical point theory and applications},
Journal of Functional Analysis, \textbf{14}, No. 4(1973), 349-381.
\bibitem{B} H. Brezis;
\emph{Functional Analysis, Sobolev Spaces and Partial Differential Equations},
Springer, New York (2011).
\bibitem{CGM} P. Cl ment, M. Garc a-Huidobro, Man sevich;
\emph{Mountain pass type solutions for quasilinear elliptic equations},
Calculus of Variations and Partial Differential Equations, \textbf{11}, No. 1(2000), 33-62.
\bibitem{CGS} M. L. M. Carvalho, J. V. A. Goncalves, E. D. D. Silva;
\emph{On quasilinear elliptic problems without the Ambrosetti-Rabinowitz condition},
Journal of Mathematical Analysis and Applications, \textbf{426}, No. 1(2015), 466-483.
\bibitem{CM} D. G. Costa, C. A. Magalhaes;
\emph{A variational approach to noncooperative elliptic systems},
Nonlinear Analysis Theory Methods and Applications, \textbf{25}, No. 7(1995), 699-715.
\bibitem{DD} D. G. De Figueiredo, Y. H. Ding;
\emph{Strongly indefinite functionals and multiple solutions of elliptic systems},
Transactions of the American Mathematical Society, \textbf {355}, No. 7(2003), 2973-2989.
\bibitem{DF} D. G. De Figueiredo, P. L. Felmer;
\emph{On superquadratic elliptic systems},
Transactions of the American Mathematical Society, \textbf {343}, No. 1(1994), 99-116.
\bibitem{DJM} G. Dinca, P. Jebelean, J. Mawhin;
\emph{A result of Ambrosetti-Rabinowitz type for p-Laplancian},
Qualitative problems for differential equations and control theory,
World Sci. Publ., River Edge, NJ, (1995), 231-242.
\bibitem{E} L. C. Evans;
\emph{Partial Differential Equations(second edition)},
University of Califolia, Berkeley, (2010).
\bibitem{GT} D. Gilbarg, N. Trudinge;
\emph{Elliptic Partial Differential Equation of Second Order},
Springer, New York (2001).
\bibitem{H} Z. Hu, L. Wang, P. H. Zhao;
\emph{A priori estimates and existence for quasilinear elliptic equations
with nonlinear Neumann boundary conditions},
Electronic Journal of Differential Equations, \textbf{2016}, No. 187 (2016), 1-8.
\bibitem{L} P. L. Lions;
\emph{On the existence of positive solutions of semilinear elliptic equations},
Siam Review, \textbf {24}, No. 4 (1982), 441-467.
\bibitem{LL} N. Lam, G. Lu;
\emph{Elliptic equations and systems with subcritical and critical exponential growth
without the Ambrosetti-Rabinowitz condition},
Journal of Geometric Analysis, \textbf{24}, No. 1(2014), 118-143.
\bibitem{LW} Z. L. Liu, Z. Q. Wang;
\emph{On the Ambrosetti-Rabinowitz superlinear condition},
Advanced Nonlinear Studies, \textbf{4}, No. 4(2004), 563-574.
\bibitem{LY} G. B. Li, C. Y. Yang;
\emph{The existence of a nontrivial solution to a nonlinear elliptic boundary
value problem of $p$-Laplacian type without the Ambrosetti-Rabinowitz condition},
Nonlinear Analysis, \textbf{72}, (2010), 4602-4613.
\bibitem{MS} O. H. Miyagaki, M. A. S. Souto;
\emph{Superlinear problems without Ambrosetti and Rabinowitz growth condition},
Journal of Differential Equations, \textbf {245}, No. 12(2008), 3628-3638.
\bibitem{NM} P. D. N pol , M. C. Mariani;
\emph{Mountain pass solutions to equations of p-Laplacian type},
Nonlinear Analysis, \textbf{54}, No. 7(2003), 1205-1219.
\bibitem{S} M. Struwe;
\emph{Variational Methods: Applications to Nonlinear Partial Differential Equations
and Hamiltonian Systems(fourth edition)}, Springer, 2012.
\bibitem{SZ} M. Schechter, W. M. Zou;
\emph{Superlinear problem}, Pacific Journal of Mathematics,
\textbf{214}, No. 1(2004), 145-160.
\bibitem{V} J. Vazquez;
\emph{A strong maximum principle for some quasilinear elliptic equations},
Applied Mathematics and Optimization, \textbf{12}, No. 1(1984), 191-202.
\bibitem{WZ}L. Wang, P. H. Zhao;
\emph{Infinitely many sign-changing solutions for concave-convex elliptic problem
with nonlinear boundary condition}, Electronic Journal of Differential Equations,
No. 266(2015), 1-9.
\bibitem{WYW} Z. Q. Wu, J. X. Yin, C. P. Wang;
\emph{Introduction to elliptic and parabolic equations}, Science Press, Beijing (2003).
\bibitem{ZFC} C. K. Zhong, X. L. Fan, W. Y. Cheng;
\emph{Introduction to Nonlinear Functional Analysis}, Lanzhou University Press,
Lanzhou, 2004.
\end{thebibliography}
\end{document}