\documentclass[reqno]{amsart}
\usepackage{hyperref}

\AtBeginDocument{{\noindent\small
\emph{Electronic Journal of Differential Equations},
Vol. 2015 (2015), No. 223, pp. 1--16.\newline
ISSN: 1072-6691. URL: http://ejde.math.txstate.edu or http://ejde.math.unt.edu
\newline ftp ejde.math.txstate.edu}
\thanks{\copyright 2015 Texas State University - San Marcos.}
\vspace{9mm}}

\begin{document}
\title[\hfilneg EJDE-2015/223\hfil Ground state solutions]
{Ground state solutions for non-local fractional Schr\"odinger
equations}

\author[Y. Pu, J. Liu, C.-L. Tang \hfil EJDE-2015/223\hfilneg]
{Yang Pu, Jiu Liu, Chun-Lei Tang}

\address{Yang Pu \newline
School of Mathematics and Statistics,
Southwest University,  Chongqing 400715, China. \newline
School of Mathematics and Information College,
China West Normal University, Nanchong, Sichuan 637002, China}
\email{pt8820@swu.edu.cn}

\address{Jiu Liu \newline
School of Mathematics and Statistics,
Southwest University, Chongqing 400715, China}
\email{jiuliu2011@163.com}

\address{Chun-Lei Tang (corresponding author)\newline
School of Mathematics and Statistics,
Southwest University,  Chongqing 400715, China}
\email{tangcl@swu.edu.cn, Phone +86 23 68253135, fax +86 23 68253135}

\thanks{Submitted June 15, 2015. Published August 28, 2015.}
\subjclass[2010]{35J60, 35J20}
\keywords{Nonlinear Schr\"odinger equation; fractional Laplacian;
\hfill\break\indent non-local operator; ground state solution}

\begin{abstract}
 In this article, we study a time-independent fractional
 Schr\"odinger equation with non-local (regional) diffusion
 $$
 (-\Delta)^{\alpha}_{\rho}u + V(x)u = f(x,u) \quad \text{in }\mathbb{R}^{N},
 $$
 where $\alpha \in (0,1)$, $N > 2\alpha$. We establish the existence of a
 non-negative ground state solution by variational methods.
\end{abstract}

\maketitle
\numberwithin{equation}{section}
\newtheorem{theorem}{Theorem}[section]
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{remark}[theorem]{Remark}
\allowdisplaybreaks


\section{Introduction and statement of main results}

Consider the fractional Schr\"odinger equation
\begin{equation}\label{e1.1}
 \begin{gathered}
   (-\Delta)^{\alpha}_{\rho}u + V(x)u = f(x,u) \quad \text{in } \mathbb{R}^{N},\\
   u\in H^{\alpha}(\mathbb{R}^{N}),
 \end{gathered}
\end{equation}
where $\alpha \in (0,1)$, $N > 2\alpha$, $(-\Delta)^{\alpha}_{\rho}$
denotes a non-local (regional) fractional Laplacian operator with a range
of scope determined by the positive function
$\rho \in C(\mathbb{R}^{N},\mathbb{R}^{+})$.

The fractional Schr\"odinger equation was firstly introduced by Laskin \cite{LA1}.
This equation was of particular interest in fractional quantum mechanics
in the study of particles on stochastic
fields modelled by L\'evy processes that give rise to equations with the
fractional Laplacian operator. Recently, the study on problems of fractional
Schr\"odinger equations has attracted much attention.
Some existence and nonexistence of Dirichlet problem involving
the fractional Laplacian on bounded domains have been established,
see \cite{TA,BA} and their references. Using the equivalent definition
of the fractional operator, some authors introduced a variational
principle and studied the existence and multiplicity of solutions in
$\mathbb{R}^{N}$. Cheng \cite{CHE} considered the  equation
 \[
 (-\Delta)^{\alpha}u + V(x)u = |u|^{p-1}u,\quad u\in H^{\alpha}(\mathbb{R}^{N})
 \]
with unbounded potential $V$, he obtained the existence of ground state
solution by a Lagrange multiplier method and the Nehari manifold method.
Dipierro, Palatucci and Valdinoci in \cite{DI} proved existence and
symmetry results for the solutions.
In \cite{FE}, the authors studied the same equation with a more general
right-hand side $f(x,u)$.  Secchi \cite{SE1} provided a generalization
of the main result to equations of the form
 \begin{equation}\label{2}
 (-\Delta)^{\alpha}u + V(x)u = f(x,u), \quad u\in H^{\alpha}(\mathbb{R}^{N}).
 \end{equation}
The existence of positive solutions was obtained using the Nehari manifold method.
Chang \cite{CHA} proved the existence of a positive ground state solution of
\eqref{2} when $f(x,t)$ is asymptotically linear with respect to $t$ at infinity.

There is another definition of regional fractional
Laplacian in \cite{GU2} and \cite{GU}, the authors introduced the regional
fractional Laplacian operator $\Delta_{G}^{\alpha/2}$ in an arbitrary open
set $G$ of $\mathbb{R}^{N}$, which is not the one used here.
More recently, Felmer and Torres \cite{FE1} studied the regional fractional
Laplacian equation
\[
\varepsilon^{2\alpha}(-\Delta)^{\alpha}_{\rho}u + u = f(u),
\quad u\in H^{\alpha}(\mathbb{R}^{N}),
\]
where $\alpha\in(0,1)$, $N > 2\alpha$. The operator $(-\Delta)^{\alpha}_{\rho}$
was defined by
\[
\int_{\mathbb{R}^{N}}(-\Delta)^{\alpha}_{\rho}u(x)v(x)dx
= \int_{\mathbb{R}^{N}}\int_{B(0,\rho(x))}\frac{[u(x+z)-u(x)][v(x+z)-v(x)]}
{|z|^{N+2\alpha}}\,dz\,dx
\]
for all $u, v\in H^{\alpha}(\mathbb{R}^{N})$.
They showed the existence of a ground state and analyzed the behavior
of the semi-classical solutions as $\varepsilon \to 0$.
Following some ideas in \cite{FE1}, Felmer and Torres \cite{FE2} proved
that the ground state level is achieved by a radially symmetry solution.

Up to now, no results for  non-autonomous regional fractional Laplacian
equations with potential have appeared in the literature.
In this note, we investigate the existence of a non-negative ground state
solution for such equations.

Throughout this article, we assume the following conditions:
\begin{itemize}
\item[(A1)]  $\rho \in C(\mathbb{R}^{N},\mathbb{R}^{+})$, there exists
a constant $\rho_0 > 0$ such that $\rho(x) \geq \rho_0$.

\item[(A2)] $V \in C(\mathbb{R}^{N},\mathbb{R}^{+})$,
$\inf_{x\in \mathbb{R}^{N}}V(x)\geq c >0$, there exists $r_0 > 0$ such that,
for any $M > 0$,
   \[
   \lim_{|y|\to\infty}\operatorname{meas}\big(\{x\in \mathbb{R}^{N}:|x-y|\leq r_0,
 V(x)\leq M\}\big)=0.
   \]

\item[(A3)] $f \in C(\mathbb{R}^{N}\times \mathbb{R}^{+}, R)$ and
$\lim_{t\to 0^{+}}\frac{f(x,t)}{t} = 0$ uniformly in $x\in \mathbb{R}^{N}$.

\item[(A4)] $\lim_{t\to +\infty}\frac{f(x,t)}{t^{2_{\alpha}^{*}-1}} = 0$
uniformly in $x\in \mathbb{R}^{N}$, where
$2_{\alpha}^{*} = \frac{2N}{N-2\alpha}$ is
the fractional critical exponent.

\item[(A5)]  $\lim_{t\to +\infty}\frac{F(x,t)}{t^{2}} = +\infty$
uniformly in $x\in \mathbb{R}^{N}$.

\end{itemize}
The main results of this article are as follows:

\begin{theorem} \label{thm1.1}
Assume  {\rm (A1)--(A5)} and
\begin{itemize}
\item[(A6)] there exists $T_{1} >0$ such that $\mu F(x,t) \leq f(x,t)t$
for $t > T_{1}$, where $\mu > 2$ is a constant.
\end{itemize}
Then \eqref{e1.1} has a non-negative ground state solution.
\end{theorem}

\begin{theorem} \label{thm1.2}
Assume  {\rm (A1)--(A5)}  and
\begin{itemize}
\item[(A7)] $\frac{f(x,t)}{t}$ is increasing on $(0, \infty)$.
\end{itemize}
Then \eqref{e1.1} has a non-negative ground state solution.
\end{theorem}

\begin{theorem} \label{thm1.3}
 Assume that {\rm (A1)--(A5)}  and
\begin{itemize}
\item[(A8)] There exist $b >0$ and $\nu > 2$ such that
$\limsup_{t\to +\infty}\frac{F(x,t)}{t^{\nu}} \leq b$;

\item[(A9)]  $\liminf_{t\to +\infty}\frac{f(x,t)t- 2F(x,t)}{t^{\sigma}}
\geq \eta >0$, where $
\sigma > \frac{N}{2\alpha}(\nu - 2)$.
\end{itemize}
Then \eqref{e1.1} has a non-negative ground state solution.
\end{theorem}

\begin{remark} \label{rmk1.4}\rm
 Note that we impose the subcritical growth condition
by a  general condition
(A4). Also note that (A6), (A7) and (A9) almost cover all
types of superquadratic conditions.
\end{remark}

When $\rho \equiv +\infty$, the regional fractional operator becomes a common
fractional operator, our results are also new under the
assumptions on $f$.
Moreover, we can find a positive ground state solution by the strong maximum
principle \cite{CA}.

\section{Preliminaries}

Firstly we give some basic notation.
The fractional Sobolev space of order $\alpha$ on $\mathbb{R}^{N}$ is defined by
\[
H^{\alpha}(\mathbb{R}^{N}) = \Big\{u\in L^{2}(\mathbb{R}^{N}):
\int_{\mathbb{R}^{N}}\int_{\mathbb{R}^{N}}\frac{|u(x)-u(y)|^{2}}{|x-y|^{N+2\alpha}}
\,dx\,dy < \infty\Big\},
\]
endowed with the norm
\[
\|u\|_{H^{\alpha}(\mathbb{R}^{N})}
= \Big(\int_{\mathbb{R}^{N}}\int_{\mathbb{R}^{N}}
\frac{|u(x)-u(y)|^{2}}{|x-y|^{N+2\alpha}}\,dx\,dy
+ \int_{\mathbb{R}^{N}}|u|^{2}dx\Big)^{1/2}.
\]
In this article, we consider the  space
\begin{align*}
E : = \Big\{& u\in L^{2}(\mathbb{R}^{N}):
\int_{\mathbb{R}^{N}}\int_{B(0,\rho(x))}\frac{|u(x+z)-u(x)|^{2}}{|z|^{N+2\alpha}}
\,dz\,dx \\
&+ \int_{\mathbb{R}^{N}}V(x)|u(x)|^{2}dx < \infty\Big\},
\end{align*}
equipped with the inner product
\begin{align*}
\langle u,v\rangle
&= \int_{\mathbb{R}^{N}}\int_{B(0,\rho(x))}\frac{[u(x+z)-u(x)][v(x+z)-v(x)]}
{|z|^{N+2\alpha}}\,dz\,dx \\
&\quad + \int_{\mathbb{R}^{N}}V(x)u(x)v(x)dx
\end{align*}
and the norm
\[
\|u\| = \Big(\int_{\mathbb{R}^{N}}\int_{B(0,\rho(x))}\frac{|u(x+z)-u(x)|^{2}}{|z|^{N+2\alpha}}\,dz\,dx +
\int_{\mathbb{R}^{N}}V(x)|u(x)|^{2}dx\Big)^{1/2}.
\]
We denote by $\|\cdot\|_{p}$ the usual $L^{p}$-norm. We define
\[
u^{\pm}(x) = \max\{\pm u(x),0\}.
\]
Obviously, $u\in E$ implies that $u^{+}$, $u^{-}\in E$.

Let $I: E \to R$ be the functional defined by
\begin{equation}\label{3}
I(u) = \frac{1}{2}\|u\|^{2} - \int_{\mathbb{R}^{N}}F(x,u^{+})dx
\end{equation} with
$F(x,t)$ being the primitive of $f(x,t)$.

Next, we give some  lemmas that play important roles
in proving our main results.

\begin{lemma} \label{lem2.1}
Suppose {\rm (A1)} and {\rm (A2)} hold. Then there exists a constant $C_0 > 0$
such that
\[
\|u\|_{2_{\alpha}^{*}}^{2}\leq C_0\|u\|^{2}.
\]
\end{lemma}

\begin{proof}
From \cite[Theorem 6.5]{DI2}, there exists $C > 0$ such that
\[
\|u\|_{2_{\alpha}^{*}}^{2}
 \leq C\int_{\mathbb{R}^{N}}
\int_{\mathbb{R}^{N}}\frac{|u(x+z)-u(x)|^{2}}{|z|^{N+2\alpha}}\,dz\,dx.
\]
As  in the proof of \cite[Proposition 2.1]{FE1}, we have
\begin{align*}
&C\int_{\mathbb{R}^{N}}\int_{\mathbb{R}^{N}}\frac{|u(x+z)
 -u(x)|^{2}}{|z|^{N+2\alpha}}\,dz\,dx\\
&=C \Big(\int_{\mathbb{R}^{N}}\int_{B(0,\rho_0)}
 \frac{|u(x+z)-u(x)|^{2}}{|z|^{N+2\alpha}}\,dz\,dx\\
&\quad +\int_{\mathbb{R}^{N}}\int_{B^{c}(0,\rho_0)}
 \frac{|u(x+z)-u(x)|^{2}}{|z|^{N+2\alpha}}\,dz\,dx\Big)
\\
&\leq C\int_{\mathbb{R}^{N}}\int_{B(0,\rho_0)}
 \frac{|u(x+z)-u(x)|^{2}}{|z|^{N+2\alpha}}\,dz\,dx
 + \frac{2C|S^{n-1}|}{\alpha\rho_0^{2\alpha}}\|u\|^{2}_2\\
&\leq C\int_{\mathbb{R}^{N}}\int_{B(0,\rho_0)}
 \frac{|u(x+z)-u(x)|^{2}}{|z|^{N+2\alpha}}\,dz\,dx
 + \frac{2C|S^{n-1}|}{c\alpha\rho_0^{2\alpha}}
 \int_{\mathbb{R}^{N}}V(x)|u(x)|^{2}dx\\
&\leq C_0\|u\|^{2},
\end{align*}
where $C_0 = \max \{C ,\frac{2C|S^{n-1}|}{c\alpha\rho_0^{2\alpha}}\}$.
This completes the proof.
\end{proof}

\begin{lemma} \label{lem2.2}
 Suppose {\rm (A1)--(A2)} hold, and that there exists a constant $K > 0$
 such that
\[
\|u\|_{q} \leq K \|u\|,
\]
where $2 \leq q \leq 2_{\alpha}^{*}$. Then the embedding
$E\hookrightarrow L^{q}(\mathbb{R}^{N})$ is continuous for
$2 \leq q \leq 2_{\alpha}^{*}$ and
$E\hookrightarrow L^{s}_{\rm loc}(\mathbb{R}^{N})$ is compact for
$2 \leq s < 2_{\alpha}^{*}$.
\end{lemma}

\begin{proof}
 When $q = 2$, by the definition of $\|\cdot\|$, there exists $C_1 > 0$ such that
\[
\|u\|_2 \leq C_1\|u\|.
\]
When $q = 2_{\alpha}^{*}$, using Lemma \ref{lem2.1}, we obtain
\[
\|u\|_{2_{\alpha}^{*}}\leq C_0^{1/2}\|u\|.
\]
When $2 < q < 2_{\alpha}^{*}$, according to the H\"older inequality and
Lemma \ref{lem2.1}, we obtain
\begin{align*}
\|u\|_{q}^{q}
& \leq \Big(\int_{\mathbb{R}^{N}}|u|^{2_{\alpha}^{*}}dx\Big)
^{\frac{q-2}{2_{\alpha}^{*}-2}}
 \Big(\int_{\mathbb{R}^{N}}|u|^{2}dx\Big)
^{\frac{2_{\alpha}^{*}-q}{2_{\alpha}^{*}-2}}\\
 & \leq  C_0^{\frac{(q-2)2_{\alpha}^{*}}{2 (2_{\alpha}^{*}-2)}}
 \|u\|^{\frac{(q-2)2_{\alpha}^{*}}{2_{\alpha}^{*}-2}}
         C_{1}^{\frac{2(2_{\alpha}^{*}-q)}{2_{\alpha}^{*}-2}}
 \|u\|^{\frac{2(2_{\alpha}^{*}-q)}{2_{\alpha}^{*}-2}}\\
 & \leq C_2^{q}\|u\|^{q},
\end{align*}
where
\[
C_2 = \Big(C_0^\frac{(q-2)2_{\alpha}^{*}}{2
(2_{\alpha}^{*}-2)}C_{1}^{\frac{2(2_{\alpha}^{*}-q)}{2_{\alpha}^{*}-2}}
\Big)^{1/q}.
\]
Let $K = \max\{C_0^{1/2},C_1,C_2\}$, we have
\[
\|u\|_{q} \leq K \|u\|
\]
for $2 \leq q \leq 2_{\alpha}^{*}$.
Therefore, the embedding $E\hookrightarrow L^{q}(\mathbb{R}^{N})$
is continuous for $2 \leq q \leq 2_{\alpha}^{*}$ and
$E\hookrightarrow L^{s}_{\rm loc}(\mathbb{R}^{N})$ is
compact for $2 \leq s < 2_{\alpha}^{*}$.
\end{proof}

\begin{lemma} \label{lem2.3}
 Suppose {\rm (A1)--(A2)} hold. Then $E\hookrightarrow L^{q}(\mathbb{R}^{N})$
is compact for any $2 \leq q < 2_{\alpha}^{*}$.
\end{lemma}

\begin{proof}
First, we show that $E\hookrightarrow L^{2}(\mathbb{R}^{N})$ is compact.
Let $\{u_n\}\subset E$ with $\{u_n\}$  bounded
in $E$. Then $u_n\rightharpoonup u$ weakly in $E$, by Lemma \ref{lem2.2},
which implies that $u_n\to u$ strongly in $L_{\rm loc}^{2}(\mathbb{R}^{N})$.

Set $\delta_n = \|u_n\|_2$, we may suppose that there is a subsequence
of $\{u_n\}$ and $\delta \in R$ such that $\delta_n\to \delta$.
For any bounded domain $\Omega$ in $\mathbb{R}^{N}$, we have
\[
\int_{\Omega}|u|^{2}dx = \lim_{n\to\infty}\int_{\Omega}|u_n|^{2}dx
\leq \lim_{n\to\infty}\int_{\mathbb{R}^{N}}|u_n|^{2}dx =
\delta^{2},
\]
so $\|u\|_2 \leq \delta$ .

Let $\Omega_{1} : = \{x\in B_{R}^{c}|V(x) \geq M\}$,
$\Omega_2 := \{x\in B_{R}^{c}|V(x) < M\}$.
On the basis of $\Omega_{1}$, we obtain
\[
\int_{\Omega_{1}}|u_n|^{2}dx
\leq \int_{\mathbb{R}^{N}}\frac{V(x)}{M}|u_n|^{2}dx
\leq \frac{\|u_n\|^{2}}{M} .
\]
Let $\{y_{i}\}$ be a sequence satisfying
$\mathbb{R}^{N} \subset \bigcup_{i=1}^{\infty}B(y_{i},r_0)$
and each point $x$ is
contained in at most $2^N$ such balls $B(y_{i},r_0)$.
By the H\"older inequality, we choose a positive constant
$s\in(1,\frac{2^{*}_{\alpha}}{2})$ such that
\begin{align*}
\int_{\Omega_2}|u_n|^{2}dx \leq & \sum_{i=1}^{\infty}\int_{\Omega_2\cap B(y_{i},r_0)}|u_n|^{2}dx\\
\leq & \sum_{i=1}^{\infty}
\Big(\int_{\Omega_2\cap B(y_{i},r_0)}|u_n|^{2s}dx\Big)^{1/s}
\Big(\int_{\Omega_2\cap B(y_{i},r_0)} 1 dx\Big)^{1-\frac{1}{s}}.
\end{align*}
Define
\begin{align*}
\|u_n\|_{B(y_{i},r_0)}^{2}
&= \int_{B(y_{i},r_0)}\int_{B(0,\rho(x))\cap B(y_{i},r_0)}
 \frac{|u(x+z)-u(x)|^{2}}{|z|^{N+2\alpha}}\,dz\,dx \\
&\quad + \int_{B(y_{i},r_0)}V(x)|u(x)|^{2}dx.
\end{align*}
Using  proof similar to the one of  Lemmas \ref{lem2.1} and \ref{lem2.2}, there exists
$C > 0$ such that
\[
\Big(\int_{\Omega_2\cap B(y_{i},r_0)}|u_n|^{2s}dx\Big)^{\frac{1}{s}}
\leq \Big(\int_{B(y_{i},r_0)}|u_n|^{2s}dx\Big)^{\frac{1}{s}}
\leq C\|u_n\|_{B(y_{i},r_0)}^{2}.
\]
Now, we can estimate
\[
\int_{\Omega_2}|u_n|^{2}dx
\leq  \sum_{i=1}^{\infty}C \|u_n\|_{B(y_{i},r_0)}^{2}\operatorname{meas}(\Omega_2\cap
B(y_{i},r_0))^{1-\frac{1}{s}}\\
\leq  2^{N}C \varepsilon_{R}\|u_n\|^{2},
\]
where $\varepsilon_{R} = \sup_{y_{i}}\operatorname{meas}
\left(\Omega_2\cap B(y_{i},r_0)\right)^{1-\frac{1}{s}}$.
Note that $\{u_n\}$ is bounded in $E$, for $M$ large enough, we have
$\frac{\|u_n\|^{2}}{M}\to 0$. Since (A2) holds, for $R$ large enough,
we obtain $\varepsilon_{R}\to 0$.
It is easy to check that, given any $\varepsilon > 0$,
for sufficiently large $R$ and $M$,
\[
\int_{B^{c}_{R}}|u_n|^{2}dx =  \int_{\Omega_{1}}|u_n|^{2}dx + \int_{\Omega_2}|u_n|^{2}dx \leq \varepsilon
\]
and
\begin{align*}
\|u\|_2^{2}
&=  \int_{B_{R}}|u|^{2}dx + \int_{B^{c}_{R}}|u|^{2}dx \\
       \geq & \lim_{n\to\infty}\int_{B_{R}}|u_n|^{2}dx \\
          = & \lim_{n\to\infty}\Big(\int_{R_{N}}|u_n|^{2}dx
- \int_{B^{c}_{R}}|u_n|^{2}dx\Big)\\
       \geq & \delta^{2} - \varepsilon.
\end{align*}
It means that $\delta \leq \|u\|_2$.
Therefore, $\delta = \|u\|_2$ and it implies that
$E\hookrightarrow L^{2}(\mathbb{R}^{N})$ is compact.

Next, we prove that $E\hookrightarrow L^{q}(\mathbb{R}^{N})$
is compact for $2 < q < 2_{\alpha}^{*}$. Let $r\in (0,1)$ be such that
$q = 2r + 2_{\alpha}^{*}(1-r)$.
Using the H\"older inequality again, we have
\[
\|u_n-u\|_{q}^{q} \leq \|u_n-u\|_2^{2r}
\|u_n-u\|_{2_{\alpha}^{*}}^{2_{\alpha}^{*}(1-r)}.
\]
Since $\{u_n - u\}$ is bounded in $E$, then $u_n - u \rightharpoonup 0$,
by Lemma \ref{lem2.2}, there exists $C > 0$ such that
$\|u_n-u\|_{2_{\alpha}^{*}}^{2_{\alpha}^{*}(1-r)} \leq C$.
Thus,
\[
\|u_n-u\|_{q}^{q} \to 0.
\]
Here, we can make a result of the proof of Lemma \ref{lem2.3}.
\end{proof}

\begin{lemma} \label{lem2.4}
Under the assumptions of Theorem \ref{thm1.1} hold, the
functional $I$ satisfies the $(Ce)_{c}$ condition for $c > 0$.
\end{lemma}

\begin{proof}
Assume that $\{u_n\} \subset E$ is a $(Ce)_c$ sequence for $c > 0$,
\[
I(u_n) \to c,\quad (1+\|u_n\|)I'(u_n) \to 0\quad \text{as} n \to \infty.
\]
For any $\varphi\in E$, we obtain that
\begin{equation}\label{48}
\begin{split}
\langle I'(u_n),\varphi\rangle
&= \int_{\mathbb{R}^{N}}\int_{B(0,\rho(x))}\frac{|u_n(x+z)-u_n(x)||\varphi(x+z)-\varphi(x)|}{|z|^{N+2\alpha}}\,dz\,dx \\
&\quad  + \int_{\mathbb{R}^{N}}V(x)|u_n(x)||\varphi(x)|dx
 - \int_{\mathbb{R}^{N}}f(x,u_n^{+})\varphi(x) dx
 \to  0
\end{split}
\end{equation}
and
\begin{equation}\label{8}
\langle I'(u_n),u_n\rangle
=  \frac{1}{2}\|u_n\|^{2} - \int_{\mathbb{R}^{N}}f(x,u_n^{+})u_n^{+}dx \to 0.
\end{equation}
First we claim that $\{u_n\}$ is bounded in $E$. In fact, if not, we may
assume by the contradiction that there exists a subsequence of
$\{u_n\}$ (still denoted by $\{u_n\}$) with $\|u_n\| \to +\infty$, and we set
\[
w_n = \frac{u_n}{\|u_n\|}.
\]
Clearly, $\{w_n\}$ is bounded in $E$. Going if necessary to a subsequence
 of $\{w_n\}$, we can assume that
\begin{gather*}
w_n \rightharpoonup w\quad \text{weakly in }E,\\
w_n \to  w\quad \text{strongly in }L^{q}(\mathbb{R}^{N}) \quad
(2 \leq q < 2_{\alpha}^{*}),\\
w_n \to  w\quad \text{a.e. }x \in \mathbb{R}^{N}.
\end{gather*}
Similarly, we denote
\[
w_n^{+} = \frac{u_n^{+}}{\|u_n\|},
\]
then
\begin{gather*}
w_n^{+} \rightharpoonup w^{+}\quad \text{weakly in }E,\\
w_n^{+} \to  w^{+}\quad \text{strongly in }L^{q}(\mathbb{R}^{N}) \quad
 (2 \leq q < 2_{\alpha}^{*}),\\
w_n^{+} \to  w^{+}\quad \text{a.e. }x \in \mathbb{R}^{N}.
\end{gather*}
Next we claim that $w \neq 0$.
Otherwise, if $w \equiv 0$, we know that $w_n^{+} \to  0$ strongly in
$L^{2} (\mathbb{R}^{N})$.
By $(f_{1})$, for each $\varepsilon >0$, there exists $T_0 > 0$ such that
\[
|f(x,t)t| < \varepsilon t^{2},
\]
for $ 0 < t < T_0$.
Through the continuity of $f$, there exists $C > 0$ such that
\[
|f(x,t)t| \leq C \leq \frac{C}{T_0^2}t^{2}
\]
for $T_0 \leq t \leq T_{1}$.
Hence, we have
\begin{equation}\label{9}
|f(x,t)t| \leq \frac{C}{T_0^2}t^{2}
\end{equation}
for $0 < t \leq T_{1}$ and
\begin{equation}\label{10}
|F(x,t)| \leq \frac{C}{2T_0^2}t^{2}
\end{equation}
for $0 < t \leq T_{1}$.
By (A6), it is easy to see that
\[
\frac{1}{\mu}f(x,t)t- F(x,t) \geq 0
\]
for $t > T_{1}$.
Combining \eqref{9} and \eqref{10}, we have
\begin{equation}\label{32}
\frac{1}{\mu}f(x,t)t- F(x,t) \geq -(\frac{1}{2}-\frac{1}{\mu}
)\frac{C}{T_0^2}t^{2}
\end{equation}
for all $(x,t)\in \mathbb{R}^{N}\times \mathbb{R}^{+}$.
Under the definition of $\{u_n\}$, we see that
\begin{align*}
&I(u_n) - \frac{1}{\mu} \langle I'(u_n),u_n\rangle\\
&=  \big(\frac{1}{2}-\frac{1}{\mu}\big)\|u_n\|^{2}
 - \int_{\mathbb{R}^{N}}\Big(F(x,u_n^{+})
-\frac{1}{\mu}f(x,u_n^{+})u_n^{+}\Big)dx\\
&=  c - o_n(1).
\end{align*}
It follows from the preceding step and \eqref{32} that
\[
\begin{split}
0& =  \lim_{n\to\infty}\frac{I(u_n) - \frac{1}{\mu}\langle I'(u_n),u_n\rangle}{\|u_n\|^{2}}\\
 & =  \frac{1}{2}-\frac{1}{\mu} - \lim_{n\to\infty}\int_{\mathbb{R}^{N}}\frac{F(x,u_n^{+})
  -\frac{1}{\mu}f(x,u_n^{+})u_n^{+}}{\|u_n\|^{2}}dx\\
&\geq  \frac{1}{2}-\frac{1}{\mu} -
 \big(\frac{1}{2}-\frac{1}{\mu}\big)\frac{C}{T_0^2}
 \lim_{n\to\infty}\int_{\mathbb{R}^{N}}\frac{(u_n^{+})^{2}}{\|u_n\|^{2}}dx\\
&\geq  \frac{1}{2}-\frac{1}{\mu} - \big(\frac{1}{2}
 -\frac{1}{\mu}\big)\frac{C}{T_0^2}\lim_{n\to\infty}\|w_n^{+}\|_2^{2}\\
&\geq \frac{1}{2}-\frac{1}{\mu}.
\end{split}
\]
Note that $\mu > 2$, this is a contradiction, so $w \neq 0$.

Since $\{u_n\}$ is $(Ce)_c$ sequence, by \eqref{48}, we know that
\[
\langle I'(u_n),u_n^{-}\rangle \leq \|I'(u_n)\|\|u_n^{-}\|
 \leq \|I'(u_n)\|\|u_n\| \to 0.
\]
Then $w_n^{-} =  \frac{u_n^{-}}{\|u_n\|}\to 0$, which implies that $w^{-} = 0$
for a.e. $x \in \mathbb{R}^{N}$.
We set
\[
h(s) = s^{\mu}F(x, \frac{t}{s}),
\]
then
\[
h'(s)=   \mu s^{\mu-1} F\big(x,\frac{t}{s}\big)
  - s^{\mu}f\big(x, \frac{t}{s}\big)\frac{t}{s^{2}}
=  s^{\mu-1}\Big(\mu F\big(x, \frac{t}{s}\big)
 - f\big(x, \frac{t}{s}\big)\frac{t}{s}\Big).
\]
By (A5), we can find a $T_2 > 0$ such that
   \[
   \inf_{t \geq T_2,\text{ }x\in \mathbb{R}^{N}}F(x,t) > 0.
   \]
Denote $T = \max\{T_1,T_2\}$. For $s\in [1, \frac{t}{T}]$, by $(f_4)$, we have
\[
\mu F(x, \frac{t}{s})- f(x, \frac{t}{s})\frac{t}{s} \leq 0.
\]
Hence, $h'(s)\leq 0$ and $h(1) \geq h(\frac{t}{T})$. This implies
\begin{equation}\label{7}
F(x,t)\geq \frac{1}{T_{1}^{\mu}}F(x,T)t^{\mu}
\end{equation}
for all $(x,t) \in (\mathbb{R}^{N},\mathbb{R}^{+})$, where
\begin{equation}
c_0 = \frac{1}{T^{\mu}}\inf_{t = T, x\in \mathbb{R}^{N}}F(x,t).
\end{equation}
In accordance with \eqref{7} that
\[
F(x,t) \geq c_0t^{\mu}
\]
for all $(x,t) \in (\mathbb{R}^{N},\mathbb{R}^{+})$.
Then, one has
\[
f(x,t)t \geq \mu c_0t^{\mu}
\]
for all $(x,t)\in \mathbb{R}^{N}\times \mathbb{R}^{+}$.
Related to \eqref{8}, we obtain
\[
\frac{\langle I'(u_n),u_n\rangle}{\|u_n\|^{\mu}} = \frac{1}{\|u_n\|^{\mu-2}} -
\int_{\mathbb{R}^{N}}\frac{f(x,u_n^{+})u_n^{+}}{\|u_n\|^{\mu}}dx = o_n(1).
\]
Therefore, we know
\begin{align*}
0 = & \lim_{n\to\infty}\int_{\mathbb{R}^{N}}\frac{f(x,u_n^{+})u_n^{+}}{\|u_n\|^{\mu}}dx\\
&\geq   \lim_{n\to\infty}\mu c_0\|w_n^{+}\|_{\mu}^{\mu}\\
&\geq  \mu c_0\|w^{+}\|_{\mu}^{\mu}
 >  0.
\end{align*}
However, this is a contradiction. So our
assumption $\|u_n\| \to +\infty$ is false, and that is to say,
$\{u_n\}$ is bounded in $E$.

Since $\{u_n\}$ is bounded in $E$, there exists $C_1 > 0$ such that
\[
\|u_n\| \leq C_1.
\]
Moreover, there exists $C_2 > 0$ such that
\[
\|u_n - u\|_{2^{*}_{\alpha}} \leq C_2.
\]
By the reflexivity of $E$, there exists a subsequence of $\{u_n\}$
(which we also denote by $\{u_n\}$)
and $u\in E$ such that
\begin{equation}\label{11}
\begin{gathered}
u_n \rightharpoonup u\quad \text{weakly in }E,\\
u_n \to u\quad \text{strongly in }L^{q}(\mathbb{R}^{N}) \quad
(2 \leq q < 2_{\alpha}^{*}).
\end{gathered}
\end{equation}
At the same time,
\[
\langle I'(u_n),u_n-u\rangle \to 0,\quad \langle I'(u),u_n-u\rangle \to 0.
\]
Hence, we have
\begin{equation}\label{12}
\begin{split}
&\langle I'(u_n)- I'(u),u_n-u\rangle\\
&=  \|u_n - u\|^{2} - \int_{\Omega}\left(f(x,u_n^{+})-f(x,u^{+})\right)(u_n^{+}
- u^{+})dx
\to  0.
\end{split}
\end{equation}
By (A3), (A4) and $f \in C(\mathbb{R}^{N}\times \mathbb{R}^{+}, R)$, for
$\varepsilon > 0$, there exists $C_{\varepsilon} > 0$ such that
\begin{equation}\label{88}
f(x,t) \leq C_{\varepsilon} t + \varepsilon t^{2^{*}_{\alpha} - 1}
\end{equation}
for all $(x,t) \in \mathbb{R}^{N}\times \mathbb{R}^{+}$.
 By \eqref{88} and the H\"{o}lder inequality, we have
\begin{align*}
 \Big|\int_{\mathbb{R}^{N}}f(x,u_n^{+})(u_n^{+}-u^{+})dx\Big|
&\leq  \int_{\mathbb{R}^{N}}|f(x,u_n^{+})||u_n - u|dx\\
&\leq  \int_{\mathbb{R}^{N}}\left(C_{\varepsilon}|u_n^{+}||u_n - u|
 + \varepsilon|u_n^{+}|^{2^{*}_{\alpha} - 1}|u_n-u|\right)dx\\
&\leq  C_{\varepsilon} \|u_n\|_2\|u_n-u\|_2
 + \varepsilon\|u_n\|^{2^{*}_{\alpha} -  1}_{2^{*}_{\alpha}}
 \|u_n-u\|_{2^{*}_{\alpha}}\\
&\leq  C_{\varepsilon}K C_{1}\|u_n-u\|_2
 + \varepsilon K^{2^{*}_{\alpha}-1}C_{1}^{2^{*}_{\alpha}-1}C_2.
\end{align*}
Related to \eqref{11}, we obtain
\[
\int_{\mathbb{R}^{N}}f(x,u_n^{+})(u_n^{+}-u^{+})dx \to 0.
\]
Consequently,
\[
\int_{\mathbb{R}^{N}}\left(f(x,u_n^{+})-f(x,u^{+})\right)(u_n^{+} - u^{+})dx \to 0.
\]
By \eqref{12}, we have $\|u_n - u\|^{2} \to 0$.
So we derive that $u_n \to u$ strongly in $E$, we conclude that
the $(Ce)_{c}$ condition is satisfied.
\end{proof}


\begin{lemma} \label{lem2.5}
 Under  the assumptions of Theorem \ref{thm1.2},  the
functional $I$ satisfies the $(Ce)_{c}$ condition for $c > 0$.
\end{lemma}

\begin{proof}
Assume that $\{u_n\} \subset E$ is a $(Ce)_c$ sequence for $c > 0$,
\[
I(u_n) \to c, \quad (1 + \|u_n\|)I'(u_n) \to 0\quad (n \to \infty).
\]
We first claim that the sequence $\{u_n\}$ is bounded in $E$.
Otherwise, there is a subsequence, again denoted
by $\{u_n\}$, such that $\|u_n\| \to +\infty$ as $n \to \infty$.
Set
\[
w_n = \frac{u_n}{\|u_n\|}, \quad  w_n^{+} = \frac{u_n^{+}}{\|u_n\|}.
\]
Clearly, $\{w_n\}$ is bounded
in $E$. Going if necessary to a subsequence of $\{w_n\}$, we can assume that
\begin{gather*}
w_n \rightharpoonup w\quad \text{weakly in }E,\\
w_n \to  w\quad \text{strongly in }L^{q}(\mathbb{R}^{N}) \quad
(2 \leq q < 2_{\alpha}^{*}),\\
w_n \to  w\quad \text{a.e. }x \in \mathbb{R}^{N}.
\end{gather*}
Then we claim that $w \neq 0$. Otherwise, if $w \equiv 0$, we obtain
 $w_n \to  0$ strongly in $L^{q} (\mathbb{R}^{N})$.
Since $I(tu_n)$($t\in[0,1]$) is continuous, there exist $t_n\in [0,1]$ such that
\[
I(t_nu_n) = \max_{t\in[0,1]}I(tu_n).
\]
For $t_n\in(0,1)$, we have
\[
\langle I'(t_nu_n), t_nu_n\rangle
= \|t_nu_n\|^{2}
- \int_{\mathbb{R}^{N}}f(x,t_nu_n^{+})t_nu_n^{+}dx = t_n
\frac{d}{dt}|_{t=t_n}I(tu_n) = 0.
\]
It is obvious for $t_n = 0$, we obtain
\[
\langle I'(t_nu_n), t_nu_n\rangle = 0.
\]
When $t_n = 1$, we obtain
\[
\langle I'(t_nu_n), t_nu_n\rangle
= \langle I'(u_n), u_n\rangle \to 0.
\]
Hence, we have
\begin{equation}\label{23}
\langle I'(t_nu_n), t_nu_n\rangle
= \|t_nu_n\|^{2} - \int_{\mathbb{R}^{N}}f(x,t_nu_n^{+})t_nu_n^{+}dx
\to 0
\end{equation}
for $t_n\in[0,1]$.
Set
\[
v_n = 2m^{1/2}w_n
\]
for each $m > 0$. By \eqref{88}, we have
\begin{equation}\label{89}
F(x,t) \leq \frac{C_{\varepsilon}}{2} t^{2}
+ \frac{\varepsilon}{2^{*}_{\alpha}}t^{2^{*}_{\alpha}}
\end{equation}
for all $(x,t) \in \mathbb{R}^{N}\times \mathbb{R}^{+}$.
Since $v_n \to 0$ strongly in $L^{2}(\mathbb{R}^{N})$ and $v_n$ is
bounded in $E$, we come to a conclusion that
\[
\int_{\mathbb{R}^{N}}F(x,v_n^{+})dx \to 0.
\]
It is apparently showed that $\frac{2m^{1/2}}{\|u_n\|} \in (0,1)$ for $n$
large enough, so
\[
I(t_nu_n)\geq I(v_n) = 2m - \int_{\mathbb{R}^{N}}F(x,v_n^{+}) \geq m.
\]
Then we have $I(t_nu_n) \to +\infty$.
Denote $H(x,t) = f(x,t)t-2F(x,t)$, it follows
\begin{equation}\label{21}
\int_{\mathbb{R}^{N}}H(x,u_n^{+})dx = 2I(u_n) - \langle I'(u_n), u_n\rangle \to 2c.
\end{equation}
Assume $0 \leq s_{1} \leq s_2$, by ($f_{5}$), we have
\begin{align*}
&H(x,s_2) - H(x,s_{1})\\
&=  2\Big[\frac{1}{2}\left(f(x,s_2)s_2 - f(x,s_{1})s_{1}\right)
- \left(F(x,s_2) - F(x,s_{1})\right)\Big]\\
&=  2\int_{s_{1}}^{s_2}\Big(\frac{f(x,s_2)}{s_2}-\frac{f(x,v)}{v}\Big)vdv
 +  2\int_{T_2}^{s_{1}}\Big(\frac{f(x,s_2)}{s_2}
-\frac{f(x,s_{1})}{s_{1}}\Big)vdv\\
&\quad + T_2^{2}\Big(\frac{f(x,s_2)}{s_2} - \frac{f(x,s_{1})}{s_{1}}\Big)
\geq  0.
\end{align*}
Thus, we have
\begin{align*}
\int_{\mathbb{R}^{N}}H(x,u^{+}_n)dx
&\geq  \int_{\mathbb{R}^{N}}H(x,t_nu^{+}_n)dx\\
&=  2I(t_nu_n) - \langle I'(t_nu_n), t_nu_n\rangle
  \to  +\infty.
\end{align*}
This contradicts \eqref{21}, so $w \neq 0$. Since $\{u_n\}$ is $(Ce)_c$ sequence,
we know that
\[
\langle I'(u_n),u_n^{-}\rangle \leq \|I'(u_n)\|\|u_n^{-}\|
\leq \|I'(u_n)\|\|u_n\| \to 0.
\]
Thus, $w_n^{-} = \frac{u_n^{-}}{\|u_n\|}\to 0$, which implies that
$w^{-} = 0$ for a.e. $x \in \mathbb{R}^{N}$.
Then we set
\[
\Omega_0 = \{x \in \mathbb{R}^{N}| w(x) = 0\},\quad
\Omega_1 = \{x \in \mathbb{R}^{N}| w(x) > 0\}.
\]
By (A5) and the Fatou Lemma, we have
\[
\liminf_{n\to\infty}\int_{\Omega_1}\frac{F(x,u_n^{+})}{u_n^{2}}w_n^{2}dx
\geq
\int_{\Omega_1}\liminf_{n\to\infty}\frac{F(x,u_n^{+})}{u_n^{2}}w_n^{2}dx
\to +\infty.
\]
In the meantime, by (A5), there exists $T_0 > 0$, such that
 \[
 \frac{F(x,t)}{t^{2}} \geq 1
 \]
for $t >T_0$. By the continuity of $F$, there exists $C_{1} > 0$ such that
\[
|F(x,t)| \leq C_{1}
\]
for $0 < t \leq T_0$. Combining the preceding two inequalities,
there exists $C_2 > 0$ such that
\begin{align*}
 \int_{\Omega_0}\frac{F(x,u_n^{+})}{u_n^{2}}w_n^{2}dx
&=  \int_{\Omega_0(0 < u_n(x) \leq T_0)}\frac{F(x,u_n)}{\|u_n\|^{2}}dx
 + \int_{\Omega_0(u_n(x) > T_0)}\frac{F(x,u_n)}{u_n^{2}}w_n^{2}dx\\
&\geq  - \frac{C_{3}}{\|u_n\|^{2}}
\operatorname{meas}\big(\Omega_0(0 < u_n(x) \\
&\leq T_0)\big) + \int_{\Omega_0}w_n^{2}dx
>  -C_2.
\end{align*}
Dividing \eqref{3} with $\|u_n\|^{2}$, we have
\[
 \frac{I(u_n)}{\|u_n\|^{2}} = \frac{1}{2}
-  \int_{\mathbb{R}^{N}}\frac{F(x,u_n^{+})}{u_n^{2}}(w_n^{+})^{2}dx = o_n(1).
\]
Hence, we obtain
\begin{align*}
\frac{1}{2} - o_n(1)
&=  \int_{\mathbb{R}^{N}}\frac{F(x,u_n^{+})}{u_n^{2}}w_n^{2}dx\\
&=  \int_{\Omega_0}\frac{F(x,u_n^{+})}{u_n^{2}}w_n^{2}dx +
             \int_{\Omega_1}\frac{F(x,u_n^{+})}{u_n^{2}}w_n^{2}dx
             \to  +\infty.
\end{align*}
It is easy to see that it is a contradiction. So $\{u_n\}$ is bounded in $E$.

By the standard processes similar to Lemma \ref{lem2.4}, we know that $u_n \to u$
strongly in $E$. This completes the proof.
\end{proof}

\begin{lemma} \label{lem2.6}
Under the assumptions of Theorem \ref{thm1.3}, the
functional $I$ satisfies the $(Ce)_{c}$ condition for $c > 0$.
\end{lemma}

\begin{proof}
Assume that $\{u_n\} \subset E$ is a $(Ce)_c$ sequence for $c > 0$,
\[
I(u_n) \to c,\quad (1 + \|u_n\|)I'(u_n) \to 0 \text{ as }n
\to \infty.
\]
We argue that the sequence $\{u_n\}$ is bounded in $E$. Otherwise,
there is a subsequence, again denoted
by $\{u_n\}$, such that $\|u_n\| \to +\infty$ as $n \to \infty$.
By ($f_{6}$), we see
that, there exists $R_1 > 0$ such that
\begin{equation}\label{16}
f(x,t)t \leq \nu bt^{\nu}
\end{equation}
for $t \geq R_1$.
By (A9), for $0 < \delta < \eta$, there exists $R_2 > 0$ such that
\begin{equation}\label{17}
f(x,t)t - 2F(x,t) \geq (\eta - \delta)t^{\sigma}
\end{equation}
for $t \geq R_2$.
Set $T = \max\{R_1, R_2 \}$, let
\[
\Omega_{1}= \{x \in  \mathbb{R}^{N}: u_n(x) \geq T\}, \quad
\Omega_2 = \{x \in  \mathbb{R}^{N}: 0 < u_n(x) < T\}.
\]
Moreover, by the continuity of $f$ and $F$, there exists $C_1 >0$ such that
\[
\Big|\int_{\Omega_2}\left(f(x,u_n)u_n - 2F(x,u_n)\right)dx\Big|\leq C_1.
\]
Since $\{u_n\} \subset E$ is a $(Ce)_c$ sequence, we obtain
\begin{align*}
2c + o_n(1)
&=  \int_{\mathbb{R}^{N}}\left(f(x,u_n^{+})u_n^{+} - 2F(x,u_n^{+})\right)dx\\
&\geq  \int_{\Omega_{1}}(f(x,u_n)u_n - 2F(x,u_n))dx - C_1.
\end{align*}
Thus,
\[
\int_{\Omega_{1}}(f(x,u_n)u_n - 2F(x,u_n))dx \leq 2c + C_1 + o_n(1).
\]
It follows from \eqref{17} that there exists $C_2 > 0$ such that
\[
\int_{\Omega_{1}}u_n^{\sigma}dx \leq C_2.
\]
Using the continuity of $f$ again, there
exists $C_3 > 0$ such that
\[
\Big|\int_{\Omega_2}f(x,u_n)u_ndx\Big| \leq C_3.
\]
Through the definition of $\{u_n\}$, it reaches
\[
\begin{split}
\|u_n\|^{2} - o_n(1)
&=  \int_{\mathbb{R}^{N}}f(x,u_n^{+})u_n^{+}dx\\
&=  \int_{\Omega_{1}}f(x,u_n)u_ndx + \int_{\Omega_2}f(x,u_n)u_ndx\\
&\leq  \int_{\Omega_{1}}f(x,u_n)u_ndx + C_3.
 \end{split}
\]
From \eqref{16}, we obtain
\[
\int_{\Omega_{1}}f(x,u_n)u_ndx \leq \nu b\int_{\Omega_{1}}u_n^{\nu}dx.
\]
Then, we will consider two cases.
\smallskip

\noindent\textbf{Case 1.}
 $\sigma \geq \nu$, there exists $t \in (0,1)$ such that
$\frac{1}{\nu} = \frac{1-t}{\sigma} + \frac{t}{2}$. By the H\"{o}lder
inequality,
\begin{align*}
\int_{\Omega_{1}}u_n^{\nu}dx
&\leq
\Big(\int_{\Omega_{1}}u_n^{\sigma}dx\Big)^{\frac{(1-t)\nu}{\sigma}}
\Big(\int_{\Omega_{1}}u_n^{2}dx\Big)^{\frac{t\nu}{2}}\\
&\leq  C_2^{\frac{(1-t)\nu}{\sigma}}K^{t\nu}\|u_n\|^{t\nu}.
\end{align*}
\smallskip

\noindent\textbf{Case 2.} $\sigma < \nu$, there exists $t \in (0,1)$
such that $\frac{1}{\nu} = \frac{1-t}{\sigma} + \frac{(N-2\alpha)t}{2N}$.
By the H\"{o}lder inequality,
\begin{align*}
\int_{\Omega_{1}}u_n^{\nu}dx \leq &
\Big(\int_{\Omega_{1}}u_n^{\sigma}dx\Big)^{\frac{(1-t)\nu}{\sigma}}
\Big(\int_{\Omega_{1}}u_n^{\frac{2N}{N-2\alpha}}dx
 \Big)^{\frac{t\nu(N-2\alpha)}{2N}}\\
&\leq  C_2^{\frac{(1-t)\nu}{\sigma}}K^{t\nu}\|u_n\|^{t\nu}.
\end{align*}
Hence, one has
\[
\|u_n\|^{2} - o_n(1)
\leq  C_3 + \nu bC_2^{\frac{(1-t)\nu}{\sigma}}K^{t\nu}\|u_n\|^{t\nu}.
\]
It is easily observed that $\sigma > \frac{N}{2\alpha}(\nu-2)$
is equivalent to $t\nu <2$, it is easy to concluce that the
assumption is false. So $\{u_n\}$ is bounded in $E$.

Since (A3) and (A4) hold, we know that $u_n \to u$ strongly in $E$ by
the standard processes similar to Lemma \ref{lem2.4}. So $I$
satisfies the $(Ce)_{c}$ condition.
\end{proof}

\begin{lemma} \label{lem2.7}
 Assume that {\rm (A1)--(A4)} hold. Then $I$ satisfies the
the following conditions:
\begin{itemize}
\item[(1)] There exist $\theta > 0$, $\xi > 0$ such that $I(u) \geq \xi > 0$
 for all $u \in E$ with $\|u\| = \theta$;

\item[(2)] There exists $e \in E$ with $\|e\| > \theta$ such that $I(e) \leq 0$.
\end{itemize}
\end{lemma}

\begin{proof}  (1) By (A3), (A4) and
$f \in C(\mathbb{R}^{N}\times \mathbb{R}^{+}, R)$, for
$0 < \varepsilon < \frac{1}{2K^{2}}$, there exists $C_{\varepsilon} > 0$
such that
\begin{equation}\label{4}
f(x,t)t \leq \varepsilon t^{2} + C_{\varepsilon}t^{2_{\alpha}^{*}}
\end{equation}
for all $(x,t)\in \mathbb{R}^{N}\times \mathbb{R}^{+}$. This implies that
\begin{equation}\label{5}
\begin{split}
F(x,t) &=  \int^{1}_0f(x,st)tds\\
    &\leq  \int^{1}_0\left(\varepsilon s t^{2}
+  C_{\varepsilon} s^{2_{\alpha}^{*}-1}t^{2_{\alpha}^{*}}\right)ds\\
    &\leq \frac{\varepsilon}{2}t^{2}
+ \frac{C_{\varepsilon}}{2_{\alpha}^{*}}t^{2_{\alpha}^{*}}
\end{split}
\end{equation}
for all $(x,t)\in \mathbb{R}^{N}\times \mathbb{R}^{+}$.
Pay attention to the definition of $I$ given in
\eqref{3}, it follows from \eqref{5} and Lemma \ref{lem2.2} that
\begin{equation}\label{6}
 \begin{split}
I(u) &=  \frac{1}{2}\|u\|^{2} -  \int_{\mathbb{R}^{N}}F(x,u^{+})dx\\
 &\geq \frac{1}{2}\|u\|^{2} - \int_{\mathbb{R}^{N}}
 \frac{\varepsilon}{2}(u^{+})^{2}dx
 -  \int_{\mathbb{R}^{N}}\frac{C_{\varepsilon}}{2_{\alpha}^{*}}(u^{+})
 ^{2_{\alpha}^{*}}dx\\
&\geq \frac{1}{2}\left(1 - \varepsilon K^{2}\right)\|u\|^{2} -
     \frac{C_{\varepsilon}K^{2_{\alpha}^{*}}}{2_{\alpha}^{*}}\|u\|
 ^{2_{\alpha}^{*}}\\
&\geq  \Big(\frac{1}{4} - \frac{C_{\varepsilon}K^{2_{\alpha}^{*}}}
 {2_{\alpha}^{*}}\|u\|^{2_{\alpha}^{*}-2}\Big)\|u\|^{2}.
\end{split}
\end{equation}
Set
\[
\theta = \Big(\frac{2_{\alpha}^{*}}{8C_{\varepsilon}K^{2_{\alpha}^{*}}}
\Big)^{\frac{1}{2_{\alpha}^{*}-2}}.
\]
Taking $\|u\| = \theta$, it follows from \eqref{6} that
\[
I(u) \geq  \frac{1}{8}\theta^{2} > 0,
\]
then (1) is proved.

(2) By (A5), for any $M > 0$, there exists a $T_0 > 0$ such that
\[
F(x,t) \geq M t^{2} > 0
\]
for all $t \geq T_0$.
By the continuity of $F$, there exists $C > 0$ such that
\[
F(x,t) \geq Mt ^{2} - C
\]
for all $(x, t)\in \mathbb{R}^{N} \times \mathbb{R}^{+}$.
We choose $\varphi\in C^{\infty}_0(\mathbb{R}^{N})$ with $\varphi\geq 0$, $\|\varphi\| = 1$ and $\operatorname{supp}(\varphi)\subset B(0,R)$ for some
 $R > 0$. we have that for $M\int_{B(0,R)}\varphi^{2}dx >1/2$.

For $t > 0$ large enough, it follows from (A5) that
$F(x,t\varphi) \geq 0$.
Hence,
\begin{align*}
I(t\varphi)
&=  \frac{t^{2}}{2}\|\varphi\|^{2} - \int_{\mathbb{R}^{N}}F(x,t\varphi)dx\\
&\leq  \frac{t^{2}}{2} - \int_{B(0,R)}F(x,t\varphi)dx\\
&\leq  t^{2}\Big(\frac{1}{2} - M\int_{B(0,R)}\varphi^{2}dx\Big) + C|B(0,R)|.
\end{align*}
Choosing $\|e\| = \|t\varphi\|> \rho$,  we have $I(e) < 0$, then (2) is proved.
\end{proof}

\section{Proof of main results}

\begin{proof}[Proof of Theorem \ref{thm1.1}]
 By Lemmas \ref{lem2.4} and  \ref{lem2.7}, it is easy to obtain a nontrivial 
critical point $u_0$ of $I$ by the mountain pass theorem, which implies that
\[
c = \inf_{\gamma\in \Gamma}\max_{t\in(0,1)}I(\gamma(t)) = I(u_0),
\]
where $\Gamma = \{\gamma \in C([0,1],E),\gamma(0) = 0, \gamma(1) = e\}$
and $I(u_0) \geq \theta > 0$.
Let
\[
\mathcal{N} = \{ u\in E \backslash \{0\}: I'(u) = 0\},
\]
since $u_0 \in \mathcal{N}$, $\mathcal{N} \neq \emptyset$.
Then we claim that $I$ is bound from below on $\mathcal{N}$, moreover,
there exists $d > 0$ such that
$I(u)> d$ for every $u \in \mathcal{N}$.
If not, there exists a sequence $\{u_n\} \subset \mathcal{N}$ such that
\[
I(u_n) < \frac{1}{n},\quad \forall n \in \mathbb{N}^{+}.
\]
Using \eqref{5}, we know that
\[
\frac{1}{n} > I(u_n) \geq \frac{1}{2}
\left(1 - \varepsilon K^{2}\right)\|u_n\|^{2} -
\frac{C_{\varepsilon}K^{2^{*}_{\alpha}}}{2^{*}_{\alpha}}\|u_n\|^{2^{*}_{\alpha}}.
\]
Since $u_n\in \mathcal{N}$, $I'(u_n) = 0$, it follows from \eqref{4} that
\begin{equation}\label{80}
\|u_n\|^{2} = \int_{\mathbb{R}^{N}}f(x,u_n^{+})u_n^{+}dx
\leq \varepsilon K^{2}\|u_n\|^{2} +
C_{\varepsilon}K^{2^{*}_{\alpha}}\|u_n\|^{2^{*}_{\alpha}}.
\end{equation}
Thus,
\[
\frac{1}{n} > \big(\frac{1}{2}-\frac{1}{2^{*}_{\alpha}}\big)
(1 - \varepsilon K^{2})\|u_n\|^{2}.
\]
By the definition of $\varepsilon < \frac{1}{2K^{2}}$, we obtain
\[
\frac{1}{n} > \big(1-\frac{2}{2^{*}_{\alpha}}\big)\|u_n\|^{2}.
\]
Since $2 < 2^{*}_{\alpha}$, it is easy to see that
$\|u_n\| \to 0$. From \eqref{80} and $\varepsilon < \frac{1}{2K^{2}}$, we know
\[
\|u_n\|^{2} < \frac{1}{2}\|u_n\|^{2} +
C_{\varepsilon}K^{2^{*}_{\alpha}}\|u_n\|^{2^{*}_{\alpha}},
\]
so there exists $C_{}$ such that $\|u_n\| > C_{}$. It is a contradiction.
Hence, there exists $c_{1} > 0$ such that
\[
c_{1} = \inf_{u\in \mathcal{N}}I(u).
\]
Clearly, $c_{1} \leq c$. Let $\{v_n\} \subset\mathcal{N}$ be a minimizing
sequence for $c_{1}$, so $\{v_n\}$ is a $(Ce)_{c_{1}}$ sequence.
By Lemma \ref{lem2.4}, $\{v_n\}$ is bounded in $E$ and it has a convergence
subsequence $\{v_n\}$ such that $v_n\to u_2$ in $E$.
Since $u_2\in \mathcal{N}$, we know that
\[
\langle I(u_2), u_2^{-}\rangle = -\|u_2^{-}\|^{2} = 0,
\]
therefore, $u_2$ is a non-negative ground state solution.
\end{proof}

\begin{proof}[Proof of Theorem \ref{thm1.2}]
 Since the $(Ce)_{c}$ condition is satisfied by Lemma \ref{lem2.5} and the mountain
geometrical structure is proved by Lemma \ref{lem2.7}.
So we can get a ground state solution by using the same method as that of
Theorem \ref{thm1.1}.
\end{proof}

\begin{proof}[Proof of Theorem \ref{thm1.3}]
 Since the $(Ce)_{c}$ condition is satisfied by Lemma \ref{lem2.6} and the mountain
geometrical structure is proved by Lemma \ref{lem2.7}.
So we can get a ground state solution by using the same method as that
of Theorem \ref{thm1.1}.
\end{proof}

\subsection*{Acknowledgments}
This research was supported by
National Natural Science Foundation of China (No. 11471267).
The authors want to  thank the anonymous  referees for their valuable comments.


\begin{thebibliography}{99}

\bibitem{LA1} N. Laskin;
\emph{Fractional quantum mechanics and L\'evy path integrals},
 Phys. Lett. 268 (2000), 298--305.

\bibitem{TA} J. Tan;
\emph{The Brezis-Nirenberg type problem involving the square root of the Laplacian},
Calc. Var. 42 (2011), 21--41.

\bibitem{BA} B. Barrios, E. Colorado, A. de Pablo, U. S\'anchez;
\emph{On some critical problems for the fractional Laplacian operator}, J.
    Differential Equations 252 (2012), 6133--162.

\bibitem{CHE} M. Cheng;
\emph{Gound state for the fractional Schr\"odinger equation with unbounded potential},
 J. Math. Phys. 53 (2012), 043507.

\bibitem{DI} S. Dipierro, G. Palatucci, E. Valdinoci;
\emph{Existence and symmetry results for a Schr\"oinger
type problem involving the fractional Laplacian},
Le Matematiche. LXVIII (2013), 201--216.

\bibitem{FE} P. Felmer, A. Quaas, J. Tan;
\emph{Positive solutions of nonlinear Schr\"odinguer equation with the
fractional laplacian}, Proc. Roy. Soc. Edinburgh Sect. A 142 (2012), 1237--1262.

\bibitem{SE1} S. Secchi;
\emph{Ground state solutions for nonlinear fractional Schr\"oinger equations
in $\mathbb{R}^{N}$}, J. Math. Phys. 54 (2013), 031501.

\bibitem{CHA} X. J. Chang;
\emph{Gound state solutions of asymptotically linear fractional Schr\"odinger
equations}, J. Math. Phys. 54 (2013), 061504.

\bibitem{DI2}  E. Di Nezza, G. Patalluci, E. Valdinoci;
\emph{Hitchhiker's guide to the fractional Sobolev spaces}, Bull. Sci.
Math. 136 (2012), 521--573.

\bibitem{GU2} Q. Y. Guan, Z. N. Ma;
\emph{The reflected-symmetric stable processes and regional fractional Laplacian},
Probab. Theory Relat. Fields 134 (2006), 649--694.

\bibitem{GU} Q. Y. Guan;
\emph{Integration by parts formula for regional fractional Laplacian},
Commun. Math. Phys. 266 (2006), 289--329.

\bibitem{FE1} P. Felmer, C. Torres;
\emph{Non-linear Schr\"odinger equation with non-local regional diffusion},
Calc. Var., to appear.

\bibitem{FE2} P. Felmer, C. Torres;
\emph{Radial symmetry of ground states for a regional fractional nonlinear
schr\"odinger  equation}, Commun. Pure Appl. Anal. 13 (2014), 2395--2406.

\bibitem{CA} X. Cabr\'e, J. D\'avila;
\emph{Nonlinear equations for fractional Laplacians I: Regularity,
    maximum principles and Hamiltonian estimates},
Ann. Inst. Henri Poincare Nonliear Anal. 31 (2014), 23--53.

\end{thebibliography}
\end{document}