\documentclass[reqno]{amsart}
\usepackage{hyperref}
\usepackage{cite}

\AtBeginDocument{{\noindent\small
\emph{Electronic Journal of Differential Equations},
Vol. 2015 (2015), No. 84, pp. 1--11.\newline
ISSN: 1072-6691. URL: http://ejde.math.txstate.edu or http://ejde.math.unt.edu
\newline ftp ejde.math.txstate.edu}
\thanks{\copyright 2015 Texas State University - San Marcos.}
\vspace{9mm}}

\begin{document}
\title[\hfilneg EJDE-2015/84\hfil Ground state solutions]
{Ground state solutions for semilinear elliptic equations with zero
mass in $\mathbb{R}^N$}

\author[J. Liu, J.-F. Liao, C.-L. Tang \hfil EJDE-2015/84\hfilneg]
{Jiu Liu, Jia-Feng Liao, Chun-Lei Tang}

\address{Jiu Liu \newline
School  of  Mathematics  and  Statistics, Southwest University,
Chongqing 400715, China}
\email{jiuliu2011@163.com}

\address{Jia-Feng Liao \newline
School  of  Mathematics  and  Statistics, Southwest University,
Chongqing 400715, China.\newline
School of Mathematics and Computational Science,
Zunyi Normal College, Zunyi, \newline
Guizhou 563002, China}
\email{liaojiafeng@163.com}

\address{Chun-Lei Tang (corresponding author)\newline
School  of  Mathematics  and  Statistics, Southwest University,
Chongqing 400715, China}
\email{tangcl@swu.edu.cn, Phone +86 23 68253135, Fax +86 23 68253135}

\thanks{Submitted January 17, 2015. Published April 7, 2015.}
\subjclass[2000]{35J20, 35J61}
\keywords{ Semilinear elliptic equation; zero mass; Nehari manifold;
\hfill\break\indent ground state solution}

\begin{abstract}
 In this article, we study the  semilinear elliptic equation
 \begin{gather*}
 -\Delta u=|u|^{p(x)-2}u,  \quad x\in \mathbb{R}^N\\
 u\in D^{1,2}(\mathbb{R}^N),
 \end{gather*}
 where $N\geq3$,
 $p(x)=\begin{cases}
 p,  &x\in\Omega\\
 2^*,  &x\not\in\Omega,
 \end{cases}$
 with $2<p<2^*:=2N/(N-2)$, $\Omega\subset\mathbb{R}^N$ is a bounded
 set with nonempty interior. By using the Nehari manifold, we obtain
 a positive ground state solution.
\end{abstract}

\maketitle
\numberwithin{equation}{section}
\newtheorem{theorem}{Theorem}[section]
\newtheorem{lemma}[theorem]{Lemma}
\allowdisplaybreaks


\section{Introduction and statement of main result}

Considering the  semilinear elliptic equation
\begin{equation}\label{formula1}
\begin{gathered}
-\Delta u=|u|^{2^*-2}u+\chi_\Omega(x)(|u|^{p-2}u-|u|^{2^*-2}u),  \quad
x\in \mathbb{R}^N\\
 u\in D^{1,2}(\mathbb{R}^N),
\end{gathered}
\end{equation}
where $N\geq3$, $2<p<2^*:=2N/(N-2)$, $\Omega\subset\mathbb{R}^N$
is  a bounded set with nonempty interior and
\begin{equation} \label{formula20}
\chi_\Omega(x)=\begin{cases}
 1,  &x\in\Omega\\
 0.  &x\not\in\Omega
\end{cases}
\end{equation}
The well-known semilinear elliptic equation with zero mass is
\begin{equation}\label{formula2}
\begin{gathered}
-\Delta u=|u|^{2^*-2}u,  \quad x\in \mathbb{R}^N\\
 u\in  D^{1,2}(\mathbb{R}^N),
\end{gathered}
\end{equation}
where $N\geq3$, which has been studied very intensely (see \cite{AU,CA,TA})
and the explicit expression of positive solutions was given.
Of course, the semilinear elliptic equation with zero mass whose nonlinear
term with subcritical growth has also been investigated by many authors,
for example \cite{AL,AZ1,AZ2,BE}.

By a transformation,  \eqref{formula1} is equivalent to the following
 elliptic equation with variable exponent
\begin{equation} \label{formula0.1}
\begin{gathered}
-\Delta u=|u|^{p(x)-2}u,  \quad x\in \mathbb{R}^N\\
 u\in D^{1,2}(\mathbb{R}^N),
\end{gathered}
\end{equation}
where
\[
p(x)=\begin{cases}
 p,  &x\in\Omega\\
 2^*.  &x\not\in\Omega
\end{cases}
\]
The equations with variable exponent appear in various mathematical models,
for example: electrorheological fluids\cite{AC,RU},
nonlinear Darcy's law in porous medium \cite{AN},
image processing  \cite{CH}.
 Recently, these equations have been investigated by many authors,
see for example, \cite{AL0,CA1,MA,MI,PU}.
However, they did not consider problem \eqref{formula0.1}; thus we study it in
this article.
Our main result reads as follows.

\begin{theorem} \label{thm1.1}
 Assume that $N\geq3$, $2<p<2^*$ and $\chi_\Omega$ satisfies \eqref{formula20}.
Then equation \eqref{formula1} has a positive ground state solution.
\end{theorem}

If $p\in C(\mathbb{R}^N,[2,2^*])$ and $p\not\equiv2^*$, we do not yet know
whether equation \eqref{formula1} has solution.
We shall consider it in the future.

This article is organized as follows.
Section 2 contains some preliminaries.
Section 3 gives the proof of theorem \ref{thm1.1}.


\section{Preliminaries}

 In what follows, we  use the following notation.

\noindent $\bullet$ $D^{1,2}(\mathbb{R}^N)$ is the completion of
$C_0^\infty(\mathbb{R}^N)$ with respect to the norm
\[
\|u\|_{D^{1,2}(\mathbb{R}^N)}^2=\int_{\mathbb{R}^N}|\nabla u|^2\,dx.
\]
$\bullet$ $L^t(\mathbb{R}^N)$, $2\leq t<+\infty$, denotes a Lebesgue
space endowed with the norm
\[
|u|_t^t=\int_{\mathbb{R}^N}|u|^t\,dx.
\]
$\bullet$ $S$ denotes the best constant of Sobolev embedding
$D^{1,2}(\mathbb{R}^N)\hookrightarrow L^{2^*}(\mathbb{R}^N)$; that is,
\[
S|u|_{2^*}^2\leq\|u\|_{D^{1,2}(\mathbb{R}^N)}^2\quad \text{for all }
 u\in D^{1,2}(\mathbb{R}^N).
\]
$\bullet$ $D^{-1}$ is the dual space of $D^{1,2}(\mathbb{R}^N)$.\\
$\bullet$ $C$, $C_i$ denote various positive constants.

For equation \eqref{formula1}, the energy functional
$I:D^{1,2}(\mathbb{R}^N)\to \mathbb{R}$ is defined by
\begin{align*}
I(u)
&=\frac{1}{2}\|u\|_{D^{1,2}(\mathbb{R}^N)}^2
 -\frac{1}{2^*}\int_{\mathbb{R}^N}|u|^{2^*}\,dx
 -\frac{1}{p}\int_{\mathbb{R}^N}\chi_\Omega(x)|u|^{p}\,dx\\
&\quad +\frac{1}{2^*}\int_{\mathbb{R}^N}\chi_\Omega(x)|u|^{2^*}\,dx\\
&=\frac{1}{2}\|u\|_{D^{1,2}(\mathbb{R}^N)}^2
 -\frac{1}{p}\int_{\Omega}|u|^{p}\,dx
 -\frac{1}{2^*}\int_{\mathbb{R}^N\backslash\Omega}|u|^{2^*}\,dx.
\end{align*}
The H\"{o}lder and Sobolev inequalities imply
\begin{equation} \label{formula11}
\begin{aligned}
\int_{\Omega}|u|^{p}\,dx
&\leq |u|_{2^*,\Omega}^p(\operatorname{meas}\Omega)^{\frac{2^*-p}{2^*}}
\leq|u|_{2^*}^p(\operatorname{meas}\Omega)^{\frac{2^*-p}{2^*}}\\
&\leq S^{-\frac{p}{2}}(\operatorname{meas}\Omega)^{\frac{2^*-p}{2^*}}
\|u\|_{D^{1,2}(\mathbb{R}^N)}^p,
\end{aligned}
\end{equation}
where
\[
|u|_{s,\Omega}=\Big(\int_{\Omega}|u|^t\,dx\Big)^{1/t}, \quad
\forall  t\in[1,+\infty).
\]
Thus the functional $I$ is well defined.
By Lemma \ref{lem3.1} in next section, $I$ is of class
$C^1(D^{1,2}(\mathbb{R}^N),\mathbb{R})$ and satisfies
\begin{equation}\label{formula7}
\begin{aligned}
\langle I'(u),v\rangle
&= \int_{\mathbb{R}^N}\nabla u\cdot\nabla v\,dx
 -\int_{\mathbb{R}^N}|u|^{2^*-2}uv\,dx
 -\int_{\mathbb{R}^N}\chi_\Omega(x)|u|^{p-2}uv\,dx\\
&\quad +\int_{\mathbb{R}^N}\chi_\Omega(x)|u|^{2^*-2}uv\,dx\\
&= \int_{\mathbb{R}^N}\nabla u\cdot\nabla v\,dx-\int_{\Omega}|u|^{p-2}uv\,dx-\int_{\mathbb{R}^N\backslash\Omega}|u|^{2^*-2}uv\,dx,
\end{aligned}
\end{equation}
for all $u,v\in D^{1,2}(\mathbb{R}^N)$. Hence weak solutions of
 \eqref{formula1} correspond to the critical point of the functional $I$. Define
\[
\mathcal{N}:=\{u\in D^{1,2}(\mathbb{R}^N)\backslash\{0\}:J(u)=0\},\quad
m:=\inf_{u\in\mathcal{N}}I(u),
\]
where
\[
J(u)= \|u\|_{D^{1,2}(\mathbb{R}^N)}^2-\int_{\Omega}|u|^{p}\,dx
-\int_{\mathbb{R}^N\backslash\Omega}|u|^{2^*}\,dx.
\]
Since all solutions of  \eqref{formula1} belong to the manifold $\mathcal{N}$,
first we seek for the minimizer $u$ for $m$ and then we prove $u$ is a
 solution of equation \eqref{formula1}.

\section{Proof of Theorem \ref{thm1.1}}

 The proof relies on the following lemmas.

\begin{lemma} \label{lem3.1}
Assume that $N\geq3$, $2<p<2^*$ and $\chi_\Omega$ satisfies \eqref{formula20}.
Then the functional $I$ is of class $C^1(D^{1,2}(\mathbb{R}^N),\mathbb{R})$ and
$I'(\cdot)$ satisfies \eqref{formula7}.
\end{lemma}

\begin{proof}
 Define
\begin{align*}
\psi(u)=\frac{1}{p}\int_{\Omega}|u|^{p}\,dx
+\frac{1}{2^*}\int_{\mathbb{R}^N\backslash\Omega}|u|^{2^*}\,dx,
\end{align*}
we need only to prove $\psi\in C^1(D^{1,2}(\mathbb{R}^N),\mathbb{R})$.
 Let $u,h\in D^{1,2}(\mathbb{R}^N).$ Given $x\in\Omega$ and $0<|t|<1$,
by the mean value theorem, there exists $\lambda_1\in(0,1)$ such that
\[
\frac{\big||u+th|^{p}-|u|^{p}\big|}{t}
= p|u+\lambda_1th|^{p-1}|h|
\leq p(|u|+|h|)^{p-1}|h|.
\]
Similarly, given $x\in\mathbb{R}^N\backslash\Omega$ and $0<|t|<1$,
there exists $\lambda_2\in(0,1)$ such that
\[
\frac{\big||u+th|^{2^*}-|u|^{2^*}\big|}{t}
= 2^*|u+\lambda_2th|^{2^*-1}|h|
\leq 2^*(|u|+|h|)^{2^*-1}|h|.
\]
The H\"{o}lder inequality implies that
\begin{align*}
\int_{\Omega}(|u|+|h|)^{p-1}|h|\,dx
&\leq \Big(\int_{\Omega}(|u|+|h|)^{p}\,dx\Big)^{\frac{p-1}{p}}|h|_{p,\Omega}\\
&\leq (|u|_{p,\Omega}+|h|_{p,\Omega})^{p-1}|h|_{p,\Omega}
<+\infty
\end{align*}
and
\begin{align*}
\int_{\mathbb{R}^N\backslash\Omega}(|u|+|h|)^{2^*-1}|h|\,dx
&\leq \Big(\int_{\mathbb{R}^N\backslash\Omega}(|u|+|h|)^{2^*}\,dx
 \Big)^{\frac{2^*-1}{2^*}}|h|_{2^*,\mathbb{R}^N\backslash\Omega}\\
&\leq (|u|_{2^*,\mathbb{R}^N\backslash\Omega}+|h|_{2^*,
 \mathbb{R}^N\backslash\Omega})^{2^*-1}|h|_{2^*,\mathbb{R}^N\backslash\Omega}
 < +\infty\,.
\end{align*}
It follows from the Lebesgue theorem that
\begin{align*}
\langle\psi'(u),h\rangle
&= \lim_{t\to 0}\frac{\psi(u+th)-\psi(u)}{t}\\
&= \lim_{t\to 0}\int_{\Omega}\frac{|u+th|^{p}-|u|^{p}}{pt}\,dx
 +\lim_{t\to 0}\int_{\mathbb{R}^N\backslash\Omega}
 \frac{|u+th|^{2^*}-|u|^{2^*}}{2^*t}\,dx\\
&= \int_{\Omega}\lim_{t\to 0}\frac{|u+th|^{p}-|u|^{p}}{pt}\,dx
 +\int_{\mathbb{R}^N\backslash\Omega}\lim_{t\to 0}
 \frac{|u+th|^{2^*}-|u|^{2^*}}{2^*t}\,dx\\
&= \int_{\Omega}|u|^{p-2}uh\,dx+\int_{\mathbb{R}^N\backslash\Omega}|u|^{2^*-2}uh\,dx.
\end{align*}
Assume that $u_n\to  u$ in $D^{1,2}(\mathbb{R}^N)$, then $u_n\to  u$ in
$L^{2^*}(\mathbb{R}^N)$ and $L^{p}(\Omega)$. If follows from
 \cite[Theorem A.2 and A.4]{WI} that $|u_n|^{p-2}u_n\to |u|^{p-2}u$
in $L^{\frac{p}{p-1}}(\Omega)$ and $|u_n|^{2^*-2}u_n\to |u|^{2^*-2}u$ in
 $L^{\frac{2^*}{2^*-1}}(\mathbb{R}^N\backslash\Omega)$.
Hence combining the H\"{o}lder and Sobolev inequalities, we obtain
\begin{align*}
&\|\psi'(u_n)-\psi'(u)\|_{D^{-1}}\\
&\leq \sup_{\|\varphi\|_{D^{1,2}(\mathbb{R}^N)}=1,\varphi\in D^{1,2}
 (\mathbb{R}^N)}\Big|\int_{\Omega}(|u_n|^{p-2}u_n-|u|^{p-2}u)\varphi \,dx\Big|\\
&\quad +\sup_{\|\varphi\|_{D^{1,2}(\mathbb{R}^N)}=1,\varphi\in D^{1,2}
 (\mathbb{R}^N)}\Big|\int_{\mathbb{R}^N\backslash\Omega}
 (|u_n|^{2^*-2}u_n-|u|^{2^*-2}u)\varphi \,dx\Big|\\
&\leq C\big||u_n|^{p-2}u_n-|u|^{p-2}u\big|_{\frac{p}{p-1},\Omega}
+C\big||u_n|^{2^*-2}u_n-|u|^{2^*-2}
 u\big|_{\frac{2^*}{2^*-1},\mathbb{R}^N\backslash\Omega}\\
&= o(1).
\end{align*}
Thus $\psi$ is $C^1.$ It is obvious that $I'(\cdot)$ satisfies \eqref{formula7}.
The proof is complete.
\end{proof}

\begin{lemma} \label{lem3.2}
Assume that $N\geq3$, $2<p<2^*$ and $\chi_\Omega$ satisfies \eqref{formula20}.
Then for any $u\in D^{1,2}(\mathbb{R}^N)\backslash\{0\}$, there exists $t_u>0$
such that $t_uu\in\mathcal{N}$.
\end{lemma}

\begin{proof}
 For any $u\in D^{1,2}(\mathbb{R}^N)\backslash\{0\}$, define
\[
f(t):=I(tu)=\frac{t^2}{2}\|u\|_{D^{1,2}(\mathbb{R}^N)}^2
-\frac{t^p}{p}\int_{\Omega}|u|^{p}\,dx-\frac{t^{2^*}}{2^*}
\int_{\mathbb{R}^N\backslash\Omega}|u|^{2^*}\,dx,\quad \forall t\in(0,+\infty).
\]
Then one has
\[
f'(t)t=\langle I'(tu),tu\rangle=t^2\|u\|_{D^{1,2}(\mathbb{R}^N)}^2
-t^{p}\int_{\Omega}|u|^{p}\,dx-t^{2^*}\int_{\mathbb{R}^N\backslash\Omega}|u|^{2^*}\,dx.
\]
Combining $2<p<2^*$, we have $f'(t)t>0$ for $t>0$ small enough and $f'(t)t<0$
for $t>0$ large enough. Thus there exists $t_u>0$ such that
 $f'(t_u)t_u=\langle I'(t_uu),t_uu\rangle=0$. That is $t_uu\in\mathcal{N}$.
The proof is complete.
\end{proof}


\begin{lemma} \label{lem3.3}
 Assume that $N\geq3$, $2<p<2^*$ and $\chi_\Omega$ satisfies \eqref{formula20}.
Then $m>0$.
\end{lemma}

\begin{proof}
 For any $u\in\mathcal{N}$, one has
\begin{align*}
\|u\|_{D^{1,2}(\mathbb{R}^N)}^2
&= \int_{\Omega}|u|^{p}\,dx+\int_{\mathbb{R}^N\backslash\Omega}|u|^{2^*}\,dx\\
&\leq C\|u\|_{D^{1,2}(\mathbb{R}^N)}^p+C\|u\|_{D^{1,2}(\mathbb{R}^N)}^{2^*},
\end{align*}
which implies that there exists $\alpha>0$ such that
\begin{equation} \label{formula8}
\|u\|_{D^{1,2}(\mathbb{R}^N)}\geq\alpha,\quad \forall u\in\mathcal{N}.
\end{equation}
Thus for any $u\in\mathcal{N}$, we have
\begin{equation}\label{formula10}
\begin{aligned}
 I(u)&=  I(u)-\frac{1}{p}\langle I'(u),u\rangle\\
&= \big(\frac{1}{2}-\frac{1}{p}\big)\|u\|_{D^{1,2}(\mathbb{R}^N)}^2
+\big(\frac{1}{p}-\frac{1}{2^*}\big)\int_{\mathbb{R}^N\backslash\Omega}|u|^{2^*}\,dx\\
&\geq \big(\frac{1}{2}-\frac{1}{p}\big)\|u\|_{D^{1,2}(\mathbb{R}^N)}^2\\
&\geq \big(\frac{1}{2}-\frac{1}{p}\big)\alpha^2.
\end{aligned}
\end{equation}
Hence $m>0$. The proof is complete.
\end{proof}

\begin{lemma} \label{lem3.4}
 Assume that $N\geq3$, $2<p<2^*$ and $\chi_\Omega$ satisfies \eqref{formula20}.
Then for any $u\in\mathcal{N}$, $J'(u)\neq0$.
\end{lemma}

\begin{proof} By \eqref{formula8}, for any $u\in\mathcal{N}$, one has
\begin{equation}\label{formula9}
\begin{aligned}
\langle J'(u),u\rangle
&= \langle J'(u),u\rangle-pJ(u)\\
&= (2-p)\|u\|_{D^{1,2}(\mathbb{R}^N)}^2-(2^*-p)
\int_{\mathbb{R}^N\backslash\Omega}|u|^{2^*}\,dx\\
&\leq (2-p)\|u\|_{D^{1,2}(\mathbb{R}^N)}^2\\
&\leq (2-p)\alpha^2
<0.
\end{aligned}
\end{equation}
Hence the proof is complete.
\end{proof}

\begin{lemma} \label{lem3.5}
 Assume that $N\geq3$, $2<p<2^*$ and $\chi_\Omega$ satisfies \eqref{formula20}.
Suppose that $u\in\mathcal{N}$ and $I(u)=m$. Then $u$ is a solution of
\eqref{formula1}.
\end{lemma}

\begin{proof}
Assume that $u\in\mathcal{N}$ and $I(u)=m$. Then by the Lagrange multiplier rule,
there exists $\lambda\in\mathbb{R}$ such that
$I'(u)=\lambda J'(u)$, which implies that
$0=\langle I'(u),u\rangle=\lambda \langle J'(u),u\rangle$.
By Lemma \ref{lem3.4},
 we obtain $\lambda=0$. Hence $I'(u)=0$. The proof is complete.
\end{proof}

\begin{lemma} \label{lem3.6}
Assume that $N\geq3$, $2<p<2^*$ and $\chi_\Omega$ satisfies \eqref{formula20}.
Then there exists a bounded sequence $\{u_n\}\subset\mathcal{N}$
satisfying $I(u_n)\to  m$ and $I'(u_n)\to 0$ in $D^{-1}$.
\end{lemma}

\begin{proof}
 By the Ekeland variational principle in \cite{WI}, there exist
$\{u_n\}\subset\mathcal{N}$ and $\{\lambda_n\}\subset\mathbb{R}$ such that
 $I(u_n)\to  m$ and $I'(u_n)-\lambda_n J'(u_n)\to 0$ in $D^{-1}$.
By \eqref{formula10}, one has
\[
I(u_n)
= I(u_n)-\frac{1}{p}\langle I'(u_n),u_n\rangle
\geq \Big(\frac{1}{2}-\frac{1}{p}\Big)\|u_n\|_{D^{1,2}(\mathbb{R}^N)}^2,
\]
which implies $\{u_n\}$ is bounded in $D^{1,2}(\mathbb{R}^N)$. Then we have
\[
0= \langle I'(u_n),u_n\rangle
= \lambda_n\langle J'(u_n),u_n\rangle+o(1).
\]
Combining \eqref{formula9}, we obtain $\lambda_n\to 0$. For any
$\varphi\in D^{1,2}(\mathbb{R}^N)$ with $\|\varphi\|_{D^{1,2}(\mathbb{R}^N)}=1$,
it follows from \eqref{formula11}, the H\"{o}lder and Sobolev inequalities that
\begin{align*}
\big|\int_{\Omega}|u_n|^{p-2}u_n\varphi \,dx\big|
&\leq \Big(\int_{\Omega}|u_n|^p\Big)^{\frac{p-1}{p}}
 \Big(\int_{\Omega}|\varphi|^p\Big)^{\frac{1}{p}}\\
&\leq \Big(S^{-\frac{p}{2}}(\operatorname{meas}\Omega)^{\frac{2^*-p}{2^*}}\|u_n\|_{D^{1,2}(\mathbb{R}^N)}^p\Big)^{\frac{p-1}{p}}
\Big(|\varphi|_{2^*}^p(\operatorname{meas}\Omega)^{\frac{2^*-p}{2^*}}
 \Big)^{\frac{1}{p}}\\
&\leq C\|u_n\|_{D^{1,2}(\mathbb{R}^N)}^{p-1}\|\varphi\|_{D^{1,2}(\mathbb{R}^N)}
\leq C.
\end{align*}
Thus combining the H\"{o}lder and Sobolev inequalities, we have
\begin{align*}
\|J'(u_n)\|_{D^{-1}}
&= \sup_{\|\varphi\|_{D^{1,2}(\mathbb{R}^N)}=1,\varphi\in D^{1,2}(\mathbb{R}^N)}|\langle J'(u_n),\varphi\rangle|\\
&= \sup_{\|\varphi\|_{D^{1,2}(\mathbb{R}^N)}=1,\varphi\in D^{1,2}(\mathbb{R}^N)}
 \Big|2\int_{\mathbb{R}^N}\nabla u_n\cdot\nabla\varphi \,dx
 -p\int_{\Omega}|u_n|^{p-2}u_n\varphi \,dx\\
&\quad -2^*\int_{\mathbb{R}^N\backslash\Omega}|u_n|^{2^*-2}u_n\varphi \,dx\Big|\\
&\leq \sup_{\|\varphi\|_{D^{1,2}(\mathbb{R}^N)}=1,\varphi\in D^{1,2}
 (\mathbb{R}^N)}[2\|u_n\|_{D^{1,2}(\mathbb{R}^N)}\|\varphi\|_{D^{1,2}
 (\mathbb{R}^N)}+C\\
&\quad +C\|u_n\|_{D^{1,2}(\mathbb{R}^N)}^{2^*-1}\|\varphi\|_{D^{1,2}(\mathbb{R}^N)}]
\leq C.
\end{align*}
Hence we obtain
\[
\|I'(u_n)\|_{D^{-1}}\leq\|I'(u_n)-\lambda_n J'(u_n)\|_{D^{-1}}
+|\lambda_n| \|J'(u_n)\|_{D^{-1}}=o(1).
\]
The proof is complete.
\end{proof}

If $p(x)\equiv2^*$, equation \eqref{formula0.1} reduces to \eqref{formula2}.
It is well known that  \eqref{formula2} has ground state solution
\begin{equation}\label{formula6}
v(x)=\frac{C_N}{(1+|x|^2)^{\frac{N-2}{2}}},
\end{equation}
where $C_N:=[N(N-2)]^{\frac{N-2}{4}}$ and $v$ satisfies
\begin{align*}
\int_{\mathbb{R}^N}|\nabla v|^2\,dx=\int_{\mathbb{R}^N}|v|^{2^*}\,dx
=S^{N/2}.
\end{align*}
Let the energy functional of  \eqref{formula2} be
\begin{align*}
I_\infty(u)=\frac{1}{2}\|u\|_{D^{1,2}(\mathbb{R}^N)}^2
-\frac{1}{2^*}\int_{\mathbb{R}^N}|u|^{2^*}\,dx.
\end{align*}
Then we have
\[
I_\infty(v)=I_\infty(v)-\frac{1}{2^*}\langle I'_\infty(v),v\rangle
=\frac{1}{N}\int_{\mathbb{R}^N}|\nabla v|^2\,dx=\frac{1}{N}S^{N/2}.
\]
Now for the energy $m$, we make the following estimation.

\begin{lemma} \label{lem3.7}
Assume that $N\geq3$, $2<p<2^*$ and $\chi_\Omega$ satisfies \eqref{formula20}.
 Then
$m<\frac{1}{N}S^{N/2}$.
\end{lemma}

\begin{proof}
Inspired by the idea in \cite{AL1,MO}. For ground state solution $v$ of equation
\eqref{formula2}, we define $v_n(x):=v(x+x_n)$, where $x_n:=(0,0,\dots ,0,n)$.
Thus $\|v_n\|_{D^{1,2}(\mathbb{R}^N)}=\|v\|_{D^{1,2}(\mathbb{R}^N)}
=S^{\frac{N}{4}}$
and then $v_n\rightharpoonup u$ in $D^{1,2}(\mathbb{R}^N)$, $v_n\to  u$ in
$L^p_{\mathrm{loc}}(\mathbb{R}^N)$, $v_n(x)\to  u(x)$ a.e. in $\mathbb{R}^N$.
Since for any $x\in\mathbb{R}^N$, $v_n(x)\to 0$, $u=0$. 
By Lemma \ref{lem3.2}, there exists
$t_n\in(0,+\infty)$ such that $t_nv_n\in\mathcal{N}$.
Then one has
\begin{equation} \label{formula4}
\|v_n\|_{D^{1,2}(\mathbb{R}^N)}^2
=t_n^{p-2}\int_{\Omega}|v_n|^{p}\,dx+t_n^{2^*-2}\int_{\mathbb{R}^N\backslash\Omega}|v_n|^{2^*}\,dx.
\end{equation}
By \eqref{formula11} and \eqref{formula4}, one has
\begin{align*}
\|v\|_{D^{1,2}(\mathbb{R}^N)}^2
&= \|v_n\|_{D^{1,2}(\mathbb{R}^N)}^2\\
&\leq C(t_n^{p-2}\|v_n\|_{D^{1,2}(\mathbb{R}^N)}^p+t_n^{2^*-2}
 \|v_n\|_{D^{1,2}(\mathbb{R}^N)}^{2^*})\\
&= C(t_n^{p-2}\|v\|_{D^{1,2}(\mathbb{R}^N)}^p+t_n^{2^*-2}
 \|v\|_{D^{1,2}(\mathbb{R}^N)}^{2^*}),
\end{align*}
which indicates that $t_n$ cannot appraoch zero,
 that is $t_n\geq t_0$ for some $t_0>0$. Since $\Omega$ is bounded,
 there exists $R>0$ such that $\Omega\subset B_R:=\{x\in\mathbb{R}^N:|x|<R\}$.
Since for $n$ large enough,
\begin{equation} \label{formula12}
\int_{|x-x_n|<R}\frac{1}{(1+|x|^2)^N}\,dx
\leq\int_{|x-x_n|<R}\frac{2^{N}}{n^{2N}}\,dx
=\frac{2^{N}}{n^{2N}}\operatorname{meas}B_R=o(1),
\end{equation}
we have
\begin{align*}
\int_{\mathbb{R}^N}|v|^{2^*}\,dx
&= \int_{\mathbb{R}^N}|v_n|^{2^*}\,dx\\
&\geq \int_{\mathbb{R}^N\backslash\Omega}|v_n|^{2^*}\,dx\\
&\geq \int_{|x|\geq R}|v_n|^{2^*}\,dx\\
&= C_N^{2^*}\int_{|x|\geq R}\frac{1}{(1+|x+x_n|^2)^N}\,dx\\
&= C_N^{2^*}\int_{|x-x_n|\geq R}\frac{1}{(1+|x|^2)^N}\,dx\\
&= C_N^{2^*}\int_{\mathbb{R}^N}\frac{1}{(1+|x|^2)^N}\,dx
 -C_N^{2^*}\int_{|x-x_n|<R}\frac{1}{(1+|x|^2)^N}\,dx\\
&= \int_{\mathbb{R}^N}|v|^{2^*}\,dx+o(1).
\end{align*}
Thus one has
\begin{equation}\label{formula5}
\int_{\mathbb{R}^N\backslash\Omega}|v_n|^{2^*}\,dx
=\int_{\mathbb{R}^N}|v|^{2^*}\,dx+o(1)=S^{N/2}+o(1).
\end{equation}
It follows from \eqref{formula4} that
\[
\|v_n\|_{D^{1,2}(\mathbb{R}^N)}^2
\Big(\int_{\mathbb{R}^N\backslash\Omega}|v_n|^{2^*}\,dx\Big)^{-1}
\geq t_n^{2^*-2},
\]
which implies
\[
\limsup_{n\to \infty}t_n^{2^*-2}\leq\|v\|_{D^{1,2}(\mathbb{R}^N)}^2
S^{-\frac{N}{2}}=1.
\]
Thus up to a subsequence, one has $t_n\to  T\in(t_0,1]$. Notice that
\begin{align*}
\int_{\Omega}|v_n|^{p}\,dx=o(1).
\end{align*}
By \eqref{formula4} and \eqref{formula5}, one has $S^{N/2}=T^{2^*-2}S^{N/2}$.
Thus $T=1$.
From \eqref{formula5} it follows that
\[
\int_{\Omega}|v_n|^{2^*}\,dx=o(1).
\]
We claim that
\begin{align*}
\frac{\frac{t_n^{2^*}}{2^*}\int_{\Omega}|v_n|^{2^*}\,dx}
{\frac{t_n^p}{p}\int_{\Omega}|v_n|^{p}\,dx}\to 0.
\end{align*}
Indeed, by \eqref{formula6} and \eqref{formula12}, for $n$ large enough, one has
\begin{align*}
\int_{\Omega} |v_n|^{2^*} dx
&\leq C_N^{2^*}\int_{B_R}\frac{1}{(1+|x+x_n|^2)^N}\,dx\\
&= C_N^{2^*}\int_{|x-x_n|<R}\frac{1}{(1+|x|^2)^N}\,dx\\
&= \frac{C_N^{2^*}2^{N}}{n^{2N}}\operatorname{meas}B_R.
\end{align*}
Since the interior of $\Omega$ is nonempty, there exist $z_0\in\mathbb{R}^N$
and $r>0$ such that $B_r(z_0):=\{x\in\mathbb{R}^N:|x-z_0|<r\}\subset\Omega$.
Thus  for $n$ large enough, one has
\begin{align*}
\int_{\Omega}|v_n|^{p}\,dx
&\geq C_N^{p}\int_{B_r(z_0)}\frac{1}{(1+|x+x_n|^2)^{\frac{(N-2)p}{2}}}\,dx\\
&\geq C_N^{p}\int_{B_r(z_0)}\frac{1}{2^{\frac{(N-2)p}{2}}n^{(N-2)p}}\,dx\\
&= \frac{C_N^{p}2^{\frac{(2-N)p}{2}}}{n^{(N-2)p}}\operatorname{meas}B_r.
\end{align*}
Then we obtain
\[
\frac{\int_{\Omega}|v_n|^{2^*}\,dx}{\int_{\Omega}|v_n|^{p}\,dx}
\leq C_N'\frac{1}{n^{2N-(N-2)p}}=o(1),
\]
since $p<\frac{2N}{N-2}$. Combining $t_n\to 1$, we implies the claim holds.
 By calculations, one has
\[
\frac{\frac{1-t_n^2}{2}}{\frac{1-t_n^{2^*}}{2^*}}
=\frac{2^*(1-t_n^2)}{2(1-t_n^{2^*})}\to 1.
\]
Recall that $t_nv_n\in\mathcal{N}$. Hence for $n$ large enough, one has
\begin{align*}
m
&\leq I(t_nv_n)\\
&= \frac{t_n^2}{2}\|v_n\|_{D^{1,2}(\mathbb{R}^N)}^2-\frac{t_n^p}{p}
 \int_{\Omega}|v_n|^{p}\,dx
-\frac{t_n^{2^*}}{2^*}\int_{\mathbb{R}^N\backslash\Omega}|v_n|^{2^*}\,dx\\
&= \frac{t_n^2}{2}\|v_n\|_{D^{1,2}(\mathbb{R}^N)}^2-\frac{t_n^{2^*}}{2^*}
 \int_{\mathbb{R}^N}|v_n|^{2^*}\,dx-\frac{t_n^p}{p}
 \int_{\Omega}|v_n|^{p}\,dx+\frac{t_n^{2^*}}{2^*}\int_{\Omega}|v_n|^{2^*}\,dx\\
&= I_\infty(v_n)-\frac{1-t_n^2}{2}\|v_n\|_{D^{1,2}(\mathbb{R}^N)}^2
 +\frac{1-t_n^{2^*}}{2^*}\int_{\mathbb{R}^N}|v_n|^{2^*}\,dx\\
&\quad -\frac{t_n^p}{p}\int_{\Omega}|v_n|^{p}\,dx
 +\frac{t_n^{2^*}}{2^*}\int_{\Omega}|v_n|^{2^*}\,dx\\
&= S^{N/2}+\Big(\frac{1-t_n^{2^*}}{2^*}-\frac{1-t_n^2}{2}\Big)S^{N/2}
 -\frac{t_n^p}{p}\int_{\Omega}|v_n|^{p}\,dx
 +\frac{t_n^{2^*}}{2^*}\int_{\Omega}|v_n|^{2^*}\,dx\\
&< S^{N/2}.
\end{align*}
The proof is complete.
\end{proof}

\begin{lemma} \label{lem3.8}
Assume that $N\geq3$, $2<p<2^*$ and $\chi_\Omega$ satisfies \eqref{formula20}.
Suppose that the sequence $\{u_n\}\subset\mathcal{N}$ is bounded in
$D^{1,2}(\mathbb{R}^N)$ and satisfies $I(u_n)\to  m<\frac{1}{N}S^{N/2}$
and $I'(u_n)\to 0$ in $D^{-1}$. Then there exists $u\in D^{1,2}(\mathbb{R}^N)$
such that up to a subsequence, $u_n\to  u$ in $D^{1,2}(\mathbb{R}^N)$.
\end{lemma}

\begin{proof}
Since $\{u_n\}\subset D^{1,2}(\mathbb{R}^N)$ is a bounded, up to a subsequence,
there exists $u\in D^{1,2}(\mathbb{R}^N)$ such that
$u_n\rightharpoonup u$ in $D^{1,2}(\mathbb{R}^N)$, $u_n\to  u$ in
$L^{p}(\Omega)$ and $u_n(x)\to  u(x)$ a.e. in $\mathbb{R}^N$.
For any $v\in D^{1,2}(\mathbb{R}^N)$, by $I'(u_n)\to  0$ in $D^{-1}$, one has
\begin{align*}
0&= \langle I'(u_n),v\rangle+o(1)\\
&= \int_{\mathbb{R}^N}\nabla u_n\cdot\nabla v\,dx
 -\int_{\Omega}|u_n|^{p-2}u_nv\,dx
 -\int_{\mathbb{R}^N\backslash\Omega}|u_n|^{2^*-2}u_nv\,dx+o(1)\\
&= \int_{\mathbb{R}^N}\nabla u\cdot\nabla v\,dx
 -\int_{\Omega}|u|^{p-2}uv\,dx-\int_{\mathbb{R}^N\backslash\Omega}|u|^{2^*-2}uv\,dx\\
&= \langle I'(u),v\rangle.
\end{align*}
Thus we have
\begin{align*}
I(u)
&= I(u)-\frac{1}{p}\langle I'(u),u\rangle\\
&= \big(\frac{1}{2}-\frac{1}{p}\big)\|u\|_{D^{1,2}(\mathbb{R}^N)}^2
 +\big(\frac{1}{p}-\frac{1}{2^*}\big)\int_{\mathbb{R}^N\backslash\Omega}|u|^{2^*}\,dx\\
&\geq \big(\frac{1}{2}-\frac{1}{p}\big)\|u\|_{D^{1,2}(\mathbb{R}^N)}^2
\geq 0.
\end{align*}
Define $v_n=u_n-u$. Thus one has
\[
\|u_n\|_{D^{1,2}(\mathbb{R}^N)}^2
=\|v_n\|_{D^{1,2}(\mathbb{R}^N)}^2+\|u\|_{D^{1,2}(\mathbb{R}^N)}^2+o(1).
\]
The Brezis-Lieb lemma implies
\[
\int_{\Omega}|u_n|^{p}\,dx=\int_{\Omega}|v_n|^{p}\,dx
+\int_{\Omega}|u|^{p}\,dx+o(1)=\int_{\Omega}|u|^{p}\,dx+o(1)
\]
and
\[
\int_{\mathbb{R}^N\backslash\Omega}|u_n|^{2^*}\,dx
=\int_{\mathbb{R}^N\backslash\Omega}|v_n|^{2^*}\,dx
+\int_{\mathbb{R}^N\backslash\Omega}|u|^{2^*}\,dx+o(1).
\]
Combining this with $I(u_n)\to  m$, we obtain
\begin{equation}\label{formula3}
\begin{aligned}
m&= \frac{1}{2}\|v_n\|_{D^{1,2}(\mathbb{R}^N)}^2
 -\frac{1}{2^*}\int_{\mathbb{R}^N\backslash\Omega}|v_n|^{2^*}\,dx+I(u)+o(1)\\
&\geq \frac{1}{2}\|v_n\|_{D^{1,2}(\mathbb{R}^N)}^2
 -\frac{1}{2^*}\int_{\mathbb{R}^N\backslash\Omega}|v_n|^{2^*}\,dx+o(1).
\end{aligned}
\end{equation}
It follows from $\langle I'(u_n),u_n\rangle=0$ and $I'(u)=0$ that
\begin{align*}
0&= \|v_n\|_{D^{1,2}(\mathbb{R}^N)}^2
 -\int_{\mathbb{R}^N\backslash\Omega}|v_n|^{2^*}\,dx+\langle I'(u),u\rangle+o(1)\\
&= \|v_n\|_{D^{1,2}(\mathbb{R}^N)}^2
 -\int_{\mathbb{R}^N\backslash\Omega}|v_n|^{2^*}\,dx+o(1).
\end{align*}
Up to a subsequence, we assume that
\begin{align*}
\|v_n\|_{D^{1,2}(\mathbb{R}^N)}^2+o(1)=b
=\int_{\mathbb{R}^N\backslash\Omega}|v_n|^{2^*}\,dx+o(1).
\end{align*}
Thus we have
\begin{align*}
Sb^{2/2^*}
&= S\Big(\int_{\mathbb{R}^N\backslash\Omega}|v_n|^{2^*}\,dx\Big)^{2/2^*}+o(1)\\
&\leq  S\Big(\int_{\mathbb{R}^N}|v_n|^{2^*}\,dx\Big)^{2/2^*}+o(1)\\
&\leq \|v_n\|_{D^{1,2}(\mathbb{R}^N)}^2+o(1)
= b.
\end{align*}
Assume that $b\neq0$. Then one has $b\geq S^{N/2}$.
From \eqref{formula3}, we obtain
\begin{align*}
m\geq\big(\frac{1}{2}-\frac{1}{2^*}\big)b\geq\frac{1}{N}S^{N/2},
\end{align*}
which is a contradiction. Hence $b=0$, and the proof is complete.
\end{proof}

\begin{proof}[Proof of Theorem \ref{thm1.1}]
 By Lemmas \ref{lem3.3}, \ref{lem3.6} and \ref{lem3.7},
there exists a bounded sequence
 $\{u_n\}\subset\mathcal{N}$ satisfying $I(u_n)\to  m\in(0,\frac{1}{N}S^{N/2})$
and $I'(u_n)\to 0$ in $D^{-1}$.
Lemma \ref{lem3.8} implies that there exists $u\in D^{1,2}(\mathbb{R}^N)$ such that
up to a subsequence, $u_n\to  u$ in $D^{1,2}(\mathbb{R}^N)$.
Then $I(u)=m$ and $J(u)=0$. That is, $m$ is achieved by a
function $u\in D^{1,2}(\mathbb{R}^N)$. Since $I(|u|)=I(u)$ and
 $J(|u|)=J(u)$, we can assume that $u$ is nonnegative. Lemma \ref{lem3.5} implies
that $u\in D^{1,2}(\mathbb{R}^N)$ is a solution of equation \eqref{formula1}.
It follows from the definition of $m$ that $u\in D^{1,2}(\mathbb{R}^N)$
is a ground state solution of equation \eqref{formula1}.
It follows from the strongly maximum principle that $u>0$.
This completes the proof.
\end{proof}

\subsection*{Acknowledgments}
The authors would like to thank Professor C. O. Alves for his suggestioins.
This research was supported by the National Natural Science Foundation of China
(No. 11471267), by the Fundamental Research Funds for
the Central Universities (No. XDJK2015D015), and by the
 Natural Science Foundation of Education of Guizhou Province
(No. LKZS[2014]22)

\begin{thebibliography}{00}

\bibitem{AC} E. Acerbi, G. Mingione;
\emph{Regularity results for stationary
electrorheological fluids}, Arch. Ration. Mech. Anal. 164 (2002) 213--259.

\bibitem{AL0} C. O. Alves;
\emph{Existence of solution for a degenerate $p(x)$-Laplacian equation
in $\mathbb{R}^N$}, J. Math. Anal. Appl. 345 (2008) 731--742.

\bibitem{AL1} C. O. Alves, P. C. Carri\~{a}o, E. S. Medeiros;
\emph{Multiplicity of solutions for a class of a quasilinear problem in
exterior domains with Neumann conditions},
Abstr. Appl. Anal. (2004) 251--268.

\bibitem{AL} C. O. Alves, M. A. S. Souto, M. Montenegro;
\emph{Existence of solution for two classes of elliptic problems in
$\mathbb{R}^N$ with zero mass}, J. Differential Equations 252 (2012) 5735--5750.

\bibitem{AN} S. N. Antontsev, S. I. Shmarev;
\emph{Elliptic equations and systems with nonstandard growth conditions:
Existence, uniqueness and localization properties of solutions},
 Nonlinear Anal. 65 (2006) 722--755.

\bibitem{AU} T. Aubin;
\emph{Probl\'{e}mes isop\'{e}rim\'{e}triques et espaces de Sobolev},
J. Differential Geom. 11 (1976) 573--598.

\bibitem{AZ1} A. Azzollini, A. Pomponio;
\emph{On a ``zero mass'' nonlinear Schr\"{o}dinger equation},
Adv. Nonlinear Stud. 7 (2007) 599--628.

\bibitem{AZ2} A. Azzollini, A. Pomponio;
\emph{Compactness results and applications to some ``zero mass'' elliptic problems},
Nonlinear Anal. 69 (2008) 3559--3576.

\bibitem{BE} H. Berestycki, P. L. Lions;
\emph{Nonlinear scalar field equations, I-Existence of a ground state},
Arch. Ration. Mech. Anal. 82 (1983) 313--346.

\bibitem{CA} L. A. Caffarelli, B. Gidas, J. Spruck;
\emph{Asymptotic symmetry and local behavior of semilinear elliptic equations
 with critical Sobolev growth}, Comm. Pure Appl. Math. 42 (1989) 271--297.

\bibitem{CA1} L. Calota;
\emph{On some quasilinear elliptic equations with critical Sobolev exponents
and non-standard growth conditions}, Bull. Belg. Math. Soc. Simon
Stevin 15 (2) (2008) 249--256.

\bibitem{CH} A. Chambolle, P.L. Lions;
\emph{Image recovery via total variation minimization and related problems},
 Numer. Math. 76 (1997) 167--188.

\bibitem{MA} R.A. Mashiyev, B. Cekic, M. Avci, Z. Yucedag;
\emph{Existence and multiplicity of weak solutions for nonuniformly 
elliptic equations with nonstandard growth condition}. 
Complex Var. Elliptic Equ. 57 (2012) 579--595.

\bibitem{MI} M. Mihailescu, V. Radulescu;
\emph{On a nonhomogeneous quasilinear eigenvalue problem in Sobolev spaces
with variable exponent}, Proc. Am. Math. Soc. 135 (2007) 2929--2937.

\bibitem{MO}  M. Montenegro, R. Abreu;
\emph{Existence of a ground state solution for an elliptic problem with
critical growth in an exterior domain}. Nonlinear Anal. 109 (2014) 341--349

\bibitem{PU} P. Pucci, Q. Zhang;
\emph{Existence of entire solutions for a class of variable exponent elliptic
 equations}, J. Differential Equations 257 (2014) 1529--1566.

\bibitem{RU} M. Ruzicka;
\emph{Electrorheological fluids: modeling and mathematical theory},
volume 1748 of Lecture Notes in Mathematics. Springer-Verlag, Berlin, 2000.

\bibitem{TA} G. Talenti;
\emph{Best constant in Sobolev inequality}, Ann. Mat. Pura Appl. 110 (1976) 353--372.

\bibitem{WI} M. Willem;
\emph{Minimax Theorems}, Birkh\"{a}user, 1996.

\end{thebibliography}

\end{document}