\documentclass[reqno]{amsart} \usepackage{hyperref} \AtBeginDocument{{\noindent\small \emph{Electronic Journal of Differential Equations}, Vol. 2012 (2012), No. 208, pp. 1--11.\newline ISSN: 1072-6691. URL: http://ejde.math.txstate.edu or http://ejde.math.unt.edu \newline ftp ejde.math.txstate.edu} \thanks{\copyright 2012 Texas State University - San Marcos.} \vspace{9mm}} \begin{document} \title[\hfilneg EJDE-2012/208\hfil Fractional systems in bounded domains] {Existence of positive solutions for nonlinear fractional systems in bounded domains} \author[I. Bachar \hfil EJDE-2012/208\hfilneg] {Imed Bachar} \address{Imed Bachar \newline King Saud University College of Science Mathematics Department P.O. Box 2455 Riyadh 11451 Kingdom of Saudi Arabia} \email{abachar@ksu.edu.sa} \thanks{Submitted September 8, 2012. Published November 25, 2012.} \subjclass[2000]{35J60, 34B27, 35B44} \keywords{Fractional nonlinear systems; Green function; positive solutions; \hfill\break\indent blow-up solutions} \begin{abstract} We prove the existence of positive continuous solutions to the nonlinear fractional system \begin{gather*} (-\Delta|_D) ^{\alpha/2}u+\lambda g(.,v) =0, \\ (-\Delta|_D) ^{\alpha/2}v+\mu f(.,u) =0, \end{gather*} in a bounded $C^{1,1}$-domain $D$ in $\mathbb{R}^n$ $(n\geq 3)$, subject to Dirichlet conditions, where $0<\alpha \leq 2$, $\lambda $ and $\mu $ are nonnegative parameters. The functions $f$ and $g$ are nonnegative continuous monotone with respect to the second variable and satisfying certain hypotheses related to the Kato class. \end{abstract} \maketitle \numberwithin{equation}{section} \newtheorem{theorem}{Theorem}[section] \newtheorem{lemma}[theorem]{Lemma} \newtheorem{proposition}[theorem]{Proposition} \newtheorem{definition}[theorem]{Definition} \newtheorem{example}[theorem]{Example} \newtheorem{remark}[theorem]{Remark} \allowdisplaybreaks \section{Introduction and statement of main results} Let $\chi =( \Omega , \mathcal{F},\mathcal{F}_t, X_t, \theta _t, P^x) $ be a Brownian motion in $\mathbb{R}^n$, $n\geq 3$ and $\pi =( \Omega , \mathcal{G}, T_t) $ be an $\frac{\alpha }{2}$-stable process subordinator starting at zero, where $0<\alpha \leq 2$ and such that $\chi $ and $\pi $ are independent. Let $D$ be a bounded $C^{1,1}$-domain in $\mathbb{R}^n$ and $Z_{\alpha }^D$ be the subordinate killed Brownian motion process. This process is obtained by killing $\chi $ at $\tau _{D}$, the first exit time of $\chi $ from $D$ giving the process $\chi ^D$ and then subordinating this killed Brownian motion using the $\alpha/2$-stable subordinator $T_t$. For more description of the process $Z_{\alpha }^D$ we refer to \cite{GPRSSV,GRSS,S,SV}. Note that the infinitesimal generator of the process $Z_{\alpha }^D$ is the fractional power$(-\Delta|_D) ^{\alpha/2}$ of the negative Dirichlet Laplacian in $D$, which is a prototype of non-local operator and a very useful object in analysis and partial differential equations, see, for instance \cite{P,Y}. In this article, we will deal with the existence of positive continuous solutions for the nonlinear fractional system \begin{equation} \begin{gathered} (-\Delta|_D) ^{\alpha/2}u+\lambda g(.,v)=0 \quad \text{in $D$, in the sense of distributions} \\ (-\Delta|_D) ^{\alpha/2}v+\mu f(.,u)=0 \quad \text{in $D$, in the sense of distributions} \\ \lim_{x\to z\in \partial D} \frac{u(x)}{M_{\alpha }^D1(x)}=\varphi (z), \quad \lim_{x\to z\in \partial D} \frac{v(x)}{M_{\alpha }^D1(x)}=\psi (z), \end{gathered} \label{e1.1} \end{equation} where $\lambda ,\mu $ are nonnegative parameters, $\varphi ,\psi $ are positive continuous functions on $\partial D$ and $M_{\alpha }^D1$ is the nonnegative harmonic function with respect to $Z_{\alpha }^D$ given by the formula (see \cite[Theorem 3.1]{GPRSSV}, \begin{equation} M_{\alpha }^D1(x)=\frac{1-\frac{\alpha }{2}}{\Gamma (\frac{\alpha }{2})} \int_0^{\infty }t^{-2+\frac{\alpha }{2}}(1-P_t^D1(x))dt, \label{e1.2} \end{equation} where $(P_t^D)_{t>0}$ is the semi-group corresponding to the killed Brownian motion $\chi ^D$. Note that from \cite[remark 3.3]{SV}, there exists a constant $C>0$ such that \begin{equation} \frac{1}{C}\big(\delta (x)\big) ^{\alpha -2}\leq M_{\alpha }^D1(x)\leq C\big(\delta (x)\big) ^{\alpha -2},\quad \text{for all }x\in D, \label{e1.3} \end{equation} where $\delta (x)$ denotes the Euclidian distance from $x$ to the boundary of $D$. In the classical case (i.e. $\alpha =2$), there exist a lot of work related to the existence and nonexistence of solutions for the problem \eqref{e1.1}; see for example, the papers of Cirstea and Radulescu \cite{CR}, Ghanmi et al \cite{GMTZ}, Ghergu and Radulescu \cite{GR}, Lair and Wood \cite{LW1,LW2} and references therein. Most of the studies of these papers turn about the existence or the nonexistence of positive radial ones. In \cite{LW2}, the authors studied the system \eqref{e1.1} with $\alpha =2$, in the case $\mu f(.,u)=pu^{s}$, $\lambda g(.,v)=qv^{r}$, $s>0$, $r>0$ and $p,q$ are nonnegative continuous and not necessarily radial. They showed that entire positive bounded solutions exist if $p$ and $q$ satisfy the following condition \[ p(x)+q(x)\leq C| x| ^{-(2+\gamma )} \] for some positive constant $\gamma$ and $| x|$ large. Throughout this article, we denote by $G_{\alpha }^D$ the Green function of $Z_{\alpha }^D$. We recall the following interesting sharp estimates on $G_{\alpha }^D$ due to \cite{S}. Namely, there exists a positive constant $C>0$ such that for all $x,y$ in $D$, we have \begin{equation} \frac{1}{C}H(x,y)\leq G_{\alpha }^D(x,y)\leq CH(x,y), \label{e1.5} \end{equation} where \begin{equation*} H(x,y)=\frac{1}{| x-y| ^{n-\alpha }}\min \Big( 1,\frac{ \delta (x)\delta (y)}{| x-y| ^{2}}\Big) . \end{equation*} We also denote by $M_{\alpha }^D\varphi $ the unique positive continuous solution of \begin{equation} \begin{gathered} (-\Delta|_D) ^{\alpha/2}u=0 \quad \text{in $D$, in the sense of distributions} \\ \lim_{x\to z\in \partial D} \frac{u(x)}{M_{\alpha }^D1(x)}=\varphi (z), \end{gathered} \label{e1.6} \end{equation} which is given (see \cite{GPRSSV}) by \begin{equation} M_{\alpha }^D\varphi (x)=\frac{1}{\Gamma (\alpha/2) } E^x(\varphi (X_{\tau _{D}})\tau _{D}^{\frac{\alpha }{2}-1}). \label{e1.7} \end{equation} We aim at giving two existence results for \eqref{e1.1} as $f$ and $g$ are nondecreasing or nonincreasing with respect to the second variable. More precisely, to state our first existence result, we assume that $f,g:D\times [ 0,\infty )\to [ 0,\infty )$ are Borel measurable functions satisfying \begin{itemize} \item[(H1)] The functions $f$ and $g$ are continuous and nondecreasing with respect to the second variable. \item[(H2)] The functions $$ \widetilde{p}(y):= \frac{1}{M_{\alpha }^D\psi (y)}f(y,M_{\alpha }^D\varphi (y))\quad \text{and}\quad \widetilde{q}(y):=\frac{1}{M_{\alpha }^D\varphi (y)}g(y,M_{\alpha}^D\psi (y)) $$ belong to the Kato class $K_{\alpha }(D)$, defined below. \end{itemize} \begin{definition}[\cite{DMZ}] \label{def1.1}\rm A Borel measurable function $q$ in $D$ belongs to the Kato class $K_{\alpha }(D)$ if \begin{equation*} \lim_{r\to 0}\Big(\sup_{x\in D} \int_{(|x-y| \leq r(\cap D}\frac{\delta (y)}{\delta (x)} G_{\alpha}^D(x,y)|q(y)|dy\Big)=0. \end{equation*} \end{definition} This class is quite rich, it contains for example any function belonging to $L^{s}(D)$, with $s>n/\alpha$ (see Example \ref{exp2.1} below). On the other hand, it has been shown in \cite{DMZ}, that \begin{equation} x\to \big(\delta (x)\big) ^{-\gamma }\in K_{\alpha }(D),\quad \text{for } \gamma <\alpha . \label{e1.7.1} \end{equation} For more examples of functions belonging to $K_{\alpha }(D)$, we refer to \cite{DMZ}. Note that for the classical case (i.e. $\alpha =2$), the class $K_2(D)$ was introduced and studied in \cite{MZ}. Our first existence result is the following. \begin{theorem}\label{thm1.2} Assume that {\rm (H1), (H2)} are satisfied. Then there exist two constants $\lambda _0>0$ and $\mu _0>0$ such that for each $\lambda \in [ 0,\lambda _0)$ and each $\mu \in [0,\mu _0)$, problem \eqref{e1.1} has a positive continuous solution such that \begin{gather*} (1-\frac{\lambda }{\lambda _0})M_{\alpha }^D\varphi \leq u\leq M_{\alpha }^D\varphi \quad \text{in }D, \\ (1-\frac{\mu }{\mu _0})M_{\alpha }^D\psi \leq v\leq M_{\alpha }^D\psi \quad \text{in }D. \end{gather*} In particular $\lim_{x\to z\in \partial D}u(x)=\infty $ and $\lim_{x\to z\in \partial D}v(x)=\infty$. \end{theorem} We note that in \cite{GMTZ}, the authors studied a problem similar to \eqref{e1.1} for the case $\alpha =2$. They have obtained positive continuous bounded solution $(u,v)$. Here, we are interesting in the fractional setting. As second existence result, we aim at proving the existence of blow-up positive continuous solutions for the system \begin{equation} \begin{gathered} (-\Delta|_D) ^{\alpha/2}u+p(x)g(v)=0 \quad \text{in $D$, in the sense of distributions} \\ (-\Delta|_D) ^{\alpha/2}v+q(x)f(u)=0 \quad \text{in $D$, in the sense of distributions} \\ \lim_{x\to z\in \partial D} \frac{u(x)}{M_{\alpha }^D1(x) }=\varphi (z), \quad \lim_{x\to z\in \partial D} \frac{v(x)}{M_{\alpha }^D1(x)}=\psi (z), \end{gathered} \label{e1.8} \end{equation} where $\varphi ,\psi $ are positive continuous functions on $\partial D$ and $p,q$ are nonnegative Borel measurable functions in $D$. To this end, we fix $\phi $ a positive continuous functions on $\partial D$, we put $h_0=M_{\alpha }^D\phi $ and we assume the following: \begin{itemize} \item[(H3)] The functions $f,g:(0,\infty)\to [ 0,\infty )$ are continuous and nonincreasing. \item[(H4)] The functions $p_0:=p\frac{ f(h_0)}{h_0}$ and $q_0:=q\frac{g(h_0)}{h_0}$ belongs to the class $K_{\alpha }(D)$. \end{itemize} As a typical example of nonlinearity $f$ and $p$ satisfying (H3)-(H4), we have $f(t)=t^{-\nu }$, for $\nu >0$, and $p$ a nonnegative Borel measurable function such that \begin{equation*} p(x)\leq \frac{C}{\big(\delta (x)\big) ^{r}},\quad \text{for all }x\in D, \end{equation*} for some $C>0$ and $r+(1+\nu )(\alpha -2)<\alpha$. Indeed, since there exists a constant $c>0$, such that for all $x\in D$, $h_0(x)\geq c\big(\delta (x)\big) ^{\alpha -2}$, we deduce by \eqref{e1.7.1}, that the function $p_0:=p\frac{f(h_0)}{h_0}\in K_{\alpha }(D) $. Using the Schauder's fixed point theorem, we prove the following result. \begin{theorem}\label{thm1.3} Under the assumptions {\rm(H3), (H4)}, there exists a constant $c>1$ such that if $\varphi \geq c\phi $ and $\psi \geq c\phi $ on $\partial D$, then problem \eqref{e1.8} has a positive continuous solution $(u,v)$ satisfying for each $x\in D$, \begin{gather*} h_0\leq u\leq M_{\alpha }^D\varphi \quad \text{in }D, \\ h_0\leq v\leq M_{\alpha }^D\psi \quad \text{in }D. \end{gather*} In particular $\lim_{x\to z\in \partial D}u(x)=\infty $ and $\lim_{x\to z\in \partial D} v(x)=\infty$. \end{theorem} This result extends the one of Athreya \cite{A}, who considered the problem \begin{equation} \label{ast} \begin{gathered} \Delta u=g(u),\quad \text{in }\Omega \\ u=\varphi \quad \text{on }\partial \Omega , \end{gathered} \end{equation} where $\Omega $ is a simply connected bounded $C^{2}$-domain and $g(u)\leq \max (1,u^{-\alpha })$, for $0<\alpha <1$. Then he proved that there exists a constant $c>1$ such that if $\varphi \geq c\widetilde{h_0}$ on $\partial \Omega $, where $\widetilde{h_0}$ is a fixed positive harmonic function in $\Omega $, problem $(\ast )$ has a positive continuous solution $u$ such that $u\geq \widetilde{h_0}$. The content of this article is organized as follows. In Section 2, we collect some properties of functions belonging to the Kato class $K_{\alpha }(D)$, which are useful to establish our results. Our main results are proved in Section 3. As usual, let $B^{+}(D)$ be the set of nonnegative Borel measurable functions in $D$. We denote by $C_0(D)$ the set of continuous functions in $\overline{D}$ vanishing continuously on $\partial D$. Note that $C_0(D)$ is a Banach space with respect to the uniform norm $\| u\|_{\infty }=\underset{x\in D}{\sup }| u(x)| $. The letter $C$ will denote a generic positive constant which may vary from line to line. When two positive functions $\rho $ and $\theta $ are defined on a set $S$, we write $\rho \approx \theta $ if the two sided inequality $\frac{1}{C} \theta \leq \rho \leq C\theta $ holds on $S$. For $\rho \in B^{+}(D)$, we define the potential kernel $G_{\alpha }^D$ of $Z_{\alpha }^D$ by \begin{equation*} G_{\alpha }^D\rho (x):=\int_{D}G_{\alpha }^D(x,y)\rho (y)dy,\quad \text{for } x\in D \end{equation*} and we denote by \begin{equation} a_{\alpha }(\rho ):=\sup_{x,y\in D}\int_{D}\frac{G_{\alpha }^D(x,z)G_{\alpha }^D(z,y)}{G_{\alpha }^D(x,y)}\rho (y) dy. \label{e1.9} \end{equation} \section{The Kato class $K_{\protect\alpha }(D)$} \begin{example} \label{exp2.1}\rm For $s>\frac{n}{\alpha }$, we have $L^{s}(D) \subset K_{\alpha }(D)$. Indeed, let $0\frac{ n}{\alpha }$. Using $( \ref{e1.5}) $, there exists a constant $C>0$, such that for each $x,y\in D$ \begin{equation} \frac{\delta (y)}{\delta (x)}G_{\alpha }^D(x,y)\leq C\frac{1}{| x-y| ^{n-\alpha }}. \label{e2.1} \end{equation} This fact and the H\"{o}lder inequality imply that \begin{align*} &\int_{B(x,r)\cap D}\Big( \frac{\delta (y)}{\delta (x)}\Big) G_{\alpha }^D(x,y)|q(y)|dy \\ & \leq C\int_{B(x,r)\cap D}\frac{|q(y)|}{| x-y| ^{n-\alpha }}dy \\ &\leq C\Big( \int_{D}|q(y)|^{s}dy\Big) ^{1/s} \Big( \int_{B(x,r)}| x-y| ^{(\alpha -n) \frac{s}{s-1}}dy\Big) ^{\frac{s-1}{s}} \\ & \leq C\Big( \int_0^{r}t^{(\alpha -n) \frac{s}{s-1} +n-1}dt\Big) ^{\frac{s-1}{s}}\to 0, \end{align*} as $r\to 0$, since $(\alpha -n) \frac{s}{s-1}+n-1>-1$ when $s>\frac{n}{\alpha }$. \end{example} \begin{proposition}[\cite{DMZ}]\label{prop2.2} Let $q$ be a function in $K_{\alpha }(D)$, then we have \begin{itemize} \item[(i)] $a_{\alpha }(q)<\infty $. \item[(ii)] Let $h$ be a positive excessive function on $D$ with respect to $Z_{\alpha }^D$. Then we have \begin{equation} \int_{D}G_{\alpha }^D(x,y)h(y)|q(y)|dy\leq a_{\alpha }(q)h(x). \label{e2.2} \end{equation} Furthermore, for each $x_0\in \overline{D}$, we have \begin{equation} \lim_{r\to 0} \Big(\sup_{x\in D} \frac{1}{h(x)} \int_{B(x_0,r)\cap D}G_{\alpha }^D(x,y)h(y)|q(y)|dy\Big)=0. \label{e2.3} \end{equation} \item[(iii)] The function $x\to \big(\delta (x)\big) ^{\alpha -1}q(x)$ is in $L^1(D)$. \end{itemize} \end{proposition} \begin{lemma}\label{lem1.4} Let $q$ be a nonnegative function in $K_{\alpha }(D)$, then the family of functions \begin{equation*} \Lambda _{q}=\Big\{\frac{1}{M_{\alpha }^D\varphi (x)}\int_{D}G_{ \alpha }^D(x,y)M_{\alpha }^D\varphi (y)\rho (y)dy,\ | \rho | \leq q\Big\} \end{equation*} is uniformly bounded and equicontinuous in $\overline{D}$. Consequently $\Lambda _{q}$ is relatively compact in $C_0(D)$. \end{lemma} \begin{proof} Taking $h\equiv M_{\alpha }^D\varphi $ in \eqref{e2.2}, we deduce that for $\rho $ such that $| \rho |\leq q$ and $x\in D$, we have \begin{equation} \big| \int_{D}\frac{G_{\alpha }^D(x,y)}{M_{\alpha }^D\varphi (x)}M_{\alpha }^D\varphi (y)\rho (y)dy\big| \leq \int_{D}\frac{G_{\alpha }^D(x,y)}{M_{\alpha }^D\varphi (x)} M_{\alpha }^D\varphi (y)q(y)dy \leq a_{\alpha }(q)<\infty . \label{e2.4} \end{equation} So the family $\Lambda _{q}$ is uniformly bounded. Next we aim at proving that the family $\Lambda _{q}$ is equicontinuous in $\overline{D}$. Let $x_0\in $ $\overline{D}$ and $\varepsilon >0$. By \eqref{e2.3}, there exists $r>0$ such that \begin{equation*} \sup_{z\in D}\frac{1}{M_{\alpha }^D\varphi (z)}\int_{B(x_0,2r) \cap D}G_{\alpha }^D(z,y)M_{\alpha }^D\varphi (y)q(y)dy\leq \frac{ \varepsilon }{2}. \end{equation*} If $x_0\in D$ and $x,x'\in B(x_0,r)\cap D$, then for $ \rho $ such that $| \rho | \leq q$, we have \begin{align*} &\Big| \int_{D}\frac{G_{\alpha }^D(x,y)}{M_{\alpha }^D\varphi (x)}M_{\alpha }^D\varphi (y)\rho (y)dy-\int_{D} \frac{G_{\alpha }^D(x',y)}{M_{\alpha }^D\varphi (x')} M_{\alpha }^D\varphi (y)\rho (y)dy\Big| \\ &\leq \int_{D}\big| \frac{G_{\alpha }^D(x,y)}{M_{\alpha }^D\varphi (x)}-\frac{G_{\alpha }^D(x',y)}{M_{\alpha }^D\varphi (x')}\big| M_{\alpha }^D\varphi (y)q(y)dy \\ &\leq 2\underset{z\in D}{\sup }\int_{B(x_0,2r)\cap D}\frac{1}{ M_{\alpha }^D\varphi (z)}G_{\alpha }^D(z,y)M_{\alpha }^D\varphi (y)q(y)dy \\ &\quad +\int_{(| x_0-y| \geq 2r)\cap D}\big| \frac{G_{\alpha }^D(x,y)}{M_{\alpha }^D\varphi (x)}-\frac{ G_{\alpha }^D(x',y)}{M_{\alpha }^D\varphi (x')} \big| M_{\alpha }^D\varphi (y)q(y)dy \\ &\leq \varepsilon +\int_{(| x_0-y| \geq 2r)\cap D}\big| \frac{G_{\alpha }^D(x,y)}{M_{\alpha }^D\varphi (x)}- \frac{G_{\alpha }^D(x',y)}{M_{\alpha }^D\varphi (x')} \big| M_{\alpha }^D\varphi (y)q(y)dy. \end{align*} On the other hand, for every $y\in B^{c}(x_0,2r)\cap D$ and $ x,x'\in B(x_0,r)\cap D$, by using \eqref{e1.5} and the fact that $M_{\alpha }^D\varphi (z)\approx ( \delta (z)) ^{\alpha-2}$, we have \begin{align*} &\big| \frac{1}{M_{\alpha }^D\varphi (x)}G_{\alpha }^D(x,y)-\frac{1 }{M_{\alpha }^D\varphi (x')}G_{\alpha }^D(x',y)\big| M_{\alpha }^D\varphi (y) \\ &\leq \frac{M_{\alpha }^D\varphi (y)}{M_{\alpha }^D\varphi (x)} G_{\alpha }^D(x,y)+\frac{M_{\alpha }^D\varphi (y)}{M_{\alpha }^D\varphi (x')}G_{\alpha }^D(x',y) \\ &\leq C\Big[ \frac{\big(\delta (x)\big) ^{3-\alpha }\big( \delta (y)\big) ^{\alpha -1}}{| x-y| ^{n+2-\alpha }}+\frac{ \big( \delta (x')\big) ^{3-\alpha }\big(\delta (y)\big) ^{\alpha -1}}{| x'-y| ^{n+2-\alpha }}\Big] \\ &\leq C\Big[ \frac{1}{| x-y| ^{n+2-\alpha }}+\frac{1}{ | x'-y| ^{n+2-\alpha }}\Big] ( \delta(y)) ^{\alpha -1} \\ &\leq C\big(\delta (y)\big) ^{\alpha -1}. \end{align*} Now since $x\mapsto \frac{1}{M_{\alpha }^D\varphi (x)} G_{\alpha }^D(x,y)$ is continuous outside the diagonal and $q\in K_{\alpha}(D)$, we deduce by the dominated convergence theorem and Proposition \ref{prop2.2} (iii), that \begin{equation*} \int_{(| x_0-y| \geq 2r)\cap D}\big| \frac{G_{\alpha }^D(x,y)}{M_{\alpha }^D\varphi (x)}-\frac{G_{\alpha }^D(x',y)}{M_{\alpha }^D\varphi (x')}\big| M_{\alpha }^D\varphi (y)q(y)dy\to 0\quad \text{as }|x-x'| \to 0. \end{equation*} If $x_0\in \partial D$ and $x\in B(x_0,r)\cap D$, then \begin{equation*} \big| \int_{D}\frac{G_{\alpha }^D(x,y)}{M_{\alpha }^D\varphi (x)}M_{\alpha }^D\varphi (y)\rho (y)dy\big| \leq \frac{ \varepsilon }{2}+\int_{(| x_0-y| \geq 2r)\cap D}\frac{G_{\alpha }^D(x,y)}{M_{\alpha }^D\varphi (x)}M_{\alpha }^D\varphi (y)q(y)dy. \end{equation*} Now, since $\frac{G_{\alpha }^D(x,y)}{M_{\alpha }^D\varphi (x)} \to 0$ as $| x-x_0| \to 0$, for $| x_0-y| \geq 2r$, then by same argument as above, we obtain \begin{equation*} \int_{(| x_0-y| \geq 2r)\cap D}\frac{ G_{\alpha }^D(x,y)}{M_{\alpha }^D\varphi (x)}M_{\alpha }^D\varphi (y)q(y)dy\to 0\quad \text{as }| x-x_0| \to 0. \end{equation*} So the family $\Lambda _{q}$ is equicontinuous in $\overline{D}$. Therefore by Ascoli's theorem, the family $\Lambda _{q}$ becomes relatively compact in $C_0(D)$. \end{proof} \section{Proofs of Theorems \ref{thm1.2} and \ref{thm1.3}} \begin{proof}[Proof of Theorem \ref{thm1.2}] Put \[ \lambda _0:=\inf_{x\in D}\frac{M_{\alpha}^D\varphi (x)}{G_{\alpha }^D (g( .,M_{\alpha }^D\psi) )(x)}, \quad \mu _0:=\inf_{x\in D}\frac{M_{\alpha }^D\psi (x)}{ G_{\alpha }^D (f( .,M_{\alpha }^D\varphi) )(x)}. \] Using (H2) and \eqref{e2.2} we deduce that $\lambda _0>0$ and $\mu _0>0$. Let $\lambda \in [ 0,\lambda _0)$ and $\mu \in [0,\mu _0)$. Then for each $x\in D$, we have \begin{gather*} \lambda _0G_{\alpha }^D(g( .,M_{\alpha }^D\psi))(x) \leq M_{\alpha }^D\varphi (x)\\ \mu _0 G_{\alpha }^D(f(.,M_{\alpha }^D\varphi) )(x)\leq M_{\alpha }^D\psi (x). \end{gather*} So we define the sequences $(u_k)_{k\geq 0}$ and $(v_k)_{k\geq 0}$ by \begin{gather*} v_0=1, \\ u_k(x)=1-\frac{\lambda }{M_{\alpha }^D\varphi (x)} \int_{D}G_{\alpha }^D(x,y)g\big( y,v_k(y)M_{\alpha }^D\psi (y)\big) dy, \\ v_{k+1}(x)=1-\frac{\mu }{M_{\alpha }^D\psi (x)}\int_{D}G_{\alpha }^D(x,y)f\big( y,u_k(y)M_{\alpha }^D\varphi (y)\big) dy. \end{gather*} By induction, we can see that \begin{gather*} 0<(1-\frac{\lambda }{\lambda _0})\leq u_k\leq 1, \\ 0<(1-\frac{\mu }{\mu _0})\leq v_{k+1}\leq 1. \end{gather*} Next, we prove that the sequence $(u_k)_{k\geq 0}$ is nondecreasing and the sequence $(v_k)_{k\geq 0}$ is nonincreasing. Indeed, we have $$ v_{1}-v_0=-\frac{\mu }{M_{\alpha }^D\psi } G_{\alpha }^D (f( .,u_0M_{\alpha }^D\varphi) )\leq 0 $$ and therefore by (H1), we obtain that \[ u_{1}-u_0=\frac{\lambda }{M_{\alpha }^D\varphi }G_{\alpha }^D [g( .,v_0M_{\alpha }^D\psi) -g( .,v_{1}M_{\alpha }^D\psi) ]\geq 0. \] By induction, we assume that $u_k\leq u_{k+1}$ and $v_{k+1}\leq v_k$. Then we have \[ v_{k+2}-v_{k+1}=\frac{\mu }{M_{\alpha }^D\psi }G_{\alpha }^D [f(.,u_kM_{\alpha }^D\varphi) -f( .,u_{k+1}M_{\alpha}^D\varphi) ]\leq 0 \] and \[ u_{k+2}-u_{k+1}=\frac{\lambda }{M_{\alpha }^D\varphi } G_{\alpha }^D[g( .,v_{k+1}M_{\alpha }^D\psi) -g(.,v_{k+2}M_{\alpha }^D\psi) ]\geq 0. \] Therefore, the sequences $(u_k)_{k\geq 0}$ and $(v_k)_{k\geq 0} $ converge respectively to two functions $\widetilde{u}$ and $\widetilde{v}$ satisfying \begin{equation} \begin{gathered} 0<(1-\frac{\lambda }{\lambda _0})\leq \widetilde{u}\leq 1, \\ 0<(1-\frac{\mu }{\mu _0})\leq \widetilde{v}\leq 1. \end{gathered} \label{e3.1} \end{equation} On the other hand, using (H1), Proposition \ref{prop2.2} and the dominate convergence theorem, we deduce that \begin{gather*} \widetilde{u}(x)=1-\frac{\lambda }{M_{\alpha }^D\varphi (x)} \int_{D}G_{\alpha }^D(x,y)g( y,\widetilde{v}(y)M_{\alpha }^D\psi (y)) dy, \\ \widetilde{v}(x)=1-\frac{\mu }{M_{\alpha }^D\psi (x)}\int_{D}G_{ \alpha }^D(x,y)f( y,\widetilde{u}(y)M_{\alpha }^D\varphi (y)) dy. \end{gather*} Now by using (H1), (H2) and similar arguments as in the proof of Lemma \ref{lem1.4}, we deduce that $\widetilde{u}$ and $\widetilde{v}$ belongs to $C(\overline{D})$. Put $u=\widetilde{u}M_{\alpha }^D\varphi $ and $v=\widetilde{ v}M_{\alpha }^D\psi $. Then $u$ and $v$ are continuous in $D$ and satisfy \begin{equation} \begin{gathered} u(x)=M_{\alpha }^D\varphi (x)-\lambda \int_{D}G_{\alpha }^D(x,y)g( y,v(y)) dy \\ v(x)=M_{\alpha }^D\psi (x)-\mu \int_{D}G_{\alpha }^D(x,y)f\big( y,u(y)\big) dy. \end{gathered} \label{e3.2} \end{equation} In addition, since for each $x\in D$, $f\big( y,u(y)\big) \leq C\big(\delta (y)\big) ^{\alpha -2}\widetilde{p}(y)$ and $g\big( y,u(y)\big) \leq C\big(\delta (y)\big) ^{\alpha -2}\widetilde{q}(y)$, we deduce by Proposition \ref{prop2.2} $(iii)$ that the map $y\to f\big( y,u(y)\big) \in L_{\rm loc}^1(D)$ and $y\to g\big( y,u(y)\big) \in L_{\rm loc}^1(D)$. On the other hand, by \eqref{e3.2}, we have that $G_{\alpha }^Df( .,u) \in L_{\rm loc}^1(D)$ and $G_{\alpha }^Dg( .,v) \in L_{\rm loc}^1(D)$. Hence, applying $(-\Delta|_D) ^{\alpha/2}$ on both sides of \eqref{e3.2}, we conclude by \cite[p. 230]{GRSS} that $(u,v) $ is the required solution. \end{proof} \begin{example} \label{examp3.1} \rm Let $\nu \geq 0$, $\sigma \geq 0$, $r+(1-\sigma )(\alpha -2)<\alpha $ and $ \beta +(1-\nu )(\alpha -2)<\alpha $. Let $p$ and $q$ be two positive Borel measurable functions such that \begin{equation*} p(x)\leq C\big(\delta (x)\big) ^{-r},\quad q(x)\leq C\big(\delta (x)\big) ^{-\beta }\quad \text{for all }x\in D. \end{equation*} Let $\varphi $ and $\psi $ be positive continuous functions on $\partial D$. Therefore by Theorem \ref{thm1.2}, there exist two constants $\lambda _0>0$ and $\mu _0>0$ such that for each $\lambda \in [0,\lambda _0)$ and each $\mu \in [ 0,\mu _0)$, the problem \begin{gather*} (-\Delta|_D) ^{\alpha/2}u+\lambda p(x)v^{\sigma }=0 \quad \text{in $D$, in the sense of distributions} \\ (-\Delta|_D) ^{\alpha/2}v+\mu q(x)u^{\nu }=0 \quad \text{in $D$, in the sense of distributions} \\ \lim_{x\to z\in \partial D} \frac{u(x)}{M_{\alpha }^D1(x)}=\varphi (z), \quad \lim_{x\to z\in \partial D} \frac{v(x)}{M_{\alpha }^D1(x)}=\psi (z), \end{gather*} has a positive continuous solution $(u,v)$ such that \begin{gather*} (1-\frac{\lambda }{\lambda _0})M_{\alpha }^D\varphi \leq u\leq M_{\alpha }^D\varphi \quad \text{in }D, \\ (1-\frac{\mu }{\mu _0})M_{\alpha }^D\psi \leq v\leq M_{\alpha }^D\psi \quad \text{in }D. \end{gather*} In particular, $\lim_{x\to z\in \partial D}u(x)=\infty $ and $\lim_{x\to z\in \partial D}v(x)=\infty$. \end{example} \begin{proof}[Proof of Theorem \protect\ref{thm1.3}] Let $c:=1+a_{\alpha }(p_0)+a_{\alpha }(q_0)$, where $a_{\alpha }(p_0)$ and $a_{\alpha }(q_0)$ are the constant defined by the formula \eqref{e1.9}. We recall that from (H4) and Proposition \ref{prop2.2} (i), we have $a_{\alpha }(p_0)<\infty $ and $a_{\alpha }(q_0)<\infty $. Let $\varphi ,\psi $ be positive continuous functions on $\partial D$ such that $\varphi \geq c\phi $ and $\psi \geq c\phi $ on $\partial D$. It follows from the integral representation of $M_{\alpha }^D\varphi (x) $ and $M_{\alpha }^D\psi (x) $ (see \cite[p. 265]{DMZ}), that for each $x\in D$ we have \begin{equation} M_{\alpha }^D\varphi (x) \geq ch_0(x) \quad\text{and}\quad M_{\alpha }^D\psi (x) \geq ch_0(x) . \label{e3.4} \end{equation} Let $\Lambda $ be the nonempty closed convex set given by \begin{equation*} \Lambda =\big\{ \omega \in C(\overline{D}):\frac{h_0}{M_{\alpha }^D\varphi }\leq \omega \leq 1\big\} . \end{equation*} We define the operator $T$ on $\Lambda $ by \begin{equation} T(\omega) =1-\frac{1}{M_{\alpha }^D\varphi }G_{\alpha }^D( pf\left[ M_{\alpha }^D\psi -G_{\alpha }^D( qg( \omega M_{\alpha }^D\varphi ) ) \right] ) . \label{e3.5} \end{equation} We will prove that $T$ $\ $has a fixed point. Since for $\omega \in \Lambda $, we have $\omega \geq \frac{h_0}{M_{\alpha }^D\varphi }$, then we deduce from hypotheses (H3), (H4) and \eqref{e2.2} that \begin{equation} G_{\alpha }^D( qg( \omega M_{\alpha }^D\varphi ) ) \leq G_{\alpha }^D( qg( h_0) ) =G_{\alpha }^D(q_0h_0)\leq a_{\alpha }(q_0)h_0. \label{e3.6} \end{equation} So by using \eqref{e3.4} and \eqref{e3.6}, we obtain \begin{align*} M_{\alpha }^D\psi -G_{\alpha }^D ( qg( \omega M_{\alpha}^D\varphi ) ) &\geq M_{\alpha }^D\psi -a_{\alpha}(q_0)h_0 \\ &\geq ch_0-a_{\alpha }(q_0)h_0 \\ &= ( 1+a_{\alpha }(p_0)) h_0 \\ &\geq h_0>0. \end{align*} Hence, by using again (H3), (H4) and \eqref{e2.2}, we deduce that \begin{equation} G_{\alpha }^D( pf\left[ M_{\alpha }^D\psi -G_{\alpha }^D( qg( \omega M_{\alpha }^D\varphi ) ) \right] ) \leq G_{\alpha }^D( pf( h_0) ) =G_{\alpha }^D( p_0h_0) \leq a_{\alpha }(p_0)h_0. \label{e3.7} \end{equation} Using the fact that $M_{\alpha }^D\varphi \approx h_0$ and Lemma \ref{lem1.4}, we deduce that the family of functions \begin{equation*} \big\{ \frac{1}{M_{\alpha }^D\varphi }G_{\alpha }^D( pf\left[ M_{\alpha }^D\psi -G_{\alpha }^D( qg( \omega M_{\alpha }^D\varphi ) ) \right] ) :\omega \in \Lambda \big\} \end{equation*} is relatively compact in $C_0(D)$. Therefore, the set $T$ $\Lambda $ is relatively compact in $C(\overline{D})$. Next, we shall prove that $T$ maps $\Lambda $ into it self. Since $M_{\alpha }^D\psi -G_{\alpha }^D( qg( \omega M_{\alpha }^D\varphi ) ) \geq h_0>0$, we have for all $ \omega \in \Lambda $, $T\omega \leq 1$. Moreover, form \eqref{e3.7}, we obtain $T\omega \geq 1-\frac{a_{\alpha }(p_0)h_0}{M_{\alpha }^D\varphi }\geq \frac{h_0}{M_{\alpha }^D\varphi }$, which proves that $T (\Lambda)\subset \Lambda $. Now, we shall prove the continuity of the operator $T$ in $\Lambda$ in the supremum norm. Let $(\omega _k) _{k\in \mathbb{N}}$ be a sequence in $\Lambda $ which converges uniformly to a function $\omega $ in $\Lambda $. Then, for each $x\in D$, we have \begin{align*} | T\omega _k(x)-T\omega (x)| &\leq \frac{1}{M_{\alpha }^D\varphi (x)}G_{\alpha }^D \Big[p\Big| f(M_{\alpha }^D\psi -G_{\alpha }^D( qg( \omega _kM_{\alpha }^D\varphi ) ) ) \\ &\quad -f( M_{\alpha }^D\psi -G_{\alpha }^D( qg( \omega M_{\alpha }^D\varphi ) ) ) \Big| \Big](x). \end{align*} On the other hand, by similar arguments as above, we have \begin{align*} &p\Big| f( M_{\alpha }^D\psi -G_{\alpha }^D( qg( \omega _kM_{\alpha }^D\varphi ) ) ) -f( M_{\alpha }^D\psi -G_{\alpha }^D( qg( \omega M_{\alpha }^D\varphi ) ) ) \Big| \\ &\leq p\Big[f( M_{\alpha }^D\psi -G_{\alpha }^D( qg( \omega _kM_{\alpha }^D\varphi ) ) ) +f( M_{\alpha }^D\psi -G_{\alpha }^D( qg( \omega M_{\alpha }^D\varphi ) ) ) \Big] \\ &\leq 2p_0h_0. \end{align*} By the fact that $M_{\alpha }^D\varphi \approx h_0$, \eqref{e2.2} and the dominated convergence theorem, We conclude that for all $x\in D$, \begin{equation*} T\omega _k(x) \to T\omega (x) \quad \text{as }k\to +\infty . \end{equation*} Consequently, as $T( \Lambda ) $ is relatively compact in $C(\overline{D})$, we deduce that the pointwise convergence implies the uniform convergence, namely, \begin{equation*} \| T\omega _k-T\omega \| _{\infty }\to 0\quad \text{as }k\to +\infty . \end{equation*} Therefore, $T$ is a continuous mapping from $\Lambda $ into itself. So, since $T( \Lambda) $ is relatively compact in $C(\overline{D})$, it follows that $T$ is compact mapping on $\Lambda $. Finally, the Schauder fixed-point theorem implies the existence of a function $\omega \in \Lambda $ such that $\omega =T\omega $. Put \begin{equation*} u(x) =\omega (x) M_{\alpha }^D\varphi (x)\quad\text{and}\quad \upsilon (x) =M_{\alpha }^D\psi (x)-G_{\alpha}^D( qg( u) ) (x), \quad \text{for }x\in D. \end{equation*} Then $(u,\upsilon )$ satisfies \begin{gather*} u(x) = M_{\alpha }^D\varphi (x)-G_{\alpha }^D(pf(\upsilon )) (x), \\ \upsilon (x) = M_{\alpha }^D\psi (x)-G_{\alpha }^D(qg( u) ) (x). \end{gather*} Finally, we verify that $( u,\upsilon ) $ is the required solution. \end{proof} \begin{example} \label{examp3.2}\rm Let $\nu >0$, $\sigma >0$, $r+(1+\nu )(\alpha -2)<\alpha $ and $\beta +(1+\sigma )(\alpha -2)<\alpha $. Let $p$ and $q$ be two nonnegative Borel measurable functions such that \begin{equation*} p(x)\leq C\big(\delta (x)\big) ^{-r},\quad q(x)\leq C\big(\delta (x)\big) ^{-\beta }\quad \text{for all }x\in D. \end{equation*} Let $\varphi ,\psi $ and $\phi $ be positive continuous functions on $\partial D$. Then there exists a constant $c>1$ such that if $\varphi \geq c\phi $ and $\psi \geq c\phi $ on $\partial D$, then the problem \begin{gather*} (-\Delta|_D) ^{\alpha/2}u+p(x)v^{-\sigma }=0 \quad \text{in $D$, in the sense of distributions} \\ (-\Delta|_D) ^{\alpha/2}v+q(x)u^{-\nu }=0 \quad \text{in $D$, in the sense of distributions} \\ \lim_{x\to z\in \partial D} \frac{u(x)}{M_{\alpha }^D1(x) }=\varphi (z), \quad \lim_{x\to z\in \partial D} \frac{v(x)}{M_{\alpha }^D1(x)}=\psi (z), \end{gather*} has a positive continuous solution $(u,v)$ satisfying that for each $x\in D $, \begin{gather*} M_{\alpha }^D\phi \leq u\leq M_{\alpha }^D\varphi \quad \text{in }D, \\ M_{\alpha }^D\phi \leq v\leq M_{\alpha }^D\psi \quad \text{in }D. \end{gather*} In particular $\ u(x)\approx \big(\delta (x)\big) ^{\alpha -2}\approx v(x)$ in $D$. \end{example} \subsection*{Acknowledgements} The research is supported by NPST Program of King Saud University; project number 11-MAT1716-02. The author is thankful to the referees for their careful reading of the paper, and for their helpful comments and suggestions. \begin{thebibliography}{00} \bibitem{A} S. Athreya: \emph{On a singular semilinear elliptic boundary value problem and the boundary Harnack principle}, Potential Analysis (3) \textbf{17} (2002), 293-301. \bibitem{BH} J. Bliedtner, W. Hansen: \emph{Potential Theory. An Analytic and Probabilistic Approach to Balayage,}.(Springer Berlin, 1986). \bibitem{CR} F. C. Cirstea, V. D. Radulescu: \emph{Entire solutions blowing up at infinity for semilinear elliptic systems}, J. Math. Pures Appl. \textbf{81} (2002), 827--846. \bibitem{D} F. David: \emph{Radial solutions of an elliptic system}, Houston J. Math. \textbf{15} (1989), 425--458. \bibitem{DMZ} A. Dhifli, H. M\^{a}agli, M. Zribi: \emph{On the subordinate killed B.M in bounded domains and existence results for nonlinear fractional Dirichlet problems, } Math. Ann. (2) \textbf{352} (2012), 259-291. \bibitem{GMTZ} A. Ghanmi, H. M\^{a}agli, S. Turki, N. Zeddini: \emph{Existence of Positive Bounded Solutions for Some Nonlinear Elliptic Systems}, J. Math. Anal. Appl. \textbf{352} (2009), 440--448. \bibitem{GPRSSV} J. Glover, Z. Pop-Stojanovic, M. Rao, H. \v{S}iki \'{c}, R. Song, Z. Vondra\v{c}ek: \emph{Harmonic functions of subordinate killed Brownian motion}, J. Funct. Anal. \textbf{215} (2004), 399--426. \bibitem{GR}M. Ghergu, V. D. Radulescu: \emph{On a class of singular Gierer--Meinhart systems arising in morphogenesis}, C. R. Math. Acad. Sci. Paris. Ser. I \textbf{344} (2007), 163--168. \bibitem{GRSS} J. Glover, M. Rao, H. \v{S}iki\'{c}, R. Song: $\Gamma$-\emph{potentials, In: GowriSankaran K., et al. (eds.) Classical and modern potential theory and applications} (Chateau de Bonas, 1993), pp. 217--232. (Kluwer Acad.Publ. Dordrecht, 1994). \bibitem{LW1}A. V. Lair, A. W. Wood: \emph{solutions of semilinear elliptic equations with nonlinear gradient terms}, Int. J. Math. Sci. \textbf{22} (1999), 869-883. \bibitem{LW2}A.V. Lair, A.W. Wood: \emph{Existence of entire large positive solutions of semilinear elliptic systems}, J. Differential Equations (2) \textbf{164} (2000), 380--394. \bibitem{MZ}H. M\^{a}agli, M. Zribi: \emph{On a new Kato class and singular solutions of a nonlinear elliptic equation in bounded domains of }$\mathbb{R}^n$, Positivity. \textbf{9\ }(2005), 667-686. \bibitem{P}A. Pazy: \emph{Semigroups of Linear Operators and Applications to Partial Differential Equations},.(Spinger New York, 1983). \bibitem{S}R. Song: \emph{Sharp bounds on the density, Green function and jumping function of subordinate killed BM}, Probab. Theory Relat. Fields \textbf{128} (2004), 606--628. \bibitem{SV} R. Song, Z. Vondra\v{c}ek: \emph{Potential theory of subordinate killed Brownian motion in a domain}, Probab.Theory Relat. Fields \textbf{125} (2003), 578--592. \bibitem{Y} K, Yosida: \emph{Functional Analysis}. (Springer Berlin, 1980). \end{thebibliography} \end{document}