\documentclass[reqno]{amsart} \usepackage{amssymb,hyperref} \AtBeginDocument{{\noindent\small {\em Electronic Journal of Differential Equations}, Vol. 2005(2005), No. 137, pp. 1--16.\newline ISSN: 1072-6691. URL: http://ejde.math.txstate.edu or http://ejde.math.unt.edu \newline ftp ejde.math.txstate.edu (login: ftp)} \thanks{\copyright 2005 Texas State University - San Marcos.} \vspace{9mm}} \begin{document} \title[\hfilneg EJDE-2005/137\hfil Coexistence and stability] {Coexistence and stability of solutions for a class of reaction-diffusion systems} \author[Z. Zhang\hfil EJDE-2005/137\hfilneg] {Zhenbu Zhang} \address{Zhenbu Zhang \hfill\break Department of Mathematics\\ Jackson State University\\ Jackson, MS 39217, USA} \email{zhenbu.zhang@jsums.edu} \date{} \thanks{Submitted March 21, 2005. Published December 1, 2005.} \subjclass[2000]{35K55, 35K57} \keywords{Coexistence; stability; reaction-diffusion; eigenvalue problem} \begin{abstract} In this paper we consider the situation of two species of predators competing for one species of prey. We use comparison principles to study the global existence, the existence of non-trivial steady states and their stability. \end{abstract} \maketitle \numberwithin{equation}{section} \newtheorem{lemma}{Lemma}[section] \newtheorem{remark}[lemma]{Remark} \newtheorem{theorem}[lemma]{Theorem} \newtheorem{proposition}[lemma]{Proposition} \section{Introduction} It is well-known that the study of coexistence problem of competing species is one of the main topics in mathematical ecology. The object of this paper is to study the problem of coexistence for three interacting species, among which two species of predators compete for one species of prey. We assume that the two competing species have different diffusion rates $d_10,\\ \frac{\partial v_i}{\partial {\nu} }(x,t)+r_i(x) v_i(x,t)=0,\quad i=1,2,\; x \in\partial \Omega,\; t>0,\\ u(x,0) = u_{0}(x), v_i(x,0)=v_{i0} (x),\quad i=1,2, \mbox{ in } \; \Omega, \end{gathered} \end{equation} where $\Omega \subset \mathbb{R}^n$ is a bounded domain with smooth boundary $\partial \Omega$. $\triangle$ is the Laplacian. $\nu$ denotes the outer unit normal to $\partial \Omega$. $u$ is the population density of the prey, and $v_1$, $v_2$ are the population densities of two competing predator species. $d_i>0$ are the diffusion rates with $d_10$, $r_2(x) \geq r_1(x) \geq 0 \not\equiv 0$, $u^0(x) \geq 0\not\equiv 0$, $u_0(x) \geq 0$, $v_{i0}(x) \geq 0$. $F_i$ are given by \begin{equation}\label{eq:in2} \begin{gathered} F_0(u, v_1, v_2)=-f_1(u)v_1-f_2(u)v_2,\\ F_1(u, v_1, v_2)=v_1(f_1(u)-v_1-v_2),\\ F_2(u, v_1, v_2)=v_2(f_2(u)-v_1-v_2). \end{gathered} \end{equation} Here $f_i(u)$ is the consumption rate of the prey per predator. The forms of $F_1$ and $F_2$ represent that, at the constant level of the prey $u$, the predators have logistic growth. A similar model has been investigated by Wang and Wu in \cite{XW2} by using bifurcation theory when the predators have a Malthusian (or exponential) growth. The fact that predator species have different diffusion rates makes it hard to study this model. If assuming equal diffusion rates, the model can be simplified to a much simpler model (e.g see \cite{HW1}). We assume that \begin{itemize} \item[(i)] $f_i: \mathbb{R}_+ \to \mathbb{R}_+$ and $f_i(0)=0$; \item[(ii)] $f_i$ is continuously differentiable and $f_i' \geq 0\not\equiv 0$. \end{itemize} A typical example of $f_i$ is the monotone Monod function, $f_i(u)=m_iu/(a_i+u)$ (see \cite{HW1}, \cite{LE1} ), where $a_i$ and $m_i$ are positive constants. $m_i$ is the maximal growth rate and $a_i$ is the Michaelis-Menten (or half-saturation) constant. Since $u$ and $v_i$ are population densities, so only non-negative solutions are of physical interest. Observe that $F_0(0, v_1, v_2)=0$ and $F_i(u,v_1,v_2)=0$ if $v_i=0$ for $i=1,2$, therefore \[\mathbb{R}_+^3=\{(u, v_1, v_2)|\; u \geq 0,\; v_i \geq 0,\; i=1,2\}\] is an invariant region of (\ref{eq:in1}) (see \cite{SMO}). Therefore in this paper we consider only nonnegative solutions of (\ref{eq:in1}) without further explanation. This paper is organized as follows: In Section 2, we will prove the global existence of solutions. In Section 3, we consider one predator population case in detail. In Section 4, we will study the existence of steady states. In Section 5, we study the local stability of equilibrium solutions. \section{Existence of a global solution} By standard existence theory, e.g. see \cite{AH1}, \cite{AH2} and \cite{AH3}, (\ref{eq:in1}) has a unique nonnegative smooth local solution $U(x, t)=(u(x,t), v_1(x,t), v_2(x,t))$ existing for $0 \leq t 0$ for all $x \in \overline\Omega$. \end{lemma} For the proof of this Lemma see \cite{LE1} and the references therein. \begin{remark} \label{rmk2.2} \rm The importance of this Lemma lies in that:\\ (i) It implies that (\ref{eq:in1}) has an equilibrium solution $(u, v_1, v_2)=(S(x), 0, 0)$. This is so-called washout equilibrium solution. In practice, attaining this equilibrium is undesirable.\\ (ii) It provides an a priori bound for $u$, which ensures the existence of the global solution. \end{remark} \begin{lemma}\label{L:102} For any solution of the form $(0, v_1(x,t), v_2(x,t))$ of (\ref{eq:in1}), we have $v_i(x,t) \to 0$ as $t \to \infty$. That is, if there is no prey, then the predators will be extinct. \end{lemma} \begin{proof} From our assumptions about $f_i$, if $u=0$, the equation for $v_i (i =1,2)$ becomes \[ \frac{\partial v_i}{\partial t} = d_i\triangle{v_i}+v_i(-\sum_{j=1}^2 v_j) \leq d_i\triangle{v_i}-v_i^2. \] Let $A_i=\max_{x \in \overline\Omega}v_{i0} (x)$ and compare $v_i$ with the solution of the initial value problem of ODE: \begin{gather*} \frac{dw_i}{dt}=-w_i^2, \quad t>0,\\ w(0)=A_i, \end{gather*} we have \[ 0 \leq v_i(x,t) \leq w_i(t). \] But $w_i(t) \to 0$ as $t \to \infty$, therefore, $v_i(x,t) \to 0$ as $t \to \infty$. \end{proof} By virtue of $S(x)$ in Lemma \ref{L:101} and comparison principle (e.g. see \cite{CVP}, \cite{HLS}, \cite{LY1}), it is easy to prove the following global existence and uniqueness theorem. We omit the proof here. \begin{theorem}\label{T:101} For any smooth nonnegative functions $u^0(x)$, $u_{0}(x)$ and $v_{i0} (x)$, \eqref{eq:in1} has a unique smooth bounded global solution. \end{theorem} This theorem implies that the solutions of (\ref{eq:in1}) generate a semidynamical system on $C_+\times C_+\times C_+$, where $C_+$ is the set of nonnegative, continuous functions on $\bar{\Omega}$ with the usual supremum norm. This semidynamical system is denoted by $\varPhi(t,x_0)$, where $t \geq 0$ and $x_0$ represents the triple of initial conditions given by (\ref{eq:in1}). \section{One Predator Case} In this section, we consider a special case when there is only one predator population, or equivalently, $v_i \equiv 0$ for $i=1$ or $i=2$. Without loss of generality, we assume that $v_2 \equiv 0$ and write $v_1$ as $v$ and $f_1$ as $f$ respectively, then (\ref{eq:in1}) becomes \begin{equation}\label{eq:n01} \begin{gathered} \frac{\partial u}{\partial t} = d_0\triangle{u}-f(u)v, \quad \mbox{on } \Omega\times \mathbb{R}_+, \\ \frac{\partial v}{\partial t} = d_1\triangle{v}+v(f(u)-v), \quad \mbox{on } \Omega\times \mathbb{R}_+, \\ \frac{\partial u}{\partial {\nu} }(x,t)+r_0(x)u(x,t)=u^0(x),\quad x \in\partial \Omega, \; t>0,\\ \frac{\partial v}{\partial {\nu} }+r_1(x)v(x,t)=0,\quad x \in\partial \Omega, \,\; t>0,\\ u(x,0) = u_{0}(x),\quad v(x,0)=v_{0} (x),\quad \mbox{in } \Omega. \end{gathered} \end{equation} \subsection{Steady States} First, from Lemma \ref{L:101}, (\ref{eq:n01}) has a steady state solution $U_w=(u, v)=(S(x), 0)$ with $S(x) >0$. The following theorem says that (\ref{eq:n01}) has a positive steady state solution $U_p=(u, v)=(\tilde u(x), \tilde v(x))$ with $\tilde u(x)>0$ and $\tilde v(x)>0$. \begin{theorem}\label{T:n01} \eqref{eq:n01} has a positive steady state solution $(u, v)=(\tilde u(x), \tilde v(x))$ with $\tilde u(x)>0$ and $\tilde v(x)>0$ provided that $\int_{\partial\Omega}r_1(x)dx$ is small enough. \end{theorem} To prove this Theorem, we need some preparations. Consider the eigenvalue problem \begin{equation}\label{eq:302} \begin{gathered} d\triangle \phi+q(x)\phi=\lambda \phi,\quad x \in \Omega,\\ \frac{\partial \phi}{\partial \nu}+r(x)\phi=0,\quad x \in\partial \Omega, \end{gathered} \end{equation} where $d>0$, $q(x) \in C^{2+\alpha}(\bar{\Omega})$ for some $\alpha>0$. It is well-known that there is a unique eigenvalue $\lambda_1=\lambda(q, d,r)$, called the \textit{`principal eigenvalue'}, such that the associated `principal eigenfunction' (unique up to a multiplicative constant) is strictly positive. Furthermore, we have the following Lemma. \begin{lemma}\label{L:301} The principal eigenvalue $\lambda(q,d,r)$ of \eqref{eq:302} is a continuous non-increasing function of $d$, and is strictly decreasing if $q(x)$ is not a constant. Furthermore, the following hold: \begin{itemize} \item[(a)] $\lambda(q,d,r)\uparrow Q=\max_{\bar{\Omega}}q(x)$ as $d\to 0$; \item[(b)] $\lambda(q,d,r)\downarrow \omega=\frac{1}{|\Omega|}\int_{\Omega} q(x)dx-\frac{1}{|\partial\Omega|}\int_{\partial\Omega} r(x)dx$ as $d \to \infty$; \item[(c)] If $q_1(x) \geq q_2(x)$ for $x \in \Omega$, then $\lambda(q_1,d,r) \geq\lambda(q_2,d,r)$ with strict inequality if $q_1 (x)\not\equiv q_2(x)$; \item[(d)] If $r_1(x) \leq r_2(x)$ for $x \in \partial\Omega$, then $\lambda(q,d,r_1) \geq\lambda(q,d,r_2)$. \end{itemize} \end{lemma} For the proof of the above Lemma, see \cite{DHMP} and the references therein. \begin{remark} \label{rmk3.3} \rm From this Lemma we can see that if $\int_{\Omega} q(x)dx>0$ and $\int_{\partial\Omega} r(x)dx$ is small such that $\omega>0$, then for any $d>0$, the principal eigenvalue $\lambda(q,d,r)$ of \eqref{eq:302} is positive. In particular, if $q(x) \geq 0\not\equiv 0$ and $r(x) \equiv 0$, i.e. for Neumann boundary condition, the principal eigenvalue of \eqref{eq:302} is always positive. \end{remark} By using this Lemma, we can prove the following Lemma. \begin{lemma}\label{L:302} If the principal eigenvalue $\lambda(q,d,r)$ of \eqref{eq:302} is positive, then the boundary-value problem \begin{equation}\label{eq:n32} \begin{gathered} d\triangle u+u(q(x)-u)=0,\quad x \in \Omega,\\ \frac{\partial u}{\partial \nu}+r(x)u=0,\quad x \in\partial \Omega, \end{gathered} \end{equation} has a unique, strictly positive solution. \end{lemma} \begin{proof} Let $\psi(x)>0$ be the principal eigenfunction of \eqref{eq:302} corresponding to $\lambda_1$, and let $ \underbar{u}(x)=\delta \psi(x)$, where $\delta>0$ is a small number to be determined, then for $\delta>0$ small enough, \begin{gather*} d\triangle \underbar{u}+ \underbar{u}(q(x)- \underbar{u})=\lambda_1 \delta \psi-\delta^2 \psi^2 >0, \quad x \in \Omega,\\ \frac{\partial \underbar{u}}{\partial \nu}+r(x) \underbar{u} =\delta(\frac{\partial \psi}{\partial \nu}+r(x)\psi)=0,\quad x \in\partial \Omega. \end{gather*} Therefore, $ \underbar{u}(x)$ is a sub-solution of \eqref{eq:n32}. It is easily seen that $\bar{u}(x)=Q=\max _{\bar{\Omega}}q(x)$ is a sup-solution of \eqref{eq:n32}. Hence \eqref{eq:n32} has a strictly positive solution $u(x)$ satisfying $0<\delta\psi \leq u(x) \leq Q$. Now we prove the uniqueness. Suppose that $u_1$ and $u_2$ both are positive solutions of \eqref{eq:n32}. Let $w=\frac{u_1}{u_2}>0$, then $w$ satisfies \begin{gather*} d\triangle w+\frac{2d \triangledown u_2}{u_2} \triangledown w+w u_2(1-w)=0, \quad x \in \Omega,\\ \partial_{\nu} w|_{\partial \Omega}=0. \end{gather*} Then using the maximum principle, we have $w\equiv 1$. This completes the proof of the Lemma. \end{proof} \begin{proof}[Proof of Theorem \ref{T:n01}] To prove the Theorem, we need to prove the existence of positive solutions of the boundary-value problem \begin{equation}\label{eq:n03} \begin{gathered} d_0\triangle{u}-f(u) v=0, \quad x \in \Omega, \\ d_1\triangle{v}+v(f(u)- v)=0, \quad x \in \Omega, \\ \frac{\partial u}{\partial {\nu} }+r_0(x)u=u^0(x),\quad x \in\partial \Omega,\\ \frac{\partial v}{\partial {\nu} }+r_1(x)v=0,\quad x \in\partial \Omega. \end{gathered} \end{equation} For notational convenience, we write \eqref{eq:n03} as \begin{equation}\label{eq:n04} \begin{gathered} -d_0\triangle{u}=g_1(u, v), \quad x \in \Omega, \\ -d_1\triangle{v}=g_2(u, v), \quad x \in \Omega, \\ \frac{\partial u}{\partial {\nu} }+r_0(x)u=u^0(x),\quad x \in\partial \Omega,\\ \frac{\partial v}{\partial {\nu} }+r_1(x)v=0,\quad x \in\partial \Omega, \end{gathered} \end{equation} where $g_1(u, v)=-f(u) v$, $g_2(u,v)=v(f(u)- v)$. Then we can see that for $u \geq 0$ and $v \geq 0$, \[ \frac{\partial g_1}{\partial {v} }=-f(u) \leq 0,\quad \frac{\partial g_2}{\partial {u} }=vf'(u) \geq 0. \] That is $g_1$ is quasi-monotonic decreasing and $g_2$ is quasi-monotonic increasing. Therefore, \eqref{eq:n04} is a so-called mixed quasi-monotonic system. For such a system, we have the following definition of upper and lower solutions (e.g. see \cite{CVP} and \cite{LY1}). \smallskip \noindent\textbf{Definition:} If $U(x)=(\bar u (x), \bar v(x))$ and $V(x)=( \underbar u(x), \underbar v(x))$ satisfy \begin{gather*} \frac{\partial\bar u}{\partial\nu}+r_0(x)\bar u \geq u^0(x) \geq \frac{\partial \underbar u}{\partial\nu}+r_0(x) \underbar u , \quad \text{ on }\partial\Omega,\\ \frac{\partial\bar v}{\partial\nu} +r_1(x)\bar v\geq 0 \geq\frac{\partial \underbar v}{\partial\nu}+r_1(x) \underbar v, \quad \text{ on } \partial\Omega, \\ -d_0\triangle \bar u-g_1(\bar u, \underbar v)\geq 0 \geq -d_0\triangle \underbar u-g_1( \underbar u,\bar v),\quad\mbox{in }\Omega, \\ -d_1\triangle \bar v-g_2(\bar u,\bar v)\geq 0 \geq -d_1\triangle \underbar v -g_2( \underbar u, \underbar v),\quad\mbox{in }\Omega, \end{gather*} then $(U(x), V(x))$ is said to be a pair of upper and lower solutions of \eqref{eq:n04}. Then we have the following result. \begin{theorem} \label{thmA} If \eqref{eq:n04} has a pair of upper and lower solutions $(U(x), V(x))$ such that $V(x) \leq U(x)$, then \eqref{eq:n04} has at least one solution $(u(x), v(x))$ satisfying \[ V(x) \leq (u(x), v(x)) \leq U(x). \] \end{theorem} The proof of this theorem can be found in \cite{LY1}. Now we construct a pair of upper and lower solutions $U(x)=(\bar u (x), \bar v(x))$ and $V(x)=( \underbar u(x), \underbar v(x))$ as follows: First observe that, by the definition, we need $U(x)=(\bar u (x), \bar v(x))$ and $V(x)=( \underbar u(x), \underbar v(x))$ satisfy \begin{gather}\label{eq:n06} -d_0\triangle \bar u+ \underbar v f(\bar u)\geq 0 \geq -d_0\triangle \underbar u +\bar v f( \underbar u), \\ \label{eq:n07} -d_1\triangle \bar v-\bar v(f(\bar u)-\bar v)\geq 0 \geq -d_1\triangle \underbar v - \underbar v(f( \underbar u)- \underbar v). \end{gather} Let $\bar u (x)=S(x)$ be the unique positive solution of \eqref{eq:201}, then, for any $ \underbar v(x) \geq 0$, the left-hand side of (\ref{eq:n06}) is satisfied. We take $\bar v(x)=M_1$ to be a positive constant such that $M_1 \geq f(\max_{\bar\Omega}S(x))$, then the left-hand side of (\ref{eq:n07}) is satisfied. With $\bar v(x)=M_1$, the right-hand side of (\ref{eq:n06}) becomes \[ -d_0\triangle \underbar u+M_1 f( \underbar u) \leq 0. \] We take $ \underbar u= \underbar u(x)>0$ to be the positive solution of the boundary-value problem \begin{equation}\label{eq:n08} \begin{gathered} -d_0\triangle \underbar u+M_1 f( \underbar u)= 0, \quad x \in \Omega, \\ \frac{\partial \underbar u}{\partial\nu}+r_0(x) \underbar u =u^0(x), \quad x \in\partial \Omega. \end{gathered} \end{equation} Now we prove that (\ref{eq:n08}) has a positive solution. In fact, since $f(0)=0$, $0$ is a lower solution of (\ref{eq:n08}). Since $f \geq 0$, any constant $K$ satisfying \[ K \geq \frac{1}{\gamma}\max_{\bar\Omega}u^0(x), \] where ${\gamma}=\min_{\bar\Omega}r_0(x)>0$, is an upper solution of (\ref{eq:n08}). Therefore (\ref{eq:n08}) has a solution $ \underbar u(x)$ satisfying $0 \leq \underbar u(x) \leq K$. By strong maximum principle, we have \[ \underbar u(x)>0,\quad \text{for} \hspace{3mm}x \in\bar\Omega. \] We claim that $\bar u (x)\geq \underbar u(x)$. In fact, let $w=\bar u(x)- \underbar u(x)$, then $w$ satisfies \begin{gather*} -d_0\triangle w=M_1f( \underbar u)\geq 0,\quad x \in \Omega,\\ \frac{\partial w}{\partial \nu}+r_0 w=0,\quad x \in\partial \Omega. \end{gather*} By the maximum principle \[ \min_{\bar \Omega} w=\min_{\partial \Omega} w. \] Now we prove that $\min_{\partial \Omega} w\geq 0$. Indeed, if $w(x_0)=\min_{\partial \Omega} w<0$, then, by Hopf's Lemma, at $x_0 $, we have $\frac{\partial w}{\partial \nu}<0$. But from the boundary condition, we have $\frac{\partial w}{\partial \nu}|_{x_0}=-r_0w(x_0)> 0$. This is a contradiction. Therefore $w \geq 0$. That is $\bar u (x)\geq \underbar u(x)$. With $ \underbar u= \underbar u(x)>0$, $f( \underbar u) \geq 0 \not\equiv 0$. Therefore, $\int_\Omega f( \underbar u(x))dx>0$. By Lemma \ref{L:302}, if $\int_{\partial\Omega} r_1(x)dx$ is small enough, then the boundary-value problem \begin{equation}\label{eq:n09} \begin{gathered} -d_1\triangle \underbar v - \underbar v (f( \underbar u)- \underbar v)= 0, \quad x \in \Omega, \\ \frac{\partial \underbar v}{\partial\nu}+r_1(x) \underbar v=0,\quad x \in\partial \Omega, \end{gathered} \end{equation} has a unique positive solution $ \underbar v= \underbar v(x)>0$. Now we claim that $ \underbar v \leq M_1=\bar v$. In fact, from the proof of Lemma \ref{L:302}, we can see that the positive solution of (\ref{eq:n09}) is bounded from above by any constant that is greater than or equal to $f(\max_{\bar\Omega} \underbar u(x))$. Since $ \underbar u(x) \leq \bar u(x)$ and $f' \geq 0$, we have \[ \underbar v(x)\leq f(\max_{\bar\Omega} \underbar u(x))\leq f(\max_{\bar\Omega}\bar u(x))\leq M_1. \] Thus we have constructed a pair of upper and lower solutions $U(x)=(\bar u (x), \bar v(x))$ and $V(x)=( \underbar u(x), \underbar v(x))$ of \eqref{eq:n04} satisfying $V(x) \leq U(x)$, then from Theorem A we know that \eqref{eq:n04} has a solution $(\tilde u(x), \tilde v(x))$ satisfying \[ V(x) \leq (\tilde u(x), \tilde v(x)) \leq U(x). \] \end{proof} For the rest of this article, we assume that $\int_{\partial\Omega} r_1(x)dx$ is small enough such that the related eigenvalue problem with Robin boundary condition has a positive principal eigenvalue, without further explanation. Now we prove some properties of positive solutions of \eqref{eq:n04}. \begin{proposition}\label{P:nn401} Suppose that $(u_1, v_1)$ and $(u_2, v_2)$ are two positive solutions of \eqref{eq:n03}. If $u_1 \geq u_2$, then $v_1 \geq v_2$. \end{proposition} Before proving the above proposition, we cite the following Lemma whose proof can be found in \cite{DHMP}. \begin{lemma}\label{L:n302} Consider the initial boundary value problem \begin{equation}\label{eq:n303} \begin{gathered} u_t=d\triangle u+u(q(x)-u),\quad x \in \Omega,\; t>0,\\ \frac{\partial u}{\partial\nu}+r_1(x)u=0,\quad x \in\partial \Omega,\; t>0,\\ u(x,0)=u_0(x). \end{gathered} \end{equation} If $\lambda(q,d,r_1) \leq 0$, then $0$ is the global attractor for positive solutions of \eqref{eq:n303}. If $\lambda(q,d,r_1)>0$, then the unique, strictly positive steady-state solution is a global attractor for non-trivial positive solutions of \eqref{eq:n303}, the convergence in both cases being in $\|\cdot\|_{\infty}$. \end{lemma} \begin{proof}[Proof of Proposition \ref{P:nn401}] We consider the following initial-boundary-value problems: \begin{equation}\label{eq:n304} \begin{gathered} V_{1t}=d_1\triangle V_1+V_1(f(u_1(x))-V_1),\quad x \in \Omega,\; t>0,\\ \frac{\partial V_1}{\partial\nu}+r_1(x)V_1=0,\quad x \in\partial \Omega,\; t>0,\\ V_1(x,0)=v_0(x), \end{gathered} \end{equation} and \begin{equation}\label{eq:n305} \begin{gathered} V_{2t}=d_1\triangle V_2+V_2(f(u_2(x))-V_2),\quad x \in \Omega,\; t>0,\\ \frac{\partial V_2}{\partial\nu}+r_1(x)V_2=0,\quad x \in\partial \Omega,\; t>0,\\ V_1(x,0)=v_0(x). \end{gathered} \end{equation} From the positivity of $f(u_i)$ and Lemma \ref{L:n302} we know that $V_1(x,t) \to v_1(x)$ as $t \to \infty$ and $V_2(x,t) \to v_2(x)$ as $t \to \infty$. $u_1 \geq u_2$ implies that $f(u_1) \geq f(u_2)$. Therefore, from (\ref{eq:n304}), we have \[ V_{1t}\geq d_1\triangle V_1+V_1(f(u_2(x))-V_1). \] Thus $V_1(x,t)$ is an upper solution of (\ref{eq:n305}). So we have $V_1(x,t) \geq V_2(x,t)$. Therefore, $v_1 \geq v_2$. \end{proof} \begin{proposition}\label{P:nn402} Suppose that $(u_1, v_1)$ and $(u_2, v_2)$ are two positive solutions of \eqref{eq:n03}. If $u_1 \geq u_2$, then $u_1 \equiv u_2$, $v_1 \equiv v_2$. \end{proposition} \begin{proof} Let $w=u_1- u_2$, then we have $w \geq 0$. $u_1$ and $u_2$ satisfy \[ d_0\triangle{u_1}=f(u_1) v_1\geq f(u_2) v_2=d_0\triangle{u_2}. \] Therefore, $w$ satisfies \begin{gather*} -d_0\triangle w\leq 0,\quad x \in \Omega,\\ \frac{\partial w}{\partial \nu}+r_0 w=0,\quad x \in\partial \Omega. \end{gather*} Hence, if $w \not\equiv 0$, by the maximum principle, \[ \max_{\bar \Omega} w=\max_{\partial \Omega} w>0. \] Assume that $x_0 \in \partial \Omega$ such that \[ w(x_0)=\max_{\partial \Omega} w>0, \] then, by Hopf's Lemma, at $x_0 $, we have $\frac{\partial w}{\partial \nu}>0$. This contradicts to the boundary condition. Therefore we have $w \equiv 0$, i.e. $u_1 \equiv u_2$. Then we must have $v_1 \equiv v_2$. \end{proof} \begin{proposition}\label{P:nn403} Suppose that $(u_1, v_1)$ and $(u_2, v_2)$ are two positive solutions of \eqref{eq:n03}. If $f(u_1)-f(u_2) \geq v_1-v_2$, then $u_1 \equiv u_2$, $v_1 \equiv v_2$. \end{proposition} \begin{proof} From Proposition \ref{P:nn402}, we need only to prove that $u_1 \geq u_2$. In fact, due to the positivity of $v_i$, $v_i$ can be looked as the principal eigenfunction of the eigenvalue problem \begin{gather*} d_1\triangle \phi+q_i(x)\phi=\lambda \phi, \quad x \in\Omega,\\ \frac{\partial \phi}{\partial \nu}+r_1(x)\phi=0,\quad x \in\partial \Omega, \end{gather*} with $q_i(x)=f({u_i})- v_i$, and then associated with the principal eigenvalue $\lambda=\lambda(q_i(x),d_1,r_1)=0$, $i =1,2$. Since $f(u_1)-f(u_2) \geq v_1-v_2$, we have \[ f(u_1)-v_1 \geq f(u_2) -v_2. \] Therefore, by Lemma \ref{L:301}, we have $f(u_1)-v_1 \equiv f(u_2) -v_2$, that is $f(u_1)-f(u_2) \equiv v_1-v_2$. Let $w=u_1- u_2$, then $w$ satisfies \begin{gather*} -d_0\triangle w+f'(\theta)(v_1+f(u_2))w= 0,\quad x \in \Omega\\ \frac{\partial w}{\partial \nu}+r_0 w=0,\quad x \in \partial\Omega. \end{gather*} Therefore, if $w < 0$ somewhere, by maximum principle, \[ \min_{\bar \Omega} w=\min_{\partial \Omega} w<0. \] Assume that $x_0 \in \partial \Omega$ such that \[ w(x_0)=\min_{\partial \Omega} w<0, \] then, by Hopf's Lemma, at $x_0 $, we have $\frac{\partial w}{\partial \nu}<0$. This contradicts to the boundary condition. Therefore, $w \geq 0$, that is $u_1 \geq u_2$. \end{proof} \subsection{Stability of Steady States} \begin{theorem}\label{P:n401} $U_w=(S(x), 0)$ is unstable. \end{theorem} \begin{proof} The linearized system of (\ref{eq:n01}) around $U_w$ is \begin{gather*} w_{0t}=d_0\triangle w_0-f(S)w_1, \quad x \in\Omega,\\ w_{1t}=d_1\triangle w_1+f(S)w_1, \quad x \in\Omega,\\ \frac{\partial w_0}{\partial\nu}+r_0(x) w_0 =u^0(x), \quad x \in\partial \Omega,\\ \frac{\partial w_1}{\partial\nu}+r_1(x) w_1 =0, \quad x \in\partial \Omega. \end{gather*} Now we study the eigenvalue problem \begin{equation}\label{eq:n403} \begin{gathered} d_0\triangle w_0-f(S)w_1=\eta w_0, \quad x \in\Omega,\\ d_1\triangle w_1+f(S)w_1=\eta w_1, \quad x \in\Omega,\\ \frac{\partial w_0}{\partial\nu}+r_0(x) w_0 =0, \quad x \in\partial \Omega,\\ \frac{\partial w_1}{\partial\nu}+r_1(x) w_1 =0, \quad x \in\partial \Omega. \end{gathered} \end{equation} To prove the Theorem, we need to prove that the largest eigenvalue of (\ref{eq:n403}) is positive. Let $\eta$ be an eigenvalue of (\ref{eq:n403}) with eigenfunction $(w_0, w_1)$. If $w_1 \not \equiv 0$, then $\eta$ is an eigenvalue of $d_1\triangle +f(S)$ with homogeneous Robin boundary condition. Therefore, $\eta$ must be real. If $w_1 \equiv 0$, then $w_0 \not \equiv 0$. So $\eta$ is an eigenvalue of $d_0\triangle $ with homogeneous Robin boundary condition. Therefore, $\eta$ is also real. So we know that all eigenvalues of (\ref{eq:n403}) are real. Let $\eta_1$ be the largest eigenvalue of (\ref{eq:n403}). Since $f(S)>0$, the principal eigenvalue $\lambda_1$ of \begin{gather*} d_1\triangle w_1+f(S)w_1=\lambda w_1, \quad x \in\Omega,\\ \frac{\partial w_1}{\partial\nu}+r_1(x) w_1 =0, \quad x \in\partial \Omega, \end{gather*} is positive and the associated eigenfunction $\tilde w_1>0$ . We claim that $\lambda_1$ is also an eigenvalue of (\ref{eq:n403}). In fact, let $\tilde w_0$ be the solution of the linear boundary-value problem \begin{gather*} d_0\triangle w_0-f(S)\tilde w_1=\lambda_1 w_0, \quad x \in\Omega,\\ \frac{\partial w_0}{\partial\nu}+r_0(x) w_0 =0, \quad x \in\partial \Omega, \end{gather*} then $(w_0, w_1)=(\tilde w_0, \tilde w_1)$ satisfies (\ref{eq:n403}) with $\eta=\lambda_1$. So $\lambda_1>0$ is an eigenvalue of (\ref{eq:n403}). Therefore, $\eta_1\geq \lambda_1>0$. Hence $U_w$ is unstable. \end{proof} \begin{lemma}\label{L:nn41} Suppose that $(u, v)$ is a positive solution of \eqref{eq:n03}, then we have \begin{gather*} 00$ and $\psi_1(x)>0$ are the normalized principal eigenfunctions of the eigenvalue problems \begin{gather*} -\triangle \phi=\lambda \phi, \quad x \in\Omega,\\ \frac{\partial \phi}{\partial {\nu} }+r_0(x)\phi=0,\quad x \in\partial \Omega, \end{gather*} and \begin{gather*} -\triangle \psi=\mu \psi, \quad x \in\Omega,\\ \frac{\partial \psi}{\partial {\nu} }+r_1(x)\psi=0,\quad x \in\partial \Omega, \end{gather*} associated with the eigenvalues $\lambda_1>0$ and $\mu_1>0$. By the definition of upper and lower solutions, $p_i(t)$, $i=1,2$, should satisfy \begin{equation}\label{eq:n408} \begin{split} &(p_1(t)\phi_1(x)+\tilde u(x))_t-d_0\triangle(p_1(t)\phi_1(x)+\tilde u(x))\\ &+(\tilde v(x)-p_2(t)\psi_1(x))f(p_1(t)\phi_1(x)+\tilde u(x))\\ & \geq 0\\ &\geq (\tilde u(x)-p_1(t)\phi_1(x))_t -d_0\triangle( \tilde u(x)-p_1(t)\phi_1(x))\\ &\quad+(p_2(t)\psi_1(x)+\tilde v(x))f(\tilde u(x)-p_1(t)\phi_1(x)), \end{split} \end{equation} and \begin{equation}\label{eq:n409} \begin{split} &(p_2(t)\psi_1(x)+\tilde v(x))_t-d_1\triangle (p_2(t)\psi_1(x)+\tilde v(x))\\ &-(p_2(t)\psi_1(x)+\tilde v(x))(f(p_1(t)\phi_1(x)+\tilde u(x)) -(p_2(t)\psi_1(x)+\tilde v(x)))\\ &\geq 0\\ &\geq (\tilde v(x)-p_2(t)\psi_1(x))_t -d_1\triangle( \tilde v(x)-p_2(t)\psi_1(x))\\ &\quad -(\tilde v(x)-p_2(t)\psi_1(x))(f(\tilde u(x)-p_1(t)\phi_1(x)) -(\tilde v(x)-p_2(t)\psi_1(x))). \end{split} \end{equation} From the left-hand side of (\ref{eq:n408}), we need \begin{align*} &p_1'(t)\phi_1(x)-d_0p_1(t)\triangle\phi_1(x)-d_0\triangle\tilde u(x) +\tilde v(x)f(p_1(t)\phi_1(x)+\tilde u(x)) \\ &-p_2(t)\psi_1(x)f(p_1(t)\phi_1(x)+\tilde u(x)) \geq 0; \end{align*} that is, \begin{align*} &p_1'(t)\phi_1(x)+\lambda_1d_0p_1(t)\phi_1(x)-\tilde v(x)f(\tilde u(x)) +\tilde v(x)f(p_1(t)\phi_1(x)+\tilde u(x)) \\ &-p_2(t)\psi_1(x)f(p_1(t)\phi_1(x)+\tilde u(x)) \geq 0. \end{align*} Therefore, we need only \[ p_1'(t)\phi_1(x)+\lambda_1d_0p_1(t)\phi_1(x) -p_2(t)\psi_1(x)f(p_1(t)\phi_1(x) +\tilde u(x)) \geq 0. \] By Taylor's Theorem, we have \[ f(p_1\phi_1+\tilde u)=f(\tilde u)+f'(\tilde u)p_1\phi_1 +\frac{f''(\theta)}{2}p_1^2\phi_1^2 \leq f(\tilde u)+f'(\tilde u)p_1\phi_1. \] Therefore, we need only \[ p_1'\phi_1+\lambda_1d_0p_1\phi_1 -p_2\psi_1f(\tilde u) -p_2\psi_1f'(\tilde u)p_1\phi_1\geq 0, \] or \begin{equation}\label{eq:n410} p_1'\phi_1+\lambda_1d_0p_1\phi_1 -p_2\psi_1f(\tilde u) \geq f'(\tilde u)p_1p_2\phi_1\psi_1. \end{equation} Similarly, from the right-hand side (\ref{eq:n408}), we need \begin{equation}\label{eq:n411} p_1'\phi_1+\lambda_1d_0p_1\phi_1 -p_2\psi_1f(\tilde u)\geq 0. \end{equation} From the left-hand side of (\ref{eq:n409}), we need \begin{equation}\label{eq:n412} p_2'\psi_1+\mu_1d_1p_2\psi_1 -p_2\psi_1f(\tilde u)- \tilde v f'(\tilde u)p_1\phi_1 \geq f'(\tilde u)p_1p_2\phi_1\psi_1, \end{equation} and from the right-hand side of (\ref{eq:n409}), we need \[ p_2'\psi_1+\mu_1d_1p_2\psi_1 -\tilde v f(\tilde u) +\tilde v f(\tilde u-p_1\phi_1)-p_2\psi_1f(\tilde u) \geq 0. \] By Taylor's Theorem, we have \[ f(\tilde u-p_1\phi_1)=f(\tilde u)-f'(\theta)p_1\phi_1\geq f(\tilde u) -f'(0)p_1\phi_1. \] Therefore, we need \begin{equation}\label{eq:n413} p_2'\psi_1+\mu_1d_1p_2\psi_1 -\tilde v f'(0)p_1\phi_1-p_2\psi_1f(\tilde u) \geq 0. \end{equation} Combining (\ref{eq:n410}) to (\ref{eq:n413}), observing that $\phi_1 \leq 1$ and $\psi_1 \leq 1$, we need \[ p_1'\phi_1+\lambda_1d_0p_1\phi_1 -p_2\psi_1f(\tilde u) \geq f'(\tilde u)p_1p_2, \] and \[ p_2'\psi_1+\mu_1d_1p_2\psi_1 -p_2\psi_1f(\tilde u)- \tilde v f'(0)p_1\phi_1 \geq f'(\tilde u)p_1p_2. \] Let $\rho =\min_{\bar\Omega}\phi_1(x)>0$, $\sigma=\min_{\bar\Omega}\psi_1(x)>0$, and take $p_1=p_2=p$. Then, by Lemma \ref{L:nn41}, we need $p$ to satisfy \begin{gather*} p'+(\lambda_1d_0 -\frac{f(\Hat S)}{\rho})p \geq \frac {f'(0)}{\rho}p^2,\\ p'+(\mu_1d_1 -\frac{f(\Hat S)}{\sigma}-\frac{\Hat V f'(0)}{\sigma})p \geq \frac {f'(0)}{\sigma}p^2. \end{gather*} Therefore, if there exists an $\epsilon>0$ such that \[ \lambda_1d_0 -\frac{f(\Hat S)}{\rho} \geq \epsilon, \quad\mbox{and}\quad \mu_1d_1 -\frac{f(\Hat S)}{\sigma}-\frac{\Hat V f'(0)}{\sigma}\geq \epsilon, \] then we need only take $p$ such that \[ p'+\epsilon p \geq M p^2, \] where $M=\max\{\frac {f'(0)}{\rho},\frac {f'(0)}{\sigma}\}$. In particular we take $p$ such that $p'+\epsilon p = M p^2$, then we have \[ p(t)=\frac{1}{M/\epsilon+(1/p(0)-M/\epsilon)e^{\epsilon t}}, \] where $ 00$. From Theorem \ref{T:n01}, we have the following theorem \begin{theorem}\label{T:302} Equation \eqref{eq:301} has two nonnegative solutions: \begin{gather*} U_2=(\tilde u(x), \tilde v_1(x), 0) \quad \text{with} \quad \tilde u(x)>0, \quad \tilde v_1(x) >0,\\ U_3=(\tilde u(x), 0, \tilde v_2(x)) \quad \text{with} \quad \tilde u(x)>0, \quad \tilde v_2(x) >0. \end{gather*} \end{theorem} \begin{theorem}\label{T:301} Assume that $f_1=f_2=f$, then \eqref{eq:301} has no positive solution. \end{theorem} \begin{proof} If $f_1=f_2=f$ and \eqref{eq:301} has a positive solution $U=(u(x), v_1(x), v_2(x))$ with $u(x)>0$ and $v_i(x)>0$, then $v_1(x)>0$ and $v_2(x)>0$ satisfy \begin{gather*} d_1\triangle{v_1}+(f(u)-v_1- v_2)v_1=0, \quad x \in \Omega, \\ d_2\triangle{v_2}+(f(u)- v_1-v_2)v_2=0, \quad x \in \Omega, \\ \frac{\partial v_i}{\partial {\nu} }+r_i(x)v_i=0,\quad i=1,2,\; x \in\partial \Omega. \end{gather*} Because of the positivity of $v_i(x)$, we can look $v_i(x)$ as the principal eigenfunction of the eigenvalue problem \begin{gather*} d_i\triangle \phi+q(x)\phi=\lambda \phi,\quad x \in \Omega,\\ \frac{\partial\phi}{\partial \nu}+r_i(x)\phi=0,\quad x \in\partial \Omega, \end{gather*} with $q(x)=f(u(x))-\sum_{j=1}^{2}v_j(x)$, associated with the principal eigenvalue $\lambda=\lambda(q(x),d_i,r_i)=0$. So we have \[ \lambda(q(x),d_1,r_1)=\lambda(q(x),d_2,r_2). \] Since $q(x)=f(u(x))-\sum_{j=1}^{2}v_j(x)$ is not constant, by Lemma \ref{L:301}, this contradicts the assumption $d_10$ and $v_2(x)>0$ if $f_1=f_2$. Thus we know that if $u^0(x) \not\equiv 0$ and $f_1=f_2$, \eqref{eq:301} has only the following three types of solutions \[ (S(x), 0, 0),\quad(\tilde u(x), \tilde{v_1}(x), 0),\quad(\tilde u(x),0, \tilde{v_2}(x)), \] with $S(x)>0$, $\tilde u(x)>0$ and $\tilde{v_i}(x)>0$. \section{Stability Analysis} In this section, we study the stability of equilibrium solutions of (\ref{eq:in1}) under the assumption $f_1=f_2=f$. From Section 4, we know that, if $f_1=f_2=f$, all equilibria of (\ref{eq:in1}) are $U_1=(S(x), 0, 0)$, $U_2=(\tilde u(x), \tilde v_1(x), 0)$ and $U_3=(\tilde u(x),0, \tilde v_2(x))$, where $S(x)>0$ is the positive solution of \eqref{eq:201}, $\tilde u(x)>0$ and $\tilde v_i(x)>0$ satisfy \begin{equation}\label{eq:401} \begin{gathered} d_0\triangle \tilde{u}-f(\tilde{u})\tilde v_i=0,\quad x \in\Omega,\\ d_i\triangle \tilde v_i+\tilde v_i (f(\tilde{u})- \tilde v_i )=0, \quad x \in\Omega,\\ \frac{\partial\tilde u}{\partial\nu}+r_0(x)\tilde u =u^0(x), \quad x \in\partial \Omega,\\ \frac{\partial\tilde v_i}{\partial {\nu} }+r_i(x)\tilde v_i=0, \quad x \in\partial \Omega. \end{gathered} \end{equation} Observe that, because of the positivity of $\tilde v_i(x)$, from the second equation of (\ref{eq:401}), we can look $\tilde v_i(x)$ as the principal eigenfunction of the eigenvalue problem \begin{gather*} d_i\triangle \phi+q(x)\phi=\lambda \phi, \quad x \in\Omega,\\ \frac{\partial \phi}{\partial {\nu} }+r_i(x)\phi=0,\quad x \in\partial \Omega, \end{gather*} with $q(x)=f(\tilde{u})- \tilde v_i$, associated with the principal eigenvalue $\lambda=\lambda(q(x),d_i,r_i)=0$, $i =1,2$. Now we study the stability of $U_i$. It is well-known (see \cite{HK1}) that the stability question for $U_i$ is answered by considering the corresponding eigenvalue problem for the linearized operator around $U_i$. Namely, let us substitute $U(x,t)=(u(x,t),v_1(x,t), v_2(x,t))=U_i+W(x,t)=U_i+(w_0(x,t),w_1(x,t), w_2(x,t))$ into (\ref{eq:in1}) and then pick up all the terms which are linear in $W$: \begin{equation}\label{eq:402} \frac{\partial W}{\partial t} = D\triangle{W}+F'(U_i)W, \end{equation} where \[D=\begin{pmatrix}d_0&0&0\\ 0&d_1&0 \\ 0&0&d_2\end{pmatrix}\] and \[ F'(U_i)= \begin{pmatrix} -f'(u)(v_1+v_2) &- f(u) & - f(u)\\ v_1f'(u)& f(u)-2 v_1-v_2&-v_1 \\ v_2f'(u)&-v_2 & f(u)- v_1-2v_2 \end{pmatrix}_{U_i} .\] \begin{theorem}\label{P:401} The solution $U_1=(S(x), 0, 0)$ is unstable. \end{theorem} The proof is similar to that of Theorem \ref{P:n401}, therefore, we omit it. \begin{theorem}\label{P:402} Assume that $f'' \leq 0$, then there exist constants $K_0$ and $K_1$ depending on $f$, $u^0$ and $r_i$ such that if $d_0 \geq K_0$ and $d_1 \geq K_1$, $U_2=(\tilde u(x), \tilde v_1(x), 0)$ is asymptotically stable. \end{theorem} \begin{proof} From (\ref{eq:402}), the linearized system of (\ref{eq:in1}) around $U_2$ is \begin{gather*} w_{0t}=d_0\triangle w_0-f'(\tilde u)\tilde v_1w_0-f(\tilde u) w_1 - f(\tilde u)w_2, \quad x \in\Omega,\\ w_{1t}=d_1\triangle w_1+\tilde v_1f'(\tilde u)w_0+(f(\tilde u) -2\tilde v_1)w_1-\tilde v_1w_2 , \quad x \in\Omega,\\ w_{2t}=d_2\triangle w_2+(f(\tilde u)-\tilde v_1 )w_2,\quad x \in\Omega,\\ \frac{\partial w_0}{\partial\nu}+r_0(x) w_0 =u^0(x), \quad x \in\partial \Omega,\\ \frac{\partial w_i}{\partial {\nu} }+r_i(x)w_i=0,\;i=1,2,\quad x \in\partial \Omega. \end{gather*} Therefore, we need to study the eigenvalue problem: \begin{equation}\label{eq:405} \begin{gathered} d_0\triangle w_0-f'(\tilde u)\tilde v_1w_0-f(\tilde u) w_1- f(\tilde u)w_2 =\eta w_0, \quad x \in\Omega,\\ d_1\triangle w_1+\tilde v_1f'(\tilde u)w_0+(f(\tilde u)-2\tilde v_1)w_1 -\tilde v_1w_2=\eta w_1, \quad x \in\Omega,\\ d_2\triangle w_2+(f(\tilde u)-\tilde v_1 )w_2=\eta w_2,\quad x \in\Omega,\\ \frac{\partial w_0}{\partial\nu}+r_0(x) w_0 =0, \quad x \in\partial \Omega,\\ \frac{\partial w_i}{\partial {\nu} }+r_i(x)w_i=0,\;i=1,2,\quad x \in\partial \Omega. \end{gathered} \end{equation} Let $\eta_1$ be the largest eigenvalue of (\ref{eq:405}) and $(w_0,w_1, w_2)$ be the corresponding eigenfunctions. If $w_2(x)\not\equiv 0$, then $\eta_1$ is an eigenvalue of $d_2\triangle+(f(\tilde u)-\tilde v_1)$ with Robin boundary condition. So, by Lemma \ref{L:301}, we have \[ \eta_1 \leq \lambda(f(\tilde u)-\tilde v_1,d_2,r_2) <\lambda(f(\tilde u)-\tilde v_1,d_1,r_1)=0. \] If $w_2(x)\equiv 0$, it is easily seen that we must have $w_1(x)\not\equiv 0$ and $w_0(x)\not\equiv 0$. In this case, $\eta_1$ satisfies \begin{gather*} d_0\triangle w_0-f'(\tilde u)\tilde v_1w_0-f(\tilde u) w_1=\eta_1 w_0, \quad x \in\Omega,\\ d_1\triangle w_1+\tilde v_1f'(\tilde u)w_0+(f(\tilde u)-2\tilde v_1)w_1 =\eta_1 w_1, \quad x \in\Omega,\\ \frac{\partial w_0}{\partial\nu}+r_0(x) w_0 =0, \quad x \in\partial \Omega,\\ \frac{\partial w_1}{\partial {\nu} }+r_1(x)w_1=0,\quad x \in\partial \Omega. \end{gather*} Therefore, $\eta_1$ is also an eigenvalue of the eigenvalue problem \begin{equation}\label{eq:n405} \begin{gathered} d_0\triangle w_0-f'(\tilde u)\tilde v w_0-f(\tilde u) w_1=\eta w_0, \quad x \in\Omega,\\ d_1\triangle w_1+\tilde vf'(\tilde u)w_0+(f(\tilde u)-2\tilde v)w_1 =\eta w_1, \quad x \in\Omega,\\ \frac{\partial w_0}{\partial\nu}+r_0(x) w_0 =0, \quad x \in\partial \Omega,\\ \frac{\partial w_1}{\partial {\nu} }+r_1(x)w_1=0,\quad x \in\partial \Omega. \end{gathered} \end{equation} This is the linearized system of (\ref{eq:n01}) around $U_p$. From Theorem \ref{P:n402}, we know that all eigenvalues of (\ref{eq:n405}) are negative. Therefore, $\eta_1<0$. Hence all eigenvalues of (\ref{eq:401}) are negative. Therefore, $U_2$ is asymptotically stable. \end{proof} \begin{theorem}\label{P:403} The solution $U_3=(\tilde u(x),0, \tilde v_2(x))$ is unstable. \end{theorem} \begin{proof} As in the proof of the previous theorem, we need to study the eigenvalue problem \begin{equation}\label{eq:408} \begin{gathered} d_0\triangle w_0-f'(\tilde u)\tilde v_2w_0-f(\tilde u) w_1- f(\tilde u)w_2 =\eta w_0, \quad x \in\Omega,\\ d_1\triangle w_1+(f(\tilde u)-\tilde v_2 )w_1=\eta w_1 , \quad x \in\Omega,\\ d_2\triangle w_2+\tilde v_2f'(\tilde u)w_0-\tilde v_2w_1+(f(\tilde u)-2\tilde v_2)w_2 =\eta w_2,\quad x \in\Omega,\\ \frac{\partial w_0}{\partial\nu}+r_0(x) w_0 =0, \quad x \in\partial \Omega,\\ \frac{\partial w_i}{\partial {\nu} }+r_i(x)w_i=0,\;i=1,2,\quad x \in\partial \Omega. \end{gathered} \end{equation} Let $\lambda_1=\lambda(f(\tilde u)-\tilde v_2, d_1,r_1)$ and $\phi_1(x)$ be the principal eigenpair of \begin{gather*} d_1\triangle \phi+(f(\tilde u)-\tilde v_2)\phi=\lambda \phi, \quad x \in\Omega,\\ \frac{\partial \phi}{\partial {\nu} }+r_1(x)\phi=0, \quad x \in\partial \Omega, \end{gather*} then we have $\lambda_1>\lambda(f(\tilde u)-\tilde v_2, d_2,r_2)=0$. Let $(\tilde w_0, \tilde w_2)$ be the solution of the following linear boundary value problem \begin{gather*} d_0\triangle w_0-(f'(\tilde u)\tilde v_2+\lambda_1)w_0- f(\tilde u)w_2 =f(\tilde u) \phi, \quad x \in\Omega,\\ d_2\triangle w_2+\tilde v_2f'(\tilde u)w_0+(f(\tilde u)-2\tilde v_2 -\lambda_1)w_2=\tilde v_2\phi,\quad x \in\Omega,\\ \frac{\partial w_0}{\partial\nu}+r_0(x) w_0 =0, \quad x \in\partial \Omega,\\ \frac{\partial w_2}{\partial {\nu} }+r_2(x)v_2=0,\quad x \in\partial \Omega, \end{gather*} then it is easily seen that $\lambda_1>0$ is an eigenvalue of \eqref{eq:408} with eigenfunction $(w_0,w_1, w_2)=(\tilde w_0, \phi_1, \tilde w_2)$. Therefore, $U_3$ is unstable. \end{proof} \subsection*{Acknowledgements} This work was partly carried out while the author was working in the University of Connecticut as a Post-doctoral Fellow. The author would like to thank Dr. Xuefeng Wang for his valuable suggestions and comments. \begin{thebibliography}{99} \bibitem{AH1} H. Amann, \emph{Dynamics theory of quasilinear parabolic equations-I. Abstract evolution equations}, Nonlinear Anal, TMA 12(1988), 895-919. \bibitem{AH2} H. Amann, \emph{Dynamics theory of quasilinear parabolic equations-II. Reaction-diffusion system}, Diff. Int. Eq. 3(1990), 13-75. \bibitem{AH3} H. Amann, \emph{Dynamics theory of quasilinear parabolic equations-III. Global Existence}, Math. Z. 202(1989), 219-250. \bibitem{DHMP} J. Dockery, V. Hutson, K. Mischaikow \& M. Pernarowski; \emph{The evolution of slow dispersal rates: a reaction diffusion model}, J. Math. Biol., 37(1998), 61-83. \bibitem{CGYL} C. Gui and Y. Lou, \emph{Uniqueness and nonuniqueness of coexistence states in the Lotka-Volterra competition model}, Comm. Pure and Applied Math. Vol XLVII(1994), 1571-1594. \bibitem{DH81} D. Henry, \emph{Geometric theory of semilinear parabolic equations}, Lecture Notes in Mathematics, No. 840, Springer-Verlag, Berlin, Heidelberg, New York, Tokyo, 1981. \bibitem{SBH} S.-B. Hsu, \emph{Steady states of a system of partial differential equations modeling microbial ecology}, SIAM J. Math Anal, Vol. 14, No. 6 (1983), 1130-1138. \bibitem{HW1} S.-B. Hsu and P. Waltman, \emph{On a system of reaction-diffusion equations arising from competition in an unstirred chemostat}, SIAM J. Appl. Math, Vol. 53, No.4, pp 1026-1044, August 1993. \bibitem{HK1} H. Kielh\"{o}fer, \emph{Stability and semilinear evolution equations in Hilbert space}, Arch. Rational Mech. Anal. 57(1974), 150-165. \bibitem{LE1} L. Dung and H. L. Smith, \emph{A parabolic system modeling microbial competition in an unmixed bio-reactor}, J. Diff. Eqns. Vol. 130 (1996), 59-91. \bibitem{LIU} Y. Liu, \emph{Positive solutions to general elliptic systems}, Nonlinear Anal, TMA 25(1995), No. 3, 229-246. \bibitem{CVP} C. V. Pao, \emph{Quasisolutions and global attractor of reaction-diffusion systems}, Nonlinear Analysis, Vol. 26, No.12(1996), 1889-1903. \bibitem{HLS} H. L. Smith, \emph{Monotone Dynamical Systems: An introduction to the theory of competitive and cooperative systems}, AMS Mathematical Surveys and Monographs, Vol. 41, 1991. \bibitem{SMO} J. Smoller, \emph{Shock waves and reaction-diffusion equations}, Springer-Verlag, New York, 1983. \bibitem{XW1} X. Wang, \emph{Qualitative behavior of solutions of chemotactic diffusion systems: effects of motility and chemotaxis and dynamics}, SIAM J. Math Anal, Vol. 31, No. 3, pp 535-560, 2000. \bibitem{XW2} X. Wang and Y. Wu, \emph{Qualitative analysis on a chemotactic diffusion model for two species competing for a limited resource}, Quart. Appl. Math. 60 (2002), No. 3, 505--531. \bibitem{LY1} Q. Ye and Z. Li, \emph{An introduction to reaction-diffusion equations}, Scientific Press, 1994. \bibitem{SW1} J. So and P. Waltman, \emph{A nonlinear boundary value problem arising from competition in the chemostat}, Appl. Math. Comput., 32(1989), pp. 169-183. \end{thebibliography} \end{document}