\documentclass[reqno]{amsart} \usepackage{hyperref} \AtBeginDocument{{\noindent\small \emph{Electronic Journal of Differential Equations}, Vol. 2014 (2014), No. 110, pp. 1--14.\newline ISSN: 1072-6691. URL: http://ejde.math.txstate.edu or http://ejde.math.unt.edu \newline ftp ejde.math.txstate.edu} \thanks{\copyright 2014 Texas State University - San Marcos.} \vspace{9mm}} \begin{document} \title[\hfilneg EJDE-2014/110 \hfil Polyconvolution and the T-H intergral equation] {Polyconvolution and the Toeplitz plus Hankel integral equation} \author[N. X. Thao, N. M. Khoa, P. T. V. Anh \hfil EJDE-2014/110\hfilneg] {Nguyen Xuan Thao, Nguyen Minh Khoa, Phi Thi Van Anh} % in alphabetical order \address{Nguyen Xuan Thao \newline School of Applied Mathematics and Informatics, Hanoi University of Science and Technology, No. 1, Dai Co Viet, Hanoi, Vietnam} \email{thaonxbmai@yahoo.com} \address{Nguyen Minh Khoa \newline Department of Mathematics, Electric Power University, 235 Hoang Quoc Viet, Cau Giay, Hanoi, Vietnam} \email{khoanm@epu.edu.vn} \address{Phi Thi Van Anh \newline Department of Mathematics, University of Transport and Communications, Lang Thuong, Dong Da, Hanoi, Vietnam} \email{vananh.utcmath@gmail.com} \thanks{Submitted November 23, 2013. Published April 16, 2014.} \subjclass[2000]{44A05, 44A15, 44A20, 44A35, 45E10} \keywords{Convolution; polyconvolution; integral equation; integral transform; \hfill\break\indent Toeplitz plus Hankel} \begin{abstract} In this article we introduce a polyconvolution which related to the Hartley and Fourier cosine transforms. We prove some properties of this polyconvolution, and then solve a class of Toeplitz plus Hankel integral equations and systems of two Toeplitz plus Hankel integral equations. \end{abstract} \maketitle \numberwithin{equation}{section} \newtheorem{theorem}{Theorem}[section] \newtheorem{corollary}[theorem]{Corollary} \newtheorem{definition}[theorem]{Definition} \newtheorem{example}[theorem]{Example} \allowdisplaybreaks \section{Introduction} In studying the physical problems related to fluid dynamics, filtering theory, and wave diffraction, the following equation has been considered \cite{ChadanSabatier,Kailath1966, KagiwadaKalaba,TsiLevy1981}, $$\label{ptTH} f(x)+\int_0^{\infty}f(y)[k_1(x-y)+k_2(x+y)]dy=g(x),\quad x>0,$$ here $k_1,k_2,g$ are given, and $f$ is unknown function. This equation is called the Toeplitz plus Hankel integral equation, with $k_1(x-y)$ is Toeplitz kernel and $k_2(x+y)$ is Hankel kernel. Up to now, solving \eqref{ptTH} in the general case is still open. In recent years, there have been several results solving this equation in special cases of $k_1,k_2$ and the solutions are obtained in closed form by convolution tool. In \cite{ThaoTuanHong2008}, the authors obtained the explicit solutions of the equation \eqref{ptTH} in case the Toeplitz kernel is even function and $k_1,k_2$ have special forms \begin{gather*} k_1(t)= \frac 1{2\sqrt{2\pi}} \operatorname{sign}(t-1)h_1(|t-1|) -\frac 1{2\sqrt{2\pi}}h_1(t+1)-\frac 1{\sqrt {2\pi}}h_2(t)\\ k_2(t)= \frac 1{2\sqrt{2\pi}} \operatorname{sign}(t+1)h_1(|t+1|) -\frac 1{2\sqrt{2\pi}}h_1(t+1)+\frac 1{\sqrt {2\pi}}h_2(|t|) \end{gather*} where $h_1(x)=(\varphi_1\underset{2}\ast \varphi_2)(x)$, $\varphi_1,\varphi_2, h_2\in L_1(\mathbb{R}_+)$ and $(\cdot \underset{2}\ast \cdot )$ is the generalized convolution for Fourier sine and Fourier cosine transforms \cite{Sneddon1972}. Some special cases of $k_1,k_2$, where $k_1$ is still even function or special right-hand-side for arbitrary kernels are considered in \cite{ThaoTuanHong2011}. In case, when $k_1, k_2$ are periodic functions with period $2\pi$, the explicit solution to a class of equation \eqref{ptTH} on a period $[0,2\pi]$, $$f(x)+\int_0^{2\pi} f(y)[k_1(|x-y|)+k_2(x+y)]dy=g(x),\quad x\in [0,2\pi].$$ is introduced in \cite{AnhTuanTuan2013}. In \cite{ThaoTuanHAnh2013}, the authors obtained the solution of \eqref{ptTH} to the case $k_1=k_2$ in real number $$f(|x|)+\frac 1{2\pi}\int_0^{\infty}f(y)[k(x-y)+k(x+y)]dy=g(x),\quad x\in \mathbb{R}.$$ In this article, we consider a modified type of the equation \eqref{ptTH}, with the integral real domain $$f(x)+\lambda\int_{-\infty}^{\infty}f(y)[k_1(x-y)+k_2(x+y)]dy=p(x),\quad x\in \mathbb{R} \label{ptTHm}$$ where, for $t\in \mathbb{R}$, \begin{align} k_1(t):=\int_0^{\infty}g(v)[h(-t+v)+h(t-v)+h(-t-v)+h(t+v)]dv, \label{k1}\\ k_2(t):=\int_0^{\infty}g(v)[-h(t+v)+h(-t-v)-h(t-v)+h(-t+v)]dv, \label{k2} \end{align} and $g,h,p$ are given functions, $f$ is an unknown function. In this case, we see that the Toeplitz kernel $k_1$ is still an even function. The tool to solve this equation in closed form is a new polyconvolution related to Hartley and Fourier cosine transforms. Convolutions have many applications \cite{BogveradzeCastro2008,Sneddon1972, ThaoTuanHong2011,ThaoKhoaPAnh2013, Titch1986, TsiLevy1981}. The concept of polyconvolution was first proposed by Kakichev in 1997 \cite{kakichev1997}. According to this definition, the polyconvolution of $n$, ($n\in \mathbb{N}$, $n\ge 3$) functions $f_1,f_2,\dots,f_n$ for $n+1$ arbitrary integral transforms $T,T_1,T_2,\dots,T_n$ with weight-function $\gamma(x)$ is denoted $\overset{\gamma}\ast (f_1,f_2,\dots,f_n)(x)$, for which the factorization property holds \begin{equation*} %\label{TT1Tn} T[\overset{\gamma}\ast (f_1,f_2,\dots,f_n)](y) =\gamma(y)\cdot (T_1f_1)(y)\cdot (T_2f_2)(y)\cdots(T_nf_n)(y). \end{equation*} Our new polyconvolution $*(f,g,h)(x)$, $x\in \mathbb{R}$ has the following factorization equalities \begin{gather} H_1[*(f,g,h)](y)=(H_1f)(y)\cdot(F_cg)(y)\cdot(H_2h)(y),\quad \forall y\in \mathbb{R},\label{dtnthH1}\\ H_2[*(f,g,h)](y)=(H_2f)(y)\cdot(F_cg)(y)\cdot(H_1h)(y),\quad \forall y\in \mathbb{R},\label{dtnthH2} \end{gather} where $H_1,H_2$ are Hartley transforms and $F_c$ is Fourier cosine transform. The paper is organized as follows. In Section 2, we recall some known related results about convolutions. In Section 3, we define the new polyconvolution $*(f,g,h)(x)$ for Hartley and Fourier cosine integral transforms, whose factorization equalities are in the forms \eqref{dtnthH1}-\eqref{dtnthH2} and prove its existence on the certain function spaces. Its boundedness property on $L_p(\mathbb{R})$ is also considered. In Section 4, with the help of new polyconvolution $*(f,g,h)(x)$, we solve the equation \eqref{ptTH} for the case $k_1,k_2$ are determined by \eqref{k1}-\eqref{k2}. The systems of two equations are considered in this section. \section{Preliminaries} The following well-known transforms are used in this paper. We denote $F$ by the Fourier transform, $F_c$ by Fourier cosine transform, $H_1$ and $H_2$ by Hartley transforms, which are known in \cite{Bracewell1986, PaleyWiener1934, Sneddon1972}. \begin{align} (Ff)(y) =\frac 1{\sqrt{2\pi}} \int_{-\infty}^{\infty}f(x)e^{-ixy}dx,\quad y\in \mathbb{R};\\ (F_cf)(y)=\sqrt{\frac 2\pi}\int_0^{\infty}f(x)\cos(xy)dx,\quad y\in \mathbb{R}_+; \label{FFc} \\ (H_1f)(y) =\frac 1{\sqrt{2\pi}} \int_{-\infty}^{\infty}f(x)\operatorname{cas}(xy)dx,\quad y\in \mathbb{R};\\ (H_2f)(y)=\frac 1{\sqrt{2\pi}}\int_0^{\infty}f(x)\operatorname{cas}(-xy)dx,\quad y\in \mathbb{R}; \label{H1H2} \end{align} here $\operatorname{cas} u = \cos u+\sin u$. Next, we recall the following convolutions, which will be used in the proof of some properties and solution of the equation \eqref{ptTHm}. First, the convolution for Hartley transform \cite{GiangMauTuan2009} , of two functions $f$ and $g$, has the form $$(f\underset{H}\ast g)(x)=\frac1{2\sqrt{2\pi}}\int_{-\infty}^{\infty}f(u)[g(x-u) +g(-x+u)-g(-x-u)+g(x+u)]du,\quad x\in \mathbb{R},$$ with its factorization equalities $$\label{conH} H_k(f\underset{H}\ast g)(y)=(H_kf)(y)(H_kg)(y),\quad\forall y\in \mathbb{R},\; k=1,2.$$ Second, the generalized convolution for Hartley and Fourier transforms \cite{ThaoHAnh2013}, of two functions $f$ and $g$, has the form $$\label{HF_equa} (f\underset{HF}\ast g)(x)=\frac1{2\sqrt{2\pi}}\int_{-\infty}^{\infty}f(u)[g(x-u)+g(x+u)+ig(-x-u)-ig(x+u)]du,\ \ \ x\in \mathbb{R},$$ where its factorization properties are $$\label{conHF} H_k(f\underset{HF}\ast g)(y)=(Ff)(y)(H_kg)(y),\quad\forall y\in \mathbb{R},\; k=1,2.$$ Third, the generalized convolution for Hartley and Fourier cosine transforms \cite{ThaoTuanHAnh2013}, of two functions $f$ and $g$ has the form \begin{equation*} (f\underset{HF_c}\ast g)(x)=\frac1{\sqrt{2\pi}}\int_0^{\infty}f(u) [g(x-u)+g(x+u)]du,\quad x\in \mathbb{R}, \end{equation*} where its factorization properties are $$\label{conHFc} H_k(f\underset{HF_c}\ast g)(y)=(F_cf)(y)(H_kg)(y),\quad\forall y\in \mathbb{R},\; k=1,2.$$ In this article, the function spaces $L_p(\mathbb{R})$ and $L_p(\mathbb{R}_+),\ p\ge 1$, are equipped with norms, $$\|f\|_{L_p(\mathbb{R})} = \Big(\int_{-\infty}^{\infty}|f(x)|^pdx\Big)^{1/p},\quad \|f\|_{L_p(\mathbb{R}+)} = \Big(\int_0^{\infty}|f(x)|^pdx\Big)^{1/p}.$$ Also, we define the function space $L_p^{\alpha,\beta,\gamma}(\mathbb{R})$, $\alpha>-1$, $\beta>0$, $\gamma>0$, $p>1$ by $$L_p^{\alpha,\beta,\gamma}(\mathbb{R}) :=\big\{f(x): \int_{-\infty}^{\infty}|x|^{\alpha} e^{-\beta |x|^{\gamma}}|f(x)|^pdx<\infty\big\}$$ with the norm $$\|f\|_{L_p^{\alpha,\beta,\gamma}(\mathbb{R})} =\Big({\int_{-\infty}^{\infty}|x|^{\alpha}e^{-\beta |x|^{\gamma}}|f(x)|^pdx} \Big)^{1/p}.$$ \section{A polyconvolution related to Hartley and Fourier cosine transforms} In this section, we define a new polyconvolution for Hartley and Fourier cosine transforms and then prove some its properties. \begin{definition}\label{dndc} \rm The polyconvolution related to Hartley and Fourier cosine transforms of three functions $f,g,h$ is defined by $$\label{poly} [*(f,g,h)](x):=\frac 1{4\pi}\int_{-\infty}^{\infty} f(u)[k_1(x-u)+k_2(x+u)]du,\quad x\in \mathbb{R},$$ where $k_1$ and $k_2$ are determined by \eqref{k1} and \eqref{k2} respectively. \end{definition} The most important feature of a new polyconvolution is its factorization property. Normally, each convolution has only one factorization equality. However, this polyconvolution is one of several convolutions or polyconvolutions, which has two factorization equalities. \begin{theorem}\label{dlnth} Assume that $f,h\in L_1(\mathbb{R})$ and $g\in L_1(\mathbb{R}_+)$. Then, the polyconvolution \eqref{poly} belongs to $L_1(\mathbb{R})$ and the norm inequality on $L_1(\mathbb{R})$ is of the form $$\label{chuanL1} \|*(f,g,h)\|_{L_1(\mathbb{R})}\le {\frac 2\pi}\|f\|_{L_1(\mathbb{R})}\|g\|_{L_1(\mathbb{R}_+)}\|h\|_{L_1(\mathbb{R})}.$$ Moreover, it satisfies the factorization identifies \eqref{dtnthH1} and \eqref{dtnthH2}. In case $h\in L_1(\mathbb{R})\cap L_2(\mathbb{R})$, the following Parseval type identity holds, \begin{align} *(f,g,h)(x) = \frac{1}{\sqrt{2\pi}}\int_{-\infty}^{\infty} (H_{\{ 1, 2 \}}f)(y)\cdot(F_cg)(y)\cdot(H_{\{2, 1\}}h)(y). \operatorname{cas}(\pm xy)dy.\label{dtPar1dc} \end{align} \end{theorem} \begin{proof} First we prove that $*(f,g,h)(x)\in L_1(\mathbb{R})$. Indeed, using the Fubini theorem, we write \begin{aligned} \int_{-\infty}^{\infty} |*(f,g,h)(x)|dx &\le \frac{1}{4\pi}\int_{-\infty}^{\infty}\int_{-\infty}^{\infty}|f(u)| \{|k_1(x-u)|+|h(x+u)|\}\,dx\,du \\ &\le \frac{1}{4\pi}\int_{-\infty}^{\infty}|f(u)|du \Big[\int_{-\infty}^{\infty}|k_1(t)|dt+\int_{-\infty}^{\infty}|k_2(t)|dt\Big] \end{aligned}\label{pTheo311} From \eqref{k1}, we have \begin{align*} &\int_{-\infty}^{\infty}|k_1(t)|dt\\ &\le \int_{-\infty}^{\infty}\int_0^{\infty}|g(v)| [|h(-t+v)|+|h(t-v)|+|h(-t-v)|+|h(t+v)|]\,dv\,dt\\ &\le 4\Big(\int_0^{\infty}|g(v)|dv\Big) \Big(\int_{-\infty}^{\infty}|h(t)|dt\Big). \end{align*} Similarly, with $k_2$, and continue with \eqref{pTheo311}, we obtain $$\int_{-\infty}^{\infty}|*(f,g,h)(x)|dx \le \frac 2\pi \Big(\int_{-\infty}^{\infty}|f(u)|du\Big) \Big(\int_0^{\infty}|g(v)|dv\Big)\Big(\int_{-\infty}^{\infty}|h(t)|dt\Big)<\infty$$ So $*(f,g,h)(x)$ belongs to $L_1(\mathbb{R})$ and we also get inequality \eqref{chuanL1}. Now we prove the factorization property \eqref{dtnthH1}. From \eqref{FFc} and \eqref{H1H2}, we write \begin{align*} &(H_1f)(y)\cdot(F_cg)(y)\cdot(H_2h)(y)\\ &= \frac{1}{\pi\sqrt{2\pi}}\int_{-\infty}^{\infty}\int_0^{\infty} \int_{-\infty}^{\infty}f(u)g(v)h(t)\operatorname{cas}(uy)\cos(vy)\operatorname{cas}(-ty)\,dt\,dv\,du, \quad \forall y\in \mathbb{R}. \end{align*} Using trigonometric transforms, one can easily see that \begin{align*} &(H_1f)(y)\cdot(F_cg)(y)\cdot(H_2h)(y)\\ &= \frac{1}{4\pi\sqrt{2\pi}}\int_{-\infty}^{\infty} \int_0^{\infty}\int_{-\infty}^{\infty}f(u)g(v)h(t)\Big\{\operatorname{cas}[(u+v-t)y]+ +\operatorname{cas}[(u+v+t)y]\\ &\quad -\operatorname{cas}[(-u-v+t)y]+\operatorname{cas}[(-u-v-t)y]+\operatorname{cas}[(u-v-t)y] \\ &\quad +\operatorname{cas}[(u-v+t)y]-\operatorname{cas}[(-u+v+t)y]+\operatorname{cas}[(-u+v-t)y]\Big\}\,dt\,dv\,du. \end{align*} Putting the corresponding substitution with each integral term in the above expression, we obtain \begin{align*} &(H_1f)(y)\cdot(F_cg)(y)\cdot(H_2h)(y)\\ &= \frac{1}{4\pi\sqrt{2\pi}} \int_{-\infty}^{\infty}\int_0^{\infty}\int_{-\infty}^{\infty} f(u)g(v)\Big[h(-x+u+v) +h(x-u-v)- h(x+u+v)\\ &\quad +h(-x-u-v)+h(-x+u-v) +h(x-u+v)- h(x+u-v)\\ &\quad +h(-x-u+v)\Big]\operatorname{cas}(xy) \,dx\,dv\,du . \end{align*} Using Fubini's theorem, change the order of integrating, and using \eqref{k1}, \eqref{k2}, we have \begin{align*} &(H_1f)(y)\cdot (F_cg)(y)\cdot (H_2h)(y)\\ &= \frac{1}{\sqrt{2\pi}}\int_{-\infty}^{\infty} \big\{\frac 1{4\pi}\int_{-\infty}^{\infty}f(u) [k_1(x-u)+k_2(x+u)]du\big\} \operatorname{cas}(xy)dx\\ &= \frac{1}{\sqrt{2\pi}}\int_{-\infty}^{\infty}[*(f,g,h)(x)]\operatorname{cas}(xy)dx \\ &= H_1(*(f,g,h))(y),\quad \forall y\in \mathbb{R}. \end{align*} This expression implies \eqref{dtnthH1}. Since $(H_1f)(y)=(H_2f)(-y)$, replacing $(y)$ by $(-y)$, we obtain the second factorization identify \eqref{dtnthH2}. Now we prove the Parseval properties \eqref{dtPar1dc}. Indeed, by the hypothesis $h\in L_1(\mathbb{R})\cap L_2(\mathbb{R})$ then we have $H_2[(H_2h)(y)](x)=h(x)$. By the help of Fubini's theorem and using trigonometric transforms, we write \begin{align*} &H_1[(H_1f)(y)\cdot(F_cg)(y)\cdot(H_2h)(y)](x) \\ &= \frac 1{\sqrt{2\pi}}\int_{-\infty}^{\infty}(H_1f)(y) \cdot (F_cg)(y)\cdot (H_2h)(y) \operatorname{cas}(xy)\, dy\\ &= \frac1{\pi\sqrt{2\pi}}\int_{-\infty}^{\infty}\int_{-\infty}^{\infty} \int_0^{\infty}f(u)g(v)(H_2h)(y)\operatorname{cas}(uy)\cos(vy)\operatorname{cas}(xy)\,dv\,du\,dy\\ &= \frac1{4\pi\sqrt{2\pi}}\int_{-\infty}^{\infty}\int_0^{\infty} \int_{-\infty}^{\infty}f(u)g(v)(H_2h)(y)\Big\{\operatorname{cas}[(x-u-v)y]\\ &\quad +\operatorname{cas}[(-x+u+v)y]-\operatorname{cas}[(-x-u-v)y]+\operatorname{cas}[(x+u+v)y]\\ &\quad +\operatorname{cas}[(x-u+v)y] +\operatorname{cas}[(-x+u-v)y]-\operatorname{cas}[(-x-u+v)y]\\ &\quad +\operatorname{cas}[(x+u-v)y]\Big\}\,dy\,dv\,du\\ &\quad = \frac 1{4\pi}\int_{-\infty}^{\infty}\int_0^{\infty}f(u)g(v) \Big[H_2(H_2h)(-x+u+v)+H_2(H_2h)(x-u-v) \\ &\quad -H_2(H_2h)(x+u+v)+H_2(H_2h)(-x-u-v)+H_2(H_2h)(-x+u-v) \\ &\quad +H_2(H_2h)(x-u+v)-H_2(H_2h)(x+u-v)+H_2(H_2h)(-x-u+v)\Big]\,dv\,du\\ &= \frac 1{4\pi}\int_{-\infty}^{\infty}\int_0^{\infty}f(u)g(v) \Big[h(-x+u+v)+h(x-u-v)-h(x+u+v)\\ &\quad +h(-x-u-v) +h(-x+u-v)+h(x-u+v)-h(x+u-v)\\ &\quad +h(-x-u+v)\Big]\,dv\, du\\ &= \frac 1{4\pi}\int_{-\infty}^{\infty}f(u)[k_1(x-u)+k_2(x+u)]du \\ &=[*(f,g,h)](x),\quad \forall x\in \mathbb{R}. \end{align*} It implies the first equality of \eqref{dtPar1dc}, the other follows from a similar process. The proof is complete. \end{proof} Using the factorization equalities \eqref{dtnthH1}, \eqref{dtnthH2}, we easily obtain the following corollary. \begin{corollary}\label{dltcdc} Let $g, l\in L_1(\mathbb{R}_+)$ and $f, k, h \in L_1(\mathbb{R})$. Then the polyconvolution \eqref{poly} satisfies the following conditions: \begin{gather*} *(*(f,g,h),l,k) =*(*(f,l,h),g,k)=*(*(f,g,k),l,h)=*(*(f,l,k),g,h),\\ *(k,l,*(f,g,h)) = *(h,l,*(f,g,k)) = *(k,g,*(f,l,h))=*(h,g,*(f,l,k)) \end{gather*} \end{corollary} Next, we study the polyconvolution in the function space $L_s^{\alpha,\beta,\gamma}(\mathbb{R})$ and its norm estimation. \begin{theorem} \label{thm2} Let $f\in L_p(\mathbb{R}),\ g\in L_q(\mathbb{R}_+),\ h\in L_r(\mathbb{R})$, such that $p,q,r>1$ and $\frac1p+\frac1q+\frac1r=2$. Then the polyconvolution \eqref{poly} is bounded in $L_s^{\alpha,\beta,\gamma}(\mathbb{R})$, where $s>1, \alpha>-1,\beta>0,\gamma>0$ and the following estimation holds. $$\|*(f,g,h)\|_{L_s^{\alpha,\beta,\gamma}(\mathbb{R})} \le C\|f\|_{L_p(\mathbb{R})} \|g\|_{L_q(\mathbb{R}_+)} \|h\|_{L_r(\mathbb{R})} \label{dgcLp},$$ where $C=\frac{2^{1+\frac1s}}{\pi\gamma^{\frac1s}} \beta^{-\frac{\alpha+1}{\gamma.s}} \Gamma^{\frac1s}\big(\frac{\alpha+1}{\gamma}\big)$ If, in addition, $f\in L_1(\mathbb{R})\cap L_p(\mathbb{R}),\ g\in L_1(\mathbb{R}_+)\cap L_q(\mathbb{R}_+)$ and $h\in L_1(\mathbb{R})\cap L_r(\mathbb{R})$, then the polyconvolution \eqref{poly} satisfies the factorization equalities \eqref{dtnthH1}, \eqref{dtnthH2} and belongs to $C_0(\mathbb{R})$. Moreover, if giving more condition on $h$, namely $h\in L_2(\mathbb{R})\cap L_1(\mathbb{R})\cap L_r(\mathbb{R})$, the Parseval identities \eqref{dtPar1dc} hold. \end{theorem} \begin{proof} Firstly, we prove $|*(f,g,h)(x)|\le \frac2{\pi}\|f\|_{L_p(\mathbb{R})}.\|g\|_{L_q(\mathbb{R}_+)}\|h\|_{L_r(\mathbb{R})}$. Indeed, from Definition \ref{dndc} and \eqref{k1}-\eqref{k2}, we have the estimate \begin{align} & |*(f,g,h)(x)| \nonumber \\ &= \frac 1{4\pi}\int_{-\infty}^{\infty}|f(u)|.[|k_1(x-u)|+|k_2(x+u)|]du \nonumber\\ &= \frac{1}{4\pi}\int_{-\infty}^{\infty}\int_0^{\infty}|f(u)||g(v)| \Big\{|h(-x+u+v)|+|h(x-u-v)| \nonumber\\ &\quad + |h(x+u+v)|+|h(-x-u-v)|+|h(-x+u-v)|+|h(x-u+v)| \nonumber\\ &\quad +|h(x+u-v)|+|h(-x-u+v)|\Big\}\,dv\,du. \label{dgmodula} \end{align} Separating the right-hand side of this expression into the sum of 8 integrals and denoting them by $I_k, k = 1,\dots,8$, respectively, without loss of generality, we have $$I_1(x)= \frac 1{4\pi}\int_{-\infty}^{\infty}\int_0^{\infty}|f(u)||g(v)||h(-x+u+v)| \,dv\,du,\quad x\in\mathbb{R}.$$ Let $p_1,q_1,r_1$ be the conjugate exponentials of $p,q,r$ and \begin{gather*} U(u,v)= |g(v)|^{q/p_1}|h(-x+u+v)|^{r/p_1}\in L_{p_1}(\mathbb{R}\times \mathbb{R}_+)\\ V(u,v)= |h(-x+u+v)|^{\frac r{q_1}}|f(u)|^{\frac p{q_1}}\in L_{q_1}(\mathbb{R}\times \mathbb{R}_+)\\ W(u,v)= |f(u)|^{p/r_1}|g(v)|^{q/r_1}\in L_{r_1}(\mathbb{R}\times \mathbb{R}_+). \end{gather*} We have $UVW= |f(u)||g(v)||h(-x+u+v)|.$ Using the definition of the norm on space $L_{p_1}(\mathbb{R}\times \mathbb{R}_+)$ and the help of Fubini's Theorem, we write \begin{align*} \|U\|_{L_{p_1}(\mathbb{R}\times \mathbb{R}_+)}^{p_1} &= \int_{-\infty}^{\infty}\int_0^{\infty} \big\{|g(v)|^{q/p_1}|h(-x+u+v)|^{r/p_1}\big\}^{p_1}\,dv\,du\\ &= \int_0^{\infty}|g(v)|^{q}\Big(\int_{-\infty}^{\infty}|h(-x+u+v)|^{r}\,du \Big)dv\\ &= \int_0^{\infty}|g(v)|^{q}\|h\|_{L_r(\mathbb{R})}^rdv\\ &=\|g\|_{L_q(\mathbb{R}_+)}^q \|h\|_{L_r(\mathbb{R})}^r. \end{align*} Similarly, we obtain $$\label{dgcVW} \|V\|_{L_{q_1}(\mathbb{R}\times \mathbb{R}_+)}^{q_1} = \|f\|_{L_p(\mathbb{R})}^p.\|h\|_{L_r(\mathbb{R})}^r;\quad \|W\|_{L_{r_1}(\mathbb{R}\times \mathbb{R}_+)}^{r_1}= \|f\|_{L_p(\mathbb{R})}^p \|g\|_{L_q(\mathbb{R}_+)}^q.$$ From the hypothesis $\frac1p+\frac1q+\frac1r=2$, it follows $\frac1{p_1}+\frac1{q_1}+\frac1{r_1}=1$. Using the H\"older inequality and \eqref{dgcVW}, we have estimate \begin{aligned} I_1&= \frac 1{4\pi}\int_{-\infty}^{\infty}\int_0^{\infty}UVW\,dv\,du\\ &\le \frac 1{4\pi}\Big(\int_{-\infty}^{\infty} \int_0^{\infty}U^{p_1}\,du\,dv\Big)^{1/p_1} \Big(\int_{-\infty}^{\infty}\int_0^{\infty}V^{q_1}dudv\Big)^{1/p_1}\\ &\quad\times \Big(\int_{-\infty}^{\infty}\int_0^{\infty}W^{r_1}dudv\Big)^{1/r_1}\\ &= \frac 1{4\pi}\|U\|_{L_{p_1}(\mathbb{R}\times \mathbb{R}_+)}\|V\|_{L_{q_1}(\mathbb{R}\times \mathbb{R}_+)} \|W\|_{L_{r_1}(\mathbb{R}\times \mathbb{R}_+)}\\ &= \frac 1{4\pi}\Big(\|g\|_{L_q(\mathbb{R}_+)}^{\frac{q}{p_1}} \|h\|_{L_r(\mathbb{R})}^{\frac{r}{p_1}}\Big)\Big(\|f\|_{L_p(\mathbb{R})}^{\frac{p}{q_1}} \|h\|_{L_r(\mathbb{R})}^{\frac{r}{q_1}}\Big) \Big(\|f\|_{L_p(\mathbb{R})}^{\frac{p}{r_1}}\|g\|_{L_q(\mathbb{R}_+)}^{\frac{q}{r_1}}\Big)\\ &= \frac 1{4\pi}\|f\|_{L_p(\mathbb{R})}\|g\|_{L_q(\mathbb{R}_+)}\|h\|_{L_r(\mathbb{R})} \end{aligned}\label{dgI1} The same way, we obtain the estimates for $I_k$, $k=2,3\dots,8$: $$I_k\le \frac 1{4\pi}\|f\|_{L_p(\mathbb{R})}\|g\|_{L_q(\mathbb{R}_+)}\|h\|_{L_r(\mathbb{R})},\quad \text{for } k=2,3,\dots,8 \label{dgI2}$$ From \eqref{dgmodula}--\eqref{dgI2}, it follows that $$\label{absdc} |*(f,g,h)(x)|\le \frac 2{\pi}\|f\|_{L_p(\mathbb{R})}\|g\|_{L_q(\mathbb{R}_+)}\|h\|_{L_r(\mathbb{R})}.$$ Now, using the \cite[formula 3.381.10]{GradRyz2007}, we have $$\label{f3.381.10} {\int_{-\infty}^{\infty}|x|^{\alpha}e^{-\beta |x|^{\gamma}}dx} =\frac2{\gamma} \beta^{-\frac{\alpha+1}\gamma}\Gamma \big(\frac{\alpha+1}\gamma\big).$$ From \eqref{absdc} and \eqref{f3.381.10}, we have \begin{align*} \|*(f,g,h)\|_{L_s^{\alpha,\beta,\gamma}(\mathbb{R})}^s &= \int_{-\infty}^{\infty}{|x|^{\alpha}e^{-\beta |x|^{\gamma}}} |*(f,g,h)(x)|^sdx\\ &\le \int_{-\infty}^{\infty}{|x|^{\alpha}e^{-\beta |x|^{\gamma}}} \big(\frac2{\pi}\big)^{s}\|f\|_{L_p(\mathbb{R})}^s \|g\|_{L_q(\mathbb{R}_+)}^s \|h\|_{L_r(\mathbb{R})}^s dx\\ &= C^s\|f\|_{L_p(\mathbb{R})}^s\|g\|_{L_q(\mathbb{R}_+)}^s\|h\|_{L_r(\mathbb{R})}^s. \end{align*} where $C = \frac{2^{1+\frac1s}}{\pi\gamma^{\frac1s}} \beta^{-\frac{\alpha+1}{\gamma.s}} \Gamma^{\frac1s} \big(\frac{\alpha+1}{\gamma}\big),$ which gives \eqref{dgcLp}. Since $f\in L_1(\mathbb{R})\cap L_p(\mathbb{R}),\ g\in L_1(\mathbb{R}_+)\cap L_q(\mathbb{R}_+)$ and $h\in L_1(\mathbb{R})\cap L_r(\mathbb{R})$, three functions $f,g$ and $h$ satisfy the hypothesis of Theorem \ref{dlnth}, it implies that $*(f,g,h)\in C_0(\mathbb{R})\cap L_1(\mathbb{R})$, then the factorization identities \eqref{dtnthH1}, \eqref{dtnthH2} hold. Moreover, if $h\in L_2(\mathbb{R})\cap L_1(\mathbb{R})\cap L_r(\mathbb{R})$, it also satisfies the hypothesis of Theorem \ref{dlnth} to get Parseval equalities \eqref{dtPar1dc}. The proof is complete. \end{proof} Next, we have a Titchmarch type theorem. \begin{theorem} \label{titchmarch} Let {$f,h\in L_1(\mathbb{R}, e^{|x|})$} and $g\in L_1(\mathbb{R}_+, e^x)$. If $*(f,g,h)(x)= 0,\ \forall x\in \mathbb{R}$, then either $f(x)=0, \forall x\in \mathbb{R}$, or $g(x)=0, \forall x\in \mathbb{R}_+$ or $h(x)=0,\forall x\in \mathbb{R}$. \end{theorem} \begin{proof} The hypothesis $*(f,g,h)(x)= 0$, for all $x\in \mathbb{R}$ implies $H_1(*(f,g,h))(y) =0$, for all $y\in \mathbb{R}$. Due to factorization equality \eqref{dtnthH1} we have $$(H_1f)(y) (F_cg)(y) (H_2h)(y) = 0,\quad \forall y\in \mathbb{R}.\label{nth}$$ Now we show that $(H_1f)(y)$, $(F_cg)(y)$, $(H_2h)(y)$ are real analytic. Without loss of generality, we prove that $(H_1f)(y)$ can be expanded into convergent Taylor series in $\mathbb{R}$. Indeed, by using the Lebesgue Dominated Convergence Theorem, we can exchange the orders of integration and differentiation, we have \begin{align*} \big|\frac{d^n}{dy^n}(H_1f)(y)\big| &\le \frac1{\sqrt{2\pi}}\int_{-\infty}^{\infty} \big|\frac{d^n}{dy^n}[f(x)\operatorname{cas}(xy))]\big|dx\\ &=\frac1{\sqrt{2\pi}}\int_{-\infty}^{\infty} \big|f(x)x^n\big[\cos\big(xy+n\frac{\pi}2)\big) +\sin\big(xy+n\frac{\pi}2)\big)\big]\big|dx\\ &\le \frac1{\sqrt{2\pi}}\int_{-\infty}^{\infty} |2f(x)x^n|dx \\ &\le \sqrt{\frac 2{\pi}}\int_{-\infty}^{\infty}e^{-|x|}\frac{|x|^n}{n!}n!|f(x)| e^{|x|}dx\\ &\le \sqrt{\frac 2{\pi}}\int_{-\infty}^{\infty}n!|f(x)|e^{|x|}dx =\sqrt{\frac 2{\pi}}n!\|f\|_{L_1(\mathbb{R},e^{|x|})}=n!M, \end{align*} here, $M=\sqrt{\frac 2{\pi}}\|f\|_{L_1(\mathbb{R},e^{|x|})}$. Thus, the remainder of Taylor expansion for $(H_1f)(y)$ at neighbourhood of an arbitrary $y_0\in \mathbb{R}$ is $$\big|\frac 1{n!}\frac{d^n(H_1f)(c)}{dy^n}(y-y_0)^n\big| \le \frac 1{n!}n!M|y-y_0|^n=M|y-y_0|^n,$$ From analytic property and \eqref{nth}, it implies $$(H_1f)(y)=0,\forall y\in \mathbb{R},\quad \text{or}\quad (F_cg)(y)=0,\forall y\in \mathbb{R}_+,\quad \text{or}\quad (H_2h)(y)=0,\forall y\in \mathbb{R}.$$ Since the uniqueness of Hartley and Fourier cosine transforms in $L_1(\mathbb{R})$, then it follows $$f(x)=0,\; \forall y\in \mathbb{R},\quad \text{or}\quad g(x)=0,\; \forall x\in \mathbb{R}_+,\quad \text{or}\quad \ h(x)=0,\;\forall x\in \mathbb{R}.$$ The proof is complete. \end{proof} \section{Applications} \subsection{Integral equations} In this subsection we apply the obtained result in solving the modified equation \eqref{ptTHm} of the Toeplitz plus Hankel integral equation \eqref{ptTH}. \begin{theorem}\label{dlpttp} Let $g\in L_1(\mathbb{R}_+)$ and $h,p\in L_1(\mathbb{R})$ be given functions, $\lambda$ is given constant. The sufficient and necessary condition for the integral equation \eqref{ptTHm} to have a unique solution in space $L_1(\mathbb{R})$ is \begin{align} 1+\lambda (F_cg)(y)(H_2h)(y)\ne 0,\quad \forall y\in \mathbb{R},\label{dkpt} \end{align} and the solution has the form $$f(x) = p(x)-(l\underset{HF}\ast p)(x),\quad \forall x\in \mathbb{R},$$ where $l\in L_1(\mathbb{R})$ is defined by $$(Fl)(y) = \frac{\lambda (F_cg)(y)(H_2h)(y)}{1+\lambda(F_cg)(y)(H_2h)(y)},\quad \forall y\in \mathbb{R}.\label{Wforl}$$ \end{theorem} \begin{proof} Using Definition \ref{dndc}, the equation \eqref{ptTHm} can be rewritten in the form $$f(x)+\lambda[*(f,g,h)](x)=p(x).$$ Applying the Hartley transform $H_1$ on both sides of the equation, using the factorization property \eqref{dtnthH1} and \eqref{conHFc}, we obtain $$\begin{gathered} (H_1f)(y)+\lambda (H_1f(y)\cdot (F_cg)(y)\cdot(H_2h)(y)=(H_1p)(y)\\ (H_1f)(y)[1+\lambda(F_cg)(y)\cdot(H_2h)(y)]=(H_1p)(y). \end{gathered}\label{H1fofpttp}$$ Due to \eqref{dkpt}, the equation \eqref{H1fofpttp} has a unique solution \begin{aligned} (H_1f)(y) &= (H_1p)(y)\frac1{[1+\lambda (F_cg)(y)\cdot (H_2h)(y)]}\\ &= (H_1p)(y)\Big[1-\frac{\lambda(F_cg)(y)\cdot n(H_2h)(y)} {1+\lambda(F_cg)(y)\cdot(H_2h)(y)}\Big]. \end{aligned} \label{BWienerlevypt} By the Wiener-Levy theorem \cite{PaleyWiener1934}, if $q$ is the Fourier transform of a some function in $L_1(\mathbb{R})$, $\varphi(z)$ is analytic, $\varphi(0)=0$ and defined at area of $q$ values, then $\varphi(q)$ also is a Fourier transform of a some function in $L_1(\mathbb{R})$. Note that we can write $(F_cg)(y)\cdot(H_2h)(y)=H_2\big(g\underset{HF_c}{*} h\big)(y)$ and we have the relationship between Hartley transform and Fourier transform $(H_2q)(y)= \frac{1+i}2(Fq)(-y)+\frac{1-i}2(Fq)(y), \quad \forall y \in \mathbb{R},$ then the expression $\frac{\lambda(F_cg)(y).(H_2h)(y)}{1+\lambda(F_cg)(y).(H_2h)(y)}$ defines the Fourier transform of a some function $l$ in $L_1(\mathbb{R})$. It means that, there exists a function $l\in L_1(\mathbb{R})$ such that $$(Fl)(y) = \frac{\lambda (F_cg)(y).(H_2h)(y)}{1+\lambda(F_cg)(y).(H_2h)(y)}. \label{Wienerlevypt}$$ So, from \eqref{BWienerlevypt}, \eqref{Wienerlevypt} and using the factorization equality \eqref{conHF}, we obtain \begin{align*} (H_1f)(y)&= (H_1p)(y)[1-(Fl)(y)] =(H_1p)(y)-(Fl)(y)(H_1p)(y)\\ &=(H_1p)(y)-H_1(l\underset{HF}\ast p)(y) =H_1[p-(l\underset{HF}\ast p)](y),\quad \forall y\in \mathbb{R}. \end{align*} It follows that $f(x) = p(x)-(l\underset{HF}\ast p)(x)\in L_1(\mathbb{R})$. The proof is complete. \end{proof} We see that the condition \eqref{dkpt} is still true for $g,h\in L_1(\mathbb{R})$. However, determining $l(x)$ from \eqref{Wforl} depends on $g,h$. Below, we will show an example to find $l(x)$ with given functions $g,h$. \begin{example} \label{examp1} \rm Choose {$g(x)=\sqrt{\frac 2\pi}K_0(x)$ and $h(x)=\sqrt{\frac 2\pi}K_0(|x|)$}, where $K_0(x)$ is Bessel function. By property of Bessel $K_0(x)$ (see the \cite[formula 6.511.12]{GradRyz2007}), we see that $\int_0^{\infty}|K_0(x)|dx = \pi/2$. This implies $K_0(x)$ is a function in $L_1(\mathbb{R}_+)$. Thus, $g(x)\in L_1(\mathbb{R}_+)$, $h(x)\in L_1(\mathbb{R})$ and from the \cite[formula 1.2.17]{Bracewell1986} (or \cite[formula 3.754.2]{GradRyz2007}) we have $$(F_cg)(y)=(H_2h)(y)=\frac 1{\sqrt{1+y^2}}.$$ Next, we choose {$\lambda = 1$}, then $$(Fl)(y)=\frac{(F_cg)(y)(H_2h)(y)}{1+(F_cg)(y)(H_2h)(y)} =\frac 1{y^2+2}\in L_1(\mathbb{R}).$$ Based on \cite[formula 17.23.14]{GradRyz2007}, we have $$l(x)=F^{-1}\big(\frac 1{y^2+2}\big)(x) =\sqrt \pi \frac{e^{-\sqrt 2 |x|}}{2}\in L_1(\mathbb{R}).$$ Now, choosing $p(x)=e^{-x^2}\in L_1(\mathbb{R})$, and using \eqref{HF_equa}, we have \begin{align*} (l\underset{HF}\ast p)(x) &= \frac1{2\sqrt{2\pi}}\int_{-\infty}^{\infty}l(u)[p(x-u)+p(x+u) +ip(-x-u)-ip(x+u)]du\\ &=\frac1{4\sqrt{2}}\int_{-\infty}^{\infty}{e^{-\sqrt 2 |u|}} \big[e^{-(x-u)^2}+e^{-(x+u)^2}\big]du\\ &=\frac{1}{4\sqrt{2}}.e^{\frac 12-\sqrt 2 x}\sqrt\pi \Big(\operatorname{Erfc}\big[\frac 1{\sqrt 2}-x\big]+e^{2\sqrt 2 x} \operatorname{Erfc}\big[\frac 1{\sqrt 2}+x\big]\Big). \end{align*} where $\operatorname{Erfc}(x)$ is the complementary error function. By using the Mathematica software program, we have $$\int_{-\infty}^{\infty}\Big|\frac{1}{4\sqrt{2}} e^{\frac 12-\sqrt 2 x}\sqrt\pi\Big(\operatorname{Erfc}[\frac 1{\sqrt 2}-x] +e^{2\sqrt 2 x}\operatorname{Erfc}[\frac 1{\sqrt 2}+x]\Big)\Big|dx =\frac{\pi}2.$$ This implies $(l\underset{HF}\ast p)(x)\in L_1(\mathbb{R})$. So, the solution of the equation \eqref{ptTHm} is \begin{align*} f(x)&= p(x)-(l\underset{HF}\ast p)(x)\\ &=e^{-x^2}-\frac{1}{4\sqrt{2}}e^{\frac 12-\sqrt 2 x}\sqrt\pi \Big(\operatorname{Erfc}[\frac 1{\sqrt 2}-x]+e^{2\sqrt 2 x}\operatorname{Erfc}[\frac 1{\sqrt 2}+x]\Big) \in L_1(\mathbb{R}). \end{align*} \end{example} \subsection{System of two integral equations} Consider a system of two Toeplitz plus Hankel integral equations of the form: $$\label{hpttp} \begin{gathered} f(x)+\lambda_1 \int_{-\infty}^{\infty}g(u)[k_1(x-u)+k_2(x+u)]du= p(x)\\ \lambda_2\int_{-\infty}^{\infty}f(u)[k_3(x-u)+k_4(x+u)]du+g(x) =q(x),\quad \forall x\in \mathbb{R}, \end{gathered}$$ where, for $t\in\mathbb{R}$, \begin{gather*} k_1(t):=\int_0^{\infty}\varphi_1(v)[\psi_1(-t+v)+\psi_1(t-v) +\psi_1(-t-v)+\psi_1(t+v)]dv,\\ k_2(t):=\int_0^{\infty}\varphi_1(v)(v)[-\psi_1(t+v)+\psi_1(-t-v)-\psi_1(t-v) +\psi_1(-t+v)]dv,\\ k_3(t):=\int_0^{\infty}\varphi_2(v)[\psi_2(-t+v)+\psi_2(t-v)+\psi_2(-t-v) +\psi_2(t+v)]dv,\\ k_4(t):=\int_0^{\infty}\varphi_2(v)(v)[-\psi_2(t+v) +\psi_2(-t-v)-\psi_2(t-v)+\psi_2(-t+v)]dv, \end{gather*} and $\lambda_1, \lambda_2$ are complex constants; $\varphi_1,\varphi_2$ are functions in $L_1(\mathbb{R}_+)$; $\psi_1,\psi_2, p(x), q(x)$ are functions in $L_1(\mathbb{R})$; and $f,g$ are unknown functions. \begin{theorem}\label{dlhpttp} If the condition $$1-\lambda_1\lambda_2(F_c\varphi_1)(y)(F_c\varphi_2)(y)(H_2\psi_1)(y)(H_2\psi_2)(y) \ne 0,\quad \forall y\in \mathbb{R}.$$ holds, then there exists a unique solution in $L_1(\mathbb{R})\times L_1(\mathbb{R})$ of system \eqref{hpttp} defined by \begin{gather*} f(x) = p(x)-\lambda_1(*(q,\varphi_1,\psi_1))(x) +\big\{l\underset{HF}*[p-\lambda_1(*(q,\varphi_1,\psi_1))]\big\}(x),\\ g(x) = q(x)-\lambda_2(*(p,\varphi_2,\psi_2))(x) +\big\{l\underset{HF}*[q-\lambda_2(*(p,\varphi_2,\psi_2))]\big\}(x), \end{gather*} here $l\in L_1(\mathbb{R})$ is given by $$(Fl)(y)=\frac{\lambda_1\lambda_2(F_c\varphi_1)(y)(F_c\varphi_2)(y)(H_2\psi_1)(y) (H_2\psi_2)(y)}{1-\lambda_1\lambda_2(F_c\varphi_1)(y)(F_c\varphi_2)(y) (H_2\psi_1)(y)(H_2\psi_2)(y)},\quad \forall y\in \mathbb{R}.$$ \end{theorem} \begin{proof} Using Definition \ref{dndc}, the system of equations \eqref{hpttp} can be rewritten in the form $$\label{hpttpdc} \begin{gathered} f(x)+\lambda_1[*(g,\varphi_1,\psi_1)](x)=p(x),\\ \lambda_2[*(f,\varphi_2,\psi_2)](x)+g(x)=q(x),\quad x\in \mathbb{R}. \end{gathered}$$ Due to the factorization property of the polyconvolution \eqref{dtnthH1}, we obtain the linear system of algebraic equations with respect to $(H_1f)(y)$ and $(H_1g)(y)$ $$\label{hpttpdc1} \begin{gathered} (H_1f)(y)+\lambda_1(H_1g)(y)(F_c\varphi_1)(y)(H_2\psi_1)(y)=(H_1p)(y),\\ \lambda_2(H_1f)(y)(F_c\varphi_2)(y)(H_2\psi_2)(y)+(H_1g)(y)=(H_1q)(y),\quad \forall y\in \mathbb{R}. \end{gathered}$$ Let $\Delta$ be the determinant of the system, \begin{align*} \Delta &= \begin{vmatrix} 1&\lambda_1(F_c\varphi_1)(y)(H_2\psi_1)(y)\\ \lambda_2(F_c\varphi_2)(y)(H_2\psi_2)(y)&1 \end{vmatrix}\\ &=1-\lambda_1\lambda_2(F_c\varphi_1)(y)(F_c\varphi_2)(y)(H_2\psi_1)(y)(H_2\psi_2)(y). \end{align*} Due to the hypothesis, $\Delta \ne 0$, the system \eqref{hpttpdc} has a unique solution. By using \eqref{conHFc} and \eqref{conH}, we present $1/\Delta$ as below \begin{align*} \frac1{\Delta} & = \frac 1{1-\lambda_1\lambda_2(F_c\varphi_1)(y)(F_c\varphi_2)(y) (H_2\psi_1)(y)(H_2\psi_2)(y)}\\ & = 1+\frac {\lambda_1\lambda_2(F_c\varphi_1)(y)(F_c\varphi_2)(y) (H_2\psi_1)(y)(H_2\psi_2)(y)}{1-\lambda_1\lambda_2(F_c\varphi_1) (y)(F_c\varphi_2)(y)(H_2\psi_1)(y)(H_2\psi_2)(y)}\\ & = 1+\frac{\lambda_1\lambda_2H_2[(\varphi_1\underset{HF_c}\ast \psi_1) \underset{H}\ast (\varphi_2\underset{HF_c}\ast \psi_2)](y)}{1-\lambda_1\lambda_2H_2[(\varphi_1\underset{HF_c}\ast \psi_1) \underset{H}\ast (\varphi_2\underset{HF_c}\ast \psi_2)](y)}. \end{align*} Furthermore, according to Wiener-Levy theorem \cite{PaleyWiener1934} and the relationship between Hartley transform and Fourier transform, it exists a function $l\in L_1(\mathbb{R})$ such that $$(Fl)(y)=\frac{\lambda_1\lambda_2H_2[(\varphi_1\underset{HF_c}\ast \psi_1) \underset{H}\ast (\varphi_2\underset{HF_c}\ast \psi_2)](y)}{1-\lambda_1\lambda_2H_2 [(\varphi_1\underset{HF_c}\ast \psi_1)\underset{H}\ast (\varphi_2\underset{HF_c} \ast \psi_2)](y)},\ \forall y\in \mathbb{R}.$$ So, we can write $\frac 1\Delta = 1+(Fl)(y)$. To find the solution of the system \eqref{hpttpdc}, we need to determine the two following determinants \begin{align*} \Delta_1 &= \begin{vmatrix} (H_1p)(y)&\lambda_1(F_c\varphi_1)(y)(H_2\psi_1)(y)\\ (H_1q)(y)&1\end{vmatrix} \\ &=(H_1p)(y)-\lambda_1H_1[*(q,\varphi_1,\psi_1)](y) = H_1[p-\lambda_1(*(q,\varphi_1,\psi_1))](y),\quad y\in \mathbb{R}. \end{align*} Based on \eqref{conHF}, we have \begin{align*} (H_1f)(y) &= \frac{\Delta_1}{\Delta}=H_1[p-\lambda_1(*(q,\varphi_1,\psi_1))](y)[1+(Fl)(y)]\\ &= H_1[p-\lambda_1(*(q,\varphi_1,\psi_1))](y) + H_1\{l\underset{HF}*[p-\lambda_1(*(q,\varphi_1,\psi_1))]\}(y)\\ &= H_1\{p-\lambda_1(*(q,\varphi_1,\psi_1))+l\underset{HF}* [p-\lambda_1(*(q,\varphi_1,\psi_1))]\}(y),\quad \forall y\in \mathbb{R}. \end{align*} It follows that $$f(x) = p(x)-\lambda_1(*(q,\varphi_1,\psi_1))(x) +\{l\underset{HF}*[p-\lambda_1(*(q,\varphi_1,\psi_1))]\}(x)\in L_1(\mathbb{R}).$$ Similarly, we compute the second component determinant of system \eqref{hpttpdc}, \begin{align*} \Delta_2 &= \begin{vmatrix} 1& (H_1p)(y)\\ \lambda_2(F_c\varphi_2)(y)(H_2\psi_2)(y)&(H_1q)(y) \end{vmatrix} \\ &=(H_1q)(y)-\lambda_2H_1[*(p,\varphi_2,\psi_2)](y)\\ &=H_1[q-\lambda_2(*(p,\varphi_2,\psi_2))](y),\quad y\in \mathbb{R}. \end{align*} Based on \eqref{conHF} we obtain \begin{align*} (H_1g)(y) &= \frac{\Delta_2}{\Delta} =H_1[q-\lambda_2(*(p,\varphi_2,\psi_2))](y).[1+(Fl)(y)]\\ &= H_1[q-\lambda_2(*(p,\varphi_2,\psi_2))](y) + H_1\{l\underset{HF}*[q-\lambda_2(*(p,\varphi_2,\psi_2))]\}(y)\\ &= H_1\big\{q-\lambda_2(*(p,\varphi_2,\psi_2))+l\underset{HF}* [q-\lambda_2(*(p,\varphi_2,\psi_2))]\big\}(y),\quad \forall y\in \mathbb{R}. \end{align*} It follows that $$g(x) = q(x)-\lambda_2(*(p,\varphi_2,\psi_2))(x) +\{l\underset{HF}*[q-\lambda_2(*(p,\varphi_2,\psi_2))]\}(x)\in L_1(\mathbb{R}).$$ The proof is complete. \end{proof} \subsection*{Acknowledgements} This research is funded by Vietnam's National Foundation for Science and Technology Development (NAFOSTED) under grant number 101.01-2011.05. The authors thank the anonymous referees for their value comments that improve this article. \begin{thebibliography}{99} \bibitem{AnhTuanTuan2013} Anh, P. K.; Tuan, N. M.; Tuan, P. D.; (2013) The Finite Hartley new convolution and solvability of the integral equations with Toeplitz plus Hankel kernels, \emph{Journal of Mathematical Analysis and Applications.} \textbf{397}(2): 537-549. \bibitem{BogveradzeCastro2008} Bogveradze, G.; Castro L. P.; (2008). Toeplitz plus Hankel operators with infinite index, \emph{Integral Equations and Operator Theory} \textbf{62}(1): 43-63. \bibitem{Bracewell1986} Bracewell, R. N.; (1986) \emph{The Hartley transform}, Oxford University Press, Inc. \bibitem{ChadanSabatier} Chadan, K.; Sabatier, P. C.; (1989) \emph{Inverse Proplems in Quantum Scattering Theory}, Springer Verlag. \bibitem{GiangMauTuan2009} Giang, B. T.; Mau N. V.; Tuan, N. M.; (2009) Operational properties of two itegral transforms of Fourier type and their convolutions, \emph{Integral Equations and Operator Theory,} \textbf{65}(3): 363-386. \bibitem{GradRyz2007} Gradshteyn, I. S.; Ryzhik, I. M.; (2007) \emph{Table of Integrals}, Series and Products 7th edn, ed A Jeffrey and D. Zwillinger (New York: Academic). \bibitem{Kailath1966} Kailath, T.; (1966) Some integral equations with 'nonrational' kernels, \emph{IEEE Transactions on Information Theory} \textbf{12}(4): 442-447. \bibitem{KagiwadaKalaba} Kagiwada, H. H.; Kalaba, R.; (1974) \emph{Integral equations via imbedding methods}, Addison-Wesley. \bibitem{kakichev1997} Kakichev, V. A.; (1997) Polyconvolution, \emph{Taganrog, Taganskij Radio-Tekhnicheskij Universitet}, 54p (in Russian). \bibitem{PaleyWiener1934} Paley, R. E. A. C.; Wiener, N.; (1934) \emph{Fourier transforms in the complex domain}, American Mathematical Society \textbf{19}. \bibitem{Sneddon1972} Sneddon, I. N.; (1972) \emph{The Use of Integral Transforms}, McGraw-Hill. \bibitem{ThaoTuanHong2008} Thao, N. X.; Tuan, V. K.; Hong, N. T. (2008); Generalized convolution transforms and Toeplitz plus Hankel integral equations, \emph{Fractional Calculus and Applied Analysis} \textbf{11}(2): 153-174. \bibitem{ThaoTuanHong2011} Thao, N. X.; Tuan, V. K.; Hong, N. T. (2011). Toeplitz plus Hankel integral equation, \emph{Integral Transforms and Special Functions} {\bf22}(10): 723-737. \bibitem{ThaoKhoaPAnh2013} Thao, N. X.; Khoa, N. M.; Van Anh, P. T. (2013); On the polyconvolution for Hartley, Fourier cosine and Fourier sine transforms, \emph{Integral Transforms and Special Functions} {\bf24}(7): 517-531. \bibitem{ThaoTuanHAnh2013} Thao, N. X.; Tuan, V. K.; Van Anh, H. T.; (2013) On the Toeplitz plus Hankel integral equation II, \emph{Integral Transforms and Special Functions}, DOI: 10.1080/10652469.2013.815185. \bibitem{ThaoHAnh2013} Thao, N. X.; Van Anh, H. T.; (2013) On the Hartley-Fourier sine generalized convolution, \emph{Mathematical Methods in the Applied Sciences}, DOI: 10.1002/mma.2980. \bibitem{Titch1986} Titchmarsh, E. C.; (1986) \emph{Introduction to the Theory of Fourier Integrals}, NewYork \textbf{69}. \bibitem{TsiLevy1981} Tsitsiklis, J. N., Levy, B. C. (1981); \emph{Integral equations and resovents of Toeplitz plus Hankel kernels}, Laboratory for Information and Decision Systems, Massachusetts Institute of Technology, Series/Report No.: LIDS-P 1170. \end{thebibliography} \end{document}