%2345678901234567890123456789012345678901234567890123456789012345678901234567890
%        1         2         3         4         5         6         7         8
% if you need a4paper
%\documentclass[a4paper, 10pt, conference]{ieeeconf} 
% Use this line for a4
% paper
% This command is only
% needed if you want to
% use the \thanks command
% See the \addtolength command later in the file to balance the column lengths
% on the last page of the document
% The following packages can be found on http:\\www.ctan.org
%\usepackage{graphics} % for pdf, bitmapped graphics files
%\usepackage{epsfig} % for postscript graphics files
%\usepackage{mathptmx} % assumes new font selection scheme installed
%\usepackage{times} % assumes new font selection scheme installed
%\usepackage{amsmath} % assumes amsmath package installed
%\usepackage{amssymb}  % assumes amsmath package installed
%\author{ \parbox{3 in}{\centering Huibert Kwakernaak*
%         \thanks{*Use the $\backslash$thanks command to put information here}\\
%         Faculty of Electrical Engineering, Mathematics and Computer Science\\
%         University of Twente\\
%         7500 AE Enschede, The Netherlands\\
%         {\tt\small h.kwakernaak@autsubmit.com}}
%         \hspace*{ 0.5 in}
%         \parbox{3 in}{ \centering Pradeep Misra**
%         \thanks{**The footnote marks may be inserted manually}\\
%        Department of Electrical Engineering \\
%         Wright State University\\
%         Dayton, OH 45435, USA\\
%         {\tt\small pmisra@cs.wright.edu}}
%}


\documentclass[letterpaper, 10 pt, conference]{ieeeconf}
\usepackage{amssymb}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%TCIDATA{Created=Thu Mar 08 10:32:58 2007}
%TCIDATA{LastRevised=Fri Sep 04 18:43:07 2009}

\include{tcilatex}     
\IEEEoverridecommandlockouts                              
\overrideIEEEmargins
\input{tcilatex}
\begin{document}

\title{{\LARGE Dealing with Stochastic Reachability\textbf{\ }}}
\author{Manuela L. Bujorianu\thanks{%
Manuela L. Bujorianu is with CICADA, \strut School of Mathematics,
University of Manchester, UK, $\mathrm{%
E-mail:Manuela.Bujorianu@manchester.ac.uk}$}}
\maketitle

\begin{abstract}
For stochastic hybrid systems, stochastic reachability is very little
supported mainly because of complexity and difficulty of the associated
mathematical problems. In this paper, we develop two main directions of
studying stochastic reachability as an optimal stopping problem. The first
approach studies the hypotheses for the dynamic programming corresponding
with the optimal stopping problem for stochastic hybrid systems. In the
second approach, we investigate the reachability problem considering
approximations of stochastic hybrid systems. The main difficulty arises when
we have to prove the convergence of the value functions of the approximating
processes to the value function of the initial process. An original proof is
provided.

Keywords: hybrid systems, reachability, Markov model, capacity.
\end{abstract}

\thispagestyle{empty} \pagestyle{empty}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Introduction}

Stochastic hybrid systems (SHS) are a class of non-linear stochastic
continuous time/space hybrid dynamical systems. For these systems different
models have been developed by many researchers in the field of hybrid
systems. These models can be used to analyse and design complex embedded
systems that operate in the presence of variability and uncertainty, and
incorporate complex (hybrid/stochastic) dynamics, randomness, multiple modes
of operations. Under some natural assumptions on their parameters, their
behaviour can be described by stochastic processes having good properties. A
very important verification problem for such systems consists mainly in
reachability analysis. The aim of reachability analysis is to determine the
probability that the system will reach a set of desirable/unsafe states, and
the difficulty of this problem comes from the interaction between
discrete/continuous dynamics and the active boundaries.

The paper addresses the reachability problem for SHS. Starting with the
characterization of the reachability problem as an optimal stopping problem
for the Markov processes that describe the semantics of SHS, we further
develop different foundational approaches that hopefully will conduct
finally to computational methods. The main difficulty comes from the fact
that these Markov processes belong to a relatively restrictive class of
stochastic processes. They have ``good'' mathematical properties (like
strong Markov property, some continuity properties of traces, etc). They are
included in the large class of Borel right (Markov) processes \cite{BL06}.
But, because of the influence of the boundaries on the dynamics, these
processes cover only a specific subclass of Borel processes, with a very
little intersection with other subclasses of processes more popular in the
theory of stochastic control (Feller-Markov processes \cite{Menaldi2002},
standard processes \cite{Shir76}, jump-diffusion processes \cite
{oksendal2005}). In consequence, for these processes, the optimal control
problems have been studied mostly at a theoretical level. This situation
compels us to study the optimal stopping problem for Borel right processes,
in a separate section of the paper, and to give an overview for the
different methods that can be employed in dealing with this problem.

For a stochastic hybrid system, the reach set probabilities coincide with
the value functions of some particular optimal stopping problems
corresponding to the indicator functions of the target sets \cite{BLL08}.
These optimal stopping problems are formulated in the language of the Markov
process that describes the realizations of the given hybrid system. We make
a guiding inventory of the possible methods that could be used for solving
the optimal stopping problems. Considering the complexity of the right
(Markov) processes that appear in the context of SHS, we further develop two
such methods. \noindent One approach is based on the characterization of the
value functions as viscosity solutions of some variational inequalities
associated to an integro-differential operator (that is represented by the
infinitesimal generator of the underlying Markov process). The main
drawbacks and difficulties when they are applied to the optimal stopping of
SHS come from the fact that we need some continuity assumptions for the
reward function or for the transition probabilities of the Markov processes
involved. But, for the reachability problem, the reward function is
discontinuous, and the continuity of the transition probabilities imply no
boundary activity. Another method is to approximate the SHS realization by
simpler Markovian processes (like purely jump processes or Markov chains)
and to derive convergence results for the sequences of value functions
associated to the optimal stopping problem corresponding to the
approximating processes. The corner stone of this approach is, in fact,
proving the convergence results under such general hypotheses (the reward
function is not continuous and the limiting process is only a Borel right
process). We use an original argument based on the correspondence between
reach set probabilities and the so-called Choquet capacities developed by us
in \cite{ManuCDC05}.

\section{Stochastic Hybrid Systems}

SHS can be described as an interleaving between a finite or countable family
of diffusion processes (or, sometimes, only dynamical systems) and a Markov
chain. Modelling and analysis of these systems have been proved to be a very
difficult task from a mathematical point of view. The stochastic analysis
apparatus, employed to study their probabilistic properties is complex and
rather difficult to manage. This study involves the ability to combine tools
available for diffusion processes and jump processes, in order to
characterise the semantics of these systems. The switching mechanism
(governed by a Markov chain in most cases) between the continuous dynamic of
the modes, together with the interaction between paths and boundaries, make
studying the stochastic processes that arise in this way very difficult and
challenging.

\subsection{General Stochastic Hybrid Systems}

We adopt the General Stochastic Hybrid System model presented in \cite{BL06}%
. This subsection describes the model and establishes the notation.

Let $Q$ be a set of discrete states. For each $q\in Q$, we consider the
Euclidean space $\Bbb{R}^{d(q)}$ with dimension $d(q)$ and we define an 
\emph{invariant }as an open subset $X^{q}$\ of $\Bbb{R}^{d(q)}$. The hybrid
state space is the set $X(Q,d,\mathcal{X})=\bigcup_{i\in Q}\{i\}\times X^{i}$
and $x=(i,z^{i})\in X(Q,d,\mathcal{X})$ is the hybrid state. The closure of
the hybrid state space will be $\overline{X}=X\cup \partial X,$ where $%
\partial X=\bigcup_{i\in Q}\{i\}\times \partial X^{i}$. It is known that $X$
can be endowed with a metric $\rho $ whose restriction to any component $%
X^{i}$ is equivalent to the usual component metric \cite{DA93}. Then $(X,%
\mathcal{B}(X))$ is a Borel space (homeomorphic to a Borel subset of a
complete separable metric space), where $\mathcal{B}(X)$ is the Borel $%
\sigma $-algebra of $X$. Let $\mathbf{B}(X)$ be the Banach space of the
bounded positive measurable functions on $X$ with the norm given by the
supremum.

\begin{definition}
A (General) Stochastic Hybrid System (SHS) is a collection \noindent $%
H=((Q,d,\mathcal{X}),(b,\sigma ),Init,(\lambda ,R))$; where

\noindent $\bullet $ $Q$ is a countable set of discrete states (modes); $%
d:Q\rightarrow \Bbb{N}$ is a map giving the dimensions of the invariants; $%
\mathcal{X}:Q\rightarrow \Bbb{R}^{d(.)}$ maps each $q\in Q$ into an open
subset $X^{q}$\ of $\Bbb{R}^{d(q)}$;

\noindent $\bullet $ $b:X(Q,d,\mathcal{X})\rightarrow \Bbb{R}^{d(.)}$ is a
vector field; $\sigma :X(Q,d,\mathcal{X})\rightarrow \Bbb{R}^{d(\cdot
)\times m}$ is a $X^{(\cdot )}$-valued matrix, $m\in \Bbb{N}$,

\noindent $\bullet $ $Init:\mathcal{B}(X)\rightarrow [0,1]$ is an initial
probability measure on $(X,\mathcal{B}(X))$;

\noindent $\bullet $ $\lambda :\overline{X}(Q,d,\mathcal{X})\rightarrow \Bbb{%
R}^{+}$ is a transition rate function; $R:\overline{X}\times \mathcal{B}(%
\overline{X})\rightarrow [0,1]$ is a stochastic kernel.
\end{definition}

The realization of an SHS is built as a \emph{Markov string} $H$ \cite{BL06}
obtained by the concatenation of some diffusion processes $(z_{t}^{i})$, $%
i\in Q$ together with a jumping mechanism given by a family of stopping
times $(S^{i})$ determined as minimum between the hitting times of the mode
boundaries and the stopping time governed by the exponential distribution
with the rate $\lambda $.

\smallskip It is known, from \cite{BL06}, that the realization of any SHS, $%
H $, under standard assumptions (about the diffusion coefficients, non-Zeno
executions, transition measure, etc, see \cite{BL06} for a detailed
presentation) is a strong Markov process (see the definition, for example,
in \cite{EK86}). Let $M=(x_{t},P_{x})$ be the Markov process associated to $%
H $. \noindent We adjoin an extra point $\Delta $ (the cemetery) to $X$ as
an isolated point, $X_{\Delta }=X\cup \{\Delta \}.$ The existence of $\Delta 
$ is assumed in order to have a probabilistic interpretation of $%
P_{x}(x_{t}\in X)<1,$ i.e. $\Delta $ is the state where the process lies
when it `dies'. Then, the `termination time' $\zeta (\omega )$ is the random
time when the process $M$ escapes to and is trapped at $\Delta $.

\noindent Let $\mathcal{P}=(P_{t})_{t>0}$ denote the\emph{\ semigroup} \emph{%
of operators }associated to $M$, which maps $\mathbf{B}(X)$ into itself
given by 
\begin{equation}
P_{t}f(x)=E_{x}f(x_{t}),\forall x\in X  \label{semigr}
\end{equation}
where $E_{x}$ is the expectation w.r.t. $P_{x}$. The semigroup $\mathcal{P}%
=(P_{t})_{t>0}$ can be thought of as an \emph{abstraction} of $M$, since
from $\mathcal{P}$ one can recuperate the initial process \cite{BG}. Recall
that a nonnegative function $f\in \mathbf{B}(X)$ is called $\alpha $-\emph{%
excessive} ($\alpha \geq 0$) if $e^{-\alpha t}P_{t}f\leq f$ for all $t\geq 0$
and $e^{-\alpha t}P_{t}f\nearrow f$ as $t\searrow 0$. If $\alpha =0$, a $0$%
-excessive function is simply called \emph{excessive function}. Let us
denote the \emph{cone of excessive functions} by $\mathcal{E}_{M}$. In the
theory of Markov processes, the excessive functions play the role of the
superharmonic functions from the theory of partial differential equations
(for e.g. a function $f\geq 0$ is superharmonic w.r.t. the Laplace operator
if $\Delta f\leq 0$). \noindent Note, that the definition of excessive
function can be given in terms of the \emph{operator resolvent} $\mathcal{U}$%
, which is the Laplace transform of $\mathcal{P}$. The \emph{operator
resolvent }$\mathcal{U}=(V_{r})_{r\geq 0}$ associated with $\mathcal{P}$ is 
\begin{equation}
V_{r}f(x)=\int_{0}^{\infty }e^{-rt}P_{t}f(x)dt,f\in \mathbf{B}(X),\text{ }%
x\in X.  \label{res_op}
\end{equation}

The infinitesimal generator $\mathcal{L}$ is the derivative of $P_{t}$ at $%
t=0$. Let $D(\mathcal{L})\subset \mathcal{B}_{b}(X)$ be the set of functions 
$f$ for which the following limit exists (denoted by $\mathcal{L}f$) 
\begin{equation}
\lim_{t\searrow 0}\frac{1}{t}(P_{t}f-f)  \label{generator}
\end{equation}

The following SHS\ property, proved in \cite{BL06}, has a major influence
over the coming results.

\begin{proposition}
Under the standard assumptions the realization $M$ of an SHS is a Borel
right process.
\end{proposition}

\medskip The sample paths of $M$ are right continuous with left limit, i.e.
are cadlags \cite{BL06}. The infinitesimal generator of an SHS is an
integro-differential operator (according with the terminology of \cite
{Menaldi2002}). The extended generator of an SHS has the following
expression: 
\begin{equation}
\mathcal{L}f(x)=\mathcal{L}_{cont}f(x)+\lambda (x)\int_{\overline{X}%
}(f(y)-f(x))R(x,dy)  \label{gen_GSHM}
\end{equation}
\noindent where $\mathcal{L}_{cont}f(x)$ has the standard form of the
diffusion infinitesimal operator. What makes this generator different from
the generator of a Feller Markov process (see \cite{Menaldi2002}) is its
domain that contains at least the set of second order differentiable
functions that satisfy the boundary condition, as follows: $f(x)=\int_{\Bbb{X%
}}f(y)R(x,dy),\;x\in \partial X.$ In the presence of forced jumps, the
generator of an SHS is an operator that is difficult to deal with, since its
domain does not even contain the set of all compactly supported $C^{\infty }$
functions.

\subsection{Stochastic Reachability\label{Subsection_RP}}

Let us consider $M=(\Omega ,\mathcal{F},\mathcal{F}_{t},x_{t},P_{x})$ being
a (strong right) Markov process, the realization of a stochastic hybrid
system. We address the following\emph{\ stochastic reachability problem}.

Given a target set, the objective of the reachability problem is to compute
the probability that the system trajectories from an arbitrary initial state
will reach the target set.

\noindent Formally, given a set $A\in \mathcal{B}(X)$ and a time horizon $%
T>0 $, let us define : 
\begin{eqnarray}
Reach_{T}(A) &=&\{\omega \in \Omega \mid \exists t\in [0,T]:x_{t}(\omega
)\in A\}  \nonumber \\
Reach_{\infty }(A) &=&\{\omega \in \Omega \mid \exists t\geq 0:x_{t}(\omega
)\in A\}.  \label{reach_set}
\end{eqnarray}
These two sets are the sets of trajectories of $M$, which reach the set $A$
(the flow that enters $A$) in the interval of time $[0,T]$ or $[0,\infty )$.

\noindent The reachability problem consists of determining the probabilities
of such sets. It can be shown that under our assumptions, since the process $%
M$ is Borel right process and has the cadlag property, the reachability
problem is well-defined, i.e. $Reach_{T}(A)$, $Reach_{\infty }(A)$ are
indeed measurable sets. Then the probabilities of reach events are 
\begin{equation}
P(T_{A}<T)\text{ or }P(T_{A}<\zeta )  \label{prob_reach_hitt}
\end{equation}
where $\zeta $ is the life time of $M$ and $T_{A}$ is the first hitting time
of $A$%
\begin{equation}
T_{A}=\inf \{t>0|x_{t}\in A\}.  \label{hit_time}
\end{equation}
$P$ is a probility that can be chosen to be $P_{x}$ (if we want to consider
the trajectories that start in $x$).

\section{Stochastic Reachability as an Optimal Stopping Problem\label%
{Sect_RP_OSP}}

In this section, in the framework of SHS, we explain how the stochastic
reachability problem can be transformed in an equivalent optimal stopping
problem.

\subsection{Optimal Stopping Problem\label{Subsection_OSP}}

In the following, the optimal stopping problem (OSP) for a (strong right)
Markov process $M=(\Omega ,\mathcal{F},\mathcal{F}_{t},x_{t},P_{x})$ taking
values in a Lusin space is briefly reviewed.

\noindent Let $\Sigma $ denote the set of stopping times (finite or not)
with respect to the filtration $\{\mathcal{F}_{t}\}$ (i.e. $\tau \in \Sigma
\Leftrightarrow \forall t,$ $\{\tau \leq t\}\in \mathcal{F}_{t}$). Consider $%
g:X\rightarrow \Bbb{R}$ a bounded measurable function called the \emph{%
reward function} (the interpretation being that if we stop the process at a
point $x\in X$ we obtain a reward $g(x)$). Obviously, the definition of OSP
requires some integrability conditions over the paths of $M$ (see, for
example \cite{KLM92}, for more details). Let $(y_{t})_{t\geq 0}$ be the 
\emph{reward process} defined by $y_{t}=g(x_{t})$, $t\geq 0$.

\noindent The \emph{maximal payoff function }(or the\emph{\ value function},
in the terminology of \cite{DA93}) is $v(x):=\sup \{E_{x}y_{\tau }|\tau \in
\Sigma \}$. The value function has been characterised in terms of the
minimal excessive function lying above the reward function for standard
Markov processes \cite{Shir76}, or more general for right Markov processes 
\cite{KLM92}.

\subsection{Stochastic reachability as an optimal stopping problem}

Let us introduce the \emph{reachability function} $w_{A}:X\rightarrow [0,1]$
associated to $A$, defined as 
\begin{equation}
w_{A}(x):=P_{x}[Reach_{\infty }(A)]\text{.}  \label{reach_fc}
\end{equation}

Taking the reward function $g$ to be equal with the indicator function of $A$%
, i.e. $g:=1_{A}$ we obtain the following result:

\begin{proposition}
\cite{BLL08} If $A\in \mathcal{B}(X)$ then the reachability function $w_{A}$
coincides with the value function of the reward process $y_{t}=1_{A}(x_{t})$%
, i.e. $w_{A}(x)=\sup \{P_{x}(x_{\tau }\in A)|\tau \in \Sigma \}$, $\forall
x\in X$.
\end{proposition}

\section{Optimal Stopping Problem for Borel Right Processes\label%
{Section_OSP}}

The realizations of SHS are (Borel) right processes, and therefore the
general theory of optimal stopping developed for right processes \cite
{Bismut77,merten73} can be applied. This theory is foundational since it
provides mathematical characterizations of the value function using
different tools available for right processes: (A) The approach presented in 
\cite{Bismut77} relies on a well-known connection between excessivity and a
special type of functional concavity. (B) The main result of \cite{merten73}
shows that the value function of an optimal stopping problem coincides with 
\emph{the Snell's enevelope} of the reward process. The \emph{Snell's envelop%
} is the smallest supermartingale that dominates the reward process.

Markov processes associated to SHS are (Borel) right processes, but they may
or may not be (i) standard Markov processes (whose theory is well-developed
in \cite{BG}) because the quasi-left continuity might fail, due to the
existence of the active boundaries when the process jumps in a new mode;
(ii) or, Feller processes (processes with continuous transition
probabilities) since they have predictable jumps (i.e. forced transitions).
See \cite{DA93}, for discussion of the Feller property for piecewise
deterministic Markov processes.

Therefore, the optimal stopping times need not exist, and the treatment of
the OSP requires some additional hypotheses. We recall the following
classical inclusions among the various classes of processes: (Feller)$%
\subset $(Hunt)$\subset $(special standard)$\subset $(right). \noindent
These different types of processes were introduced at various stages during
of the recent theory of Markov processes. This remark leads to the fact that
the well developed OSP\ methods available for standard Markov processes \cite
{Shir76}, or those available for Feller Markov processes (and their elliptic
integro-differential operators) \cite{Menaldi2002} are certainly not
directly applicable for the Borel right processes that arise in the SHS
context.

For right Markov processes, the value function has been characterised as the
minimal excessive function lying above the reward function \cite{KLM92}.
Therefore, for Borel right processes, the computation of the value function
corresponding to the OSP has to be based on specific features of these
processes. Then, we distinguish:

\noindent $\bullet $ \emph{analytic methods}, when the characterization of
the value function for the OSP has to consider: \noindent (i) different
representation way of excessive functions (as integrals of the Green kernel
of the process or the Riesz decomposition); \noindent (ii) variational
inequalities associated to some energy functional constructed using the
hitting/balayage operator \cite{GS87}; \noindent (iii) proving that the
value function is the solution of some variational inequality \cite
{Gatarek92}, then solving numerically this inequality;

\noindent $\bullet $ \emph{probabilistic methods} that consist in: \noindent
(i) approximations of the underlying Markov process by a Markov chain and
compute the value function corresponding to the chain by some specific
algorithms \cite{kushner77,Prandini06}; \noindent (ii) martingale methods
based on Snell's envelope; \noindent (iii) Monte Carlo Methods \cite{Blom05}.

\subsection{Variational inequalities}

Let $X$ be a bounded open set in $\Bbb{R}^{N}$ with \emph{smooth boundary}. $%
\Bbb{R}^{N}$ can be thought of as the Euclidean space where the state space
of a stochastic hybrid system can be embedded. According to \cite{Barles2007}%
, for the existence of the viscosity solutions some assumptions are
necessary. For the \emph{Dirichlet problem} given by (\ref{dyn_progr_eq})
and (\ref{boundary_cond}), these assumptions can be formulated as follows:

\noindent (A.1) $F\in C(\Bbb{R}^{N}\times \Bbb{R}\times \Bbb{R}^{N}\times 
\mathcal{S}_{N}\times \Bbb{R})$,

\noindent (A.2) $F$ satisfies the local and non-local \emph{degenerate
ellipticity condition(s)}: for any $x\in \Bbb{R}^{N}$, $u\in \Bbb{R}$, $p\in 
\Bbb{R}^{N}$, $A,B\in \mathcal{S}^{N}$, $l_{1},l_{2}\in \Bbb{R}$ 
\[
F(x,u,p,A,l_{1})\leq F(x,u,p,B,l_{2})\text{ if }A\geq B\text{, }l_{1}\geq
l_{2} 
\]

\noindent (A.3) $R(x,\cdot )$ is a probability measure on $X$ for $x\in
\partial X$ such that the linear operator 
\begin{equation}
Rv(x)=\int_{X}v(y)R(x,dy)  \label{op_reset}
\end{equation}
satisfies $|Rv(x)|\leq C||v||_{L^{1}(X)}$, for all $v\in L^{1}(X)$, where $C$
does not depend on $v$.

\noindent (A.4) The function $x\longmapsto Rv(x)$ is continuous w.r.t. $x\in 
\overline{X}$, uniformly for $v\in L^{\infty }(X)$.

Motivated by the expression of the generator associated to an SHS, let us
consider the linear integro-differential equations of the following form: 
\begin{equation}
F(x,u,D_{x}u,D_{x}^{2}u,\int_{X}u(y)R(x,dy))=0\text{,}  \label{dyn_progr_eq}
\end{equation}
where $D_{x}u$ denotes the space gradient, $D_{x}^{2}u$ the matrix of second
derivatives and $R(x,\cdot )$ is a probability kernel. Here, $\mathcal{S}%
^{N} $ denotes the space of symmetric $N\times N$ real valued matrices. The
applications for (\ref{dyn_progr_eq}) are dynamic programming equations
associated with the control of the right Markov processes that appear as
SHS\ realizations.

In the case when the state space $X$ is a bounded domain of a Euclidean
space, the process jumps back into $X$ upon hitting the boundary, which
leads to the following boundary condition to be coupled with the equation (%
\ref{dyn_progr_eq}), 
\begin{equation}
u(x)=\int_{X}u(y)R(x,dy)\text{, }x\in \partial X\text{.}
\label{boundary_cond}
\end{equation}

For a bounded function $u:X\rightarrow \Bbb{R}$, its upper/lower
semicontinuous envelopes can be defined in a standard way \cite{Barles2007}.
Furthermore, the definitions of the viscosity (sub/super) solutions for
second-order elliptic integro-differential equations are well established
now in the literature \cite{Barles2007}.

A bounded function $u:\overline{X}\rightarrow \Bbb{R}$ is a \emph{viscosity}
subsolution (resp. supersolution) of the \emph{Dirichlet problem} given by (%
\ref{dyn_progr_eq}) and (\ref{boundary_cond}), if it is a subsolution (resp.
supersolution) of (\ref{dyn_progr_eq}) in $X$ and, any $\phi \in C^{2}(X)$
and any local maximum (resp. minimum) $x\in \partial X$ for $u^{*}-\phi $
(resp. $u_{*}-\phi $) $\min \{u^{*}(x)-k(x),F(x,u^{*},D_{x}\phi
,D_{x}^{2}\phi ,\int_{X}u^{*}(y)R(x,dy)\}\leq 0$ (resp.

\noindent $\max \{u_{*}(x)-k(x),F(x,u_{*},D_{x}\phi ,D_{x}^{2}\phi
,\int_{X}u_{*}(y)R(x,dy)\}$

\noindent $\geq 0$) where $k(x):=\int_{X}u(y)R(x,dy)$, $x\in \partial X$.

\noindent In general, the existence of the solutions is proved by \emph{%
Perron's method}, introduced in the viscosity setting in \cite{Ishii87}.
That is, one proves that the supremum of a suitable set of subsolutions is
the solution. In order to do this, one needs the help of a \emph{comparison
principle}.

In particular, for an appropriate choice of $F$, this Dirichlet problem
becomes 
\begin{eqnarray}
\min (-\mathcal{L}u,u-g) &=&0\text{ in }X\text{,}  \label{dyn_prog} \\
u(x) &=&\int_{X}u(y)R(x,dy)\text{ on }\partial X  \label{bound_dyn}
\end{eqnarray}
where, $\mathcal{L}$ is the generator associated to an SHS, given by (\ref
{gen_GSHM}). Equation (\ref{dyn_prog}) with the boundary condition (\ref
{bound_dyn}) is\emph{\ the dynamic programming equation associated with the
optimal stopping problem for SHS }\cite{CDC07}. In this case, the assumption
A.2 involves that the diffusion term is non-degenerate. The assumption A.3
hints at the stochastic kernel $R$ (the SHS reset map) that should provide a
bounded linear operator and the assumption A.4 involves the Feller property
of the SHS\ realization \cite{DA93}. For the case of Feller processes, the
reward function is allowed to be semicontinuous, and the value function will
be also semicontinuous.

\noindent The main problem, in this context, is that an SHS\ is not a Feller
process unless there are no active boundaries. Then, these results can be
applied only to some particular cases.

\section{Approximation Methods for Stochastic Reachability}

In Section \ref{Sect_RP_OSP}, we have seen that computation of the reach set
probabilities could be reduced to the computation of the \emph{value function%
} of an optimal stopping problem with the reward function given by the
indicator function of the target set involved.

The OSP\ methods discussed in the Section \ref{Section_OSP} can be adapted
to SHS realizations considering the special features of SHS, in order to
obtain specific optimal stopping methods where the randomness and
hybridicity of SHS are clearly illustrated. On the other hand, these
features can be employed in order to obtain direct approximations of value
function of the OSP. We consider that for the stochastic processes that
arise in the SHS semantics, numerical computation of the reach set
probabilities as value functions for some OSPs could be supported by the
following methods: \noindent 1. approximate the underlying Markov process by
a Markov chain \cite{kushner77} and then compute the value function
corresponding to the chain; \noindent 2. prove that the value function of
the indicator function of a measurable set is the fixed point of some ``jump
operators'' \cite{DA93}.

\subsection{Approximations of SHS}

In this section, we summarize briefly the exponentially timestepping
approximation scheme (ETAS) for strong Markov processes with cadlag property
developed in \cite{BBB08}.

Let us consider a strong Markov process $M=(\Omega ,\mathcal{F},\mathcal{F}%
_{t},x_{t},P_{x})$. Suppose that $M$ has the cadlag property and the state
space $(X,\mathcal{B})$. $M$ is thought of as the realization of a
stochastic hybrid system $H$. Let $d$ be a compatible metric on $X$. Let $%
(P_{t})_{t>0}$ (resp. $(V_{r})_{r\geq 0}$) be its operator semigroup \ref
{semigr} (resp. operator resolvent \ref{res_op}).

For $x\in X$, $P_{x}$ is the law of $M$ under the initial condition $x_{0}=x$%
. \noindent In order to construct the sequence of jump processes that
approximate $M$, we need the following ingredients:

\noindent 1. A sequence of Markov chains $(\alpha ^{n})$. Each $\alpha
^{n}=(\alpha _{k}^{n})_{k=0,1,2,...}$ is a Markov chain on $X_{\Delta }$
with some initial distribution $\nu $ and the (homogeneous) transition
function, $K_{n}$ (i.e. a time-homogeneous Markov chain), given by $%
K_{n}(x,dy):=nV_{n}(x,dy)$, with $V_{n}$ is the stochastic kernel computed
from formula (\ref{res_op}).

\noindent 2. A sequence of Poisson processes $(\theta ^{n})$. Each $\theta
^{n}=(\theta _{t}^{n})_{t\geq 0}$ is a Poisson process\footnote{%
i.e. $P(\theta _{t}^{n}=k)=\exp (-nt)\frac{(nt)^{k}}{k!}$} with the
parameter $n$, independent of $\alpha ^{n}$.

\noindent Using these ingredients, we then define, for each $n\geq 1$, a 
\emph{continuous-time (regular) Markov step or Markov jump }process on $%
X_{\Delta }$ by $\rho _{t}^{n}:=\alpha _{\theta _{t}^{n}}^{n}$, $t\geq 0$;
whose embedded marked point process has the intensity equal to $n$ and state
space $X_{\Delta }$. This means that the jump times of the process $(\rho
_{t}^{n})$ are given by the arrival times of the Poisson process $(\theta
_{t}^{n})$ and its values between jumps are provided by the Markov chain $%
(\alpha _{k}^{n})$.

The above sequence of step processes converges in the Skorokhod topology and
consequently it converges weakly (in distribution)\footnote{%
A sequence of r.v. $(x_{n})_{n=1,2,...}$ $X$-valued, defined on $(\Omega ,%
\mathcal{F},\mathbf{P})$ \emph{converges weakly }(or \emph{in distribution})
to a r.v. $x_{0}$ if $\mathbf{E}f(x_{n})\rightarrow \mathbf{E}f(x_{0})$ as $%
n\rightarrow \infty $ $\forall f$ bounded continuous on $X$. Here, $\mathbf{E%
}f(x_{n})=\int_{X}f(x)P_{n}(dx)$ for $n\geq 0$, where $P_{n}=\mathbf{P}\circ
x_{n}^{-1}$ for $n\geq 0$.} to the initial Markov process.

\begin{theorem}
\cite{BBB08}\label{Th_convslaba} If $\alpha _{0}^{n}=x$, then the sequence $%
\{\rho ^{n}\}_{n\geq 1}$ of step processes converges weakly to $M$ (under $%
P_{x}$) as $n\rightarrow \infty $.
\end{theorem}

We explain how the hybrid structure of an SHS dynamics is considered in
ETAS. For each $\omega \in \Omega $, a hybrid trajectory $x_{t}(\omega
)=(q_{t}(\omega ),z_{t}(\omega ))$ of an SHS, $H$, can be thought of as the
union of `diffusion components' $\{z_{t}(\omega )|T_{k}(\omega )\leq
t<T_{k+1}(\omega )$, $k=1,2,...\}$ where $T_{1}<T_{2}<...$ represent the
jump times of $H$. Each component is provided with the label $%
q_{T_{k}(\omega )}(\omega )$ since $q_{t}(\omega )$ is constant in the
random time interval $[T_{k}(\omega ),T_{k+1}(\omega ))$. Then, a cadlag
trajectory of $H$ is implicitly carrying the hybrid dynamics structure. In
the ETAS, we do \emph{not }interpolate the Poisson times of step processes
considered there with the jumping times of $H$. The reason for not doing
this is that the latter jumping times can not be explicitly computed since a
jumping time might be the first boundary hitting time of some diffusion
process or some random time exponentially distributed with a rate depending
on the piece of diffusion trajectory covered until that moment.

In the ETAS, the trajectories of the system are considered `first class
citizens' and the methodology is heavily based on the use of a metric
defined on the space of all possible trajectories. Moreover, in this
approximation scheme, the approximating processes are step processes that
can be thought of as the realizations of some simpler SHSs. Since, the
forced transitions are removed, step processes belong to a nicer class of
Markov processes, namely standard Markov processes \cite{BG}. Then,
computational methods for the optimal stopping problem of step processes are
well developed \cite{Shir76}. Moreover, we will see that because of the weak
convergence of the approximating processes in ETAS, we can derive
convergence results for the optimal stopping value functions. Then, the
reach set probabilities for an SHS can be approximated by the reach set
probabilities of the step processes constructed in the ETAS.

\subsection{Approximation of the reach set probabilities}

Let us consider the reachability problem defined in Subsection \ref
{Subsection_RP} for an SHS, $H$. Suppose that the target set $A$ is a
measurable set of $X$. If $A$ is open (closed) then its indicator function $%
g:=1_{A}$ is a lower (upper) semicontinuous function. We can define also the
reward processes associated to the step processes $\rho ^{n}$ that are
defined in ETAS $y_{t}^{n}:=1_{A}(\rho _{t}^{n})$. Even in the case when the
reward function is semicontinuous, the reward processes $(y_{t})$, $%
(y_{t}^{n})$ are not longer cadlag processes (as the realization of $H$).
Therefore, the results about convergence of values in optimal stopping
nicely developed in \cite{CT07} are not directly applicable.

Practically, when we are studying the convergence of value functions $%
(v^{n}) $ (w.r.t. $(y_{t}^{n})$) to the desired value function $v$ (w.r.t. $%
(y_{t})$), we need to consider different aspects related to:

\noindent (i) the convergence of $(\rho ^{n})$ to $M$ (in the Skorokhod
topology); (ii) the semicontinuity of the reward function $g$; (iii) the
convergence of $(y_{t}^{n})$ to $(y_{t})$. Since the realization $M$ of $H$
is a Borel right process that may or may not have the property of quasi left
continuity (i.e. whenever $T_{n}$ is an increasing sequence of stopping
times with limit $T$, then almost surely $x_{T_{n}}\rightarrow x_{T}$), we
can not work under the hypotheses of the papers \cite{CT07}, where this
property is a datum from the beginning. Moreover, the results about the
convergence of the sequence of value functions have been also studied in a
particular case in \cite{LP90}. In the above cited paper, the Markov
processes considered are Feller, and the authors make some assumptions about
the density of $C_{0}(X)$ (the space of continuous real functions on $X$
vanishing at infinity) in the intersections of generator domains
corresponding to approximating processes $(\rho _{t}^{n})$ and initial
process $(x_{t})$. For the approximation scheme described in the previous
subsection, these assumptions are not longer in forced due to the
peculiarity of the generator domain of an SHS (see the boundary condition (%
\ref{boundary_cond})).

In the above mentioned papers, the convergence results are based on the
martingale problem associated to the Markov processes involved. The type of
functions that belong to the domain of an SHS generator constitutes a corner
stone of using its associated martingale in studying the OSP defined in
section \ref{Sect_RP_OSP}. Since the indicator function $1_{A}$ does not
belong to $D(\mathcal{L})$ (the domain of the SHS generator) we can not
reason about the OSP (that appears in related to stochastic reachability)
using the martingale problem.

The above discussion expresses, in fact, a very difficult situation and a
major contribution needs to be done. In the following, we propose a solution
for proving the convergence of the value functions based on the
correspondence between the reach set probabilities (\ref{prob_reach_hitt})
and the so-called Choquet capacities \cite{ManuCDC05}.

In the context of stochastic reachability, one can define a \emph{random set 
}$S:\Omega \rightarrow \mathcal{B}$; $\omega \mapsto \{x_{t}|0\leq t\leq
T\}. $ Then $Reach_{T}(A)=\{\omega |S(\omega )\cap A\neq \emptyset \}$, and
the reach set probability gives rise to a subadditive set function (called 
\emph{capacity}): $cap_{T}:\mathcal{B}\rightarrow [0,1]$; $%
cap_{T}(A):=P[Reach_{T}(A)]$ that, w.r.t. the random set $S$, plays the same
role as a distribution for a random variable. In the same way, $cap_{\infty
} $ can be defined. Analogously, we may define the capacities $cap_{T}^{n}$, 
$cap_{\infty }^{n}$ corresponding to the step processes $(\rho _{t}^{n})$.
Then, studying the convergence of the reach set probabilities corresponding
to the approximating processes means studying the convergence $%
cap_{T}^{n}\rightarrow cap_{T}$.

\begin{theorem}
If the sequence $\{\rho _{t}^{n}\}_{n\geq 1}$ of strong Markov processes
converges weakly to $(x_{t})$ (under $P$) as $n\rightarrow \infty $, then $%
cap_{T}^{n}\rightarrow cap_{T}$.
\end{theorem}

%TCIMACRO{\TeXButton{Proof}{\proof} }
%BeginExpansion
\proof%
%EndExpansion
The proof is lengthy and very technical. Due to the inherent room
limitations, we describe the main steps: 1) Weak convergence can be
characterized in terms of extended generators of the processes. 2) The
convergence of generators can be characterised in terms of convergence of
the operator semigroups and resolvents (Trotter-Kato theorem \cite{EK86}).
3) Convergence of resolvents involves the convergence of the associated
Dirichlet forms \cite{Wei99}. 4) Convergence of the Dirichlet forms implies
the convergence of their capacities \cite{Wei99}. 5) Convergence of the
Dirichlet form capacities conducts to the convergence the capacities
associated to the corresponding Markov processes. 6) \noindent The key of
the proof is provided by the characterization of Markov processes by
Dirichlet forms. A Dirichlet form is a quadratic form that can be naturally
associated to the generator of a Markov process \cite{MR 90}. $\blacksquare $

\section{Conclusions}

In this paper, we have investigated further developments using the
characterisation of the reachability problem of SHS as an optimal stopping
problem with a discontinuous reward function developed in \cite{BLL08}.
Because of the fact that the realization of SHS cover only a particular
subset of the class of (right) Markov processes, solving the optimal
stopping problem for such processes is difficult and challenging. For these
processes, characterizing the reachability problem as a viscosity solution
for some variational inequalities corresponding to their stochastic
generators needs additional assumptions regarding the continuity properties
of their internal structure (transition probabilities, stopping times). For
SHS, due to the interaction between continuous dynamics and boundary, these
assumptions may not be fulfilled. Therefore, to deal with the stochastic
reachability, we need to consider new approaches..

One of the major contribution of this paper is to provide rigorous evidence
about reliability of verification of SHS by optimal control. The behaviour
of a complex SHS can be constructively approximated by simpler Markov
processes, for which the optimal stopping problem is well understood. The
key for achieving this is the mathematical result that provides the
approximation of the reach set probabilities.

\begin{thebibliography}{99}
\bibitem{Barles2007}  Barles, C., Chasseigne, E., Imbert, C.: \emph{On the
Dirichlet Problem for Second-Order Elliptic Integro-Differential Equations}.
Preprint (2007).

\bibitem{Bismut77}  Bismut, J.-M.; Skalli, B.: \emph{Temps d'arr\^{e}t
Optimal, Th\'{e}orie G\'{e}n\'{e}rale des Processus et Processus de Markov}.
Prob. Th. Rel. Fields \textbf{36} (4) (1977): 301-313.

\bibitem{BL06Hyb}  Blom, H.A.P., Lygeros, J. (Eds.): ``\emph{Stochastic
Hybrid Systems: Theory and Safety Critical Applications''}. LNCIS \textbf{%
337 }(2006).

\bibitem{BG}  Blumenthal,\thinspace \ R.M., Getoor, R.K.: ``\emph{Markov
Processes and Potential Theory}'', Academic Press, New York and London
(1968).

\bibitem{BLL08}  Bujorianu, M.L., Lygeros, J., Langerak, R.: \emph{%
Reachability Analysis of Stochastic Hybrid Systems by Optimal Control}.
HSCC, LNCS \textbf{4981} (2008): 610-613.

\bibitem{BBB08}  Bujorianu, M.L., Bujorianu, M.C., Blom, H.A.P.: \emph{%
Approximate Abstractions of Stochastic Hybrid Systems}. IFAC WC (2008). In
press.

\bibitem{BL06}  Bujorianu, M.L., Lygeros, J.: \emph{Towards Modelling of
General Stochastic Hybrid Systems}. In \cite{BL06Hyb}: 3-30.

\bibitem{ManuCDC05}  Bujorianu, M.L.: \emph{A Statistical Inference Method
for the Stochastic Reachability Analysis}. CDC-ECC'05. 44th IEEE\ Conference
on Decision and Control (2005): 8088-8093.

\bibitem{CDC07}  Bujorianu, M.L., Lygeros, J.: \emph{New Insights on
Stochastic Reachability}. Proc. 46th Conference in Decision and Control
(2007).

\bibitem{CT07}  Coquet, F., Toldo, S.: \emph{Convergence of values in
optimal stopping and convergence of optimal stopping times}. Electr. J.
Prob. \textbf{12} (2007): 207-228.

\bibitem{DA93}  Davis, M.H.A.: ``\emph{Markov Models and Optimization}''
Chapman \& Hall, (1993).

\bibitem{KLM92}  El Karoui, N., Lepeltier, J.-P.; Millet, A.: \emph{A
Probabilistic Approach to the Reduite in Optimal Stopping}. Probab. Math.
Statist. \textbf{13 }(1992), no.1, 97-121.

\bibitem{EK86}  Ethier, S.N., Kurtz, T.G.: ``\emph{Markov Processes:
Characterization and Convergence''}. New York: John Wiley and Sons, (1986).

\bibitem{Gatarek92}  Gatarek, D.: \emph{On First Order Quasi-variational
Inequalities with Integral Terms.} Appl. Mathematics and Optimisation 
\textbf{24} (1992): 85-98.

\bibitem{Menaldi2002}  Garroni, M.G., Menaldi, J.L.:``\emph{Second Order
Elliptic Integro-Differential Problems}''. Chapman\&Hall/CRC (2002).

\bibitem{GS87}  Getoor, R.K., Steffens, J.: \emph{The Energy Functional,
Balayage, and Capacity}. Ann.I.H.P. \textbf{23}(2) (1987): 321-357.

\bibitem{Blom05}  Krystul, J., Blom, H.A.P.: \emph{Sequential Monte Carlo
Simulation of Rare Event Probability in Stochastic Hybrid Systems. }16th
IFAC World Congress (2005).

\bibitem{kushner77}  Kushner, H.J.: ``\emph{Probability Methods for
Approximations in Stochastic Control and for Elliptic Equations}''. Academic
Press, New York (1977).

\bibitem{Ishii87}  Ishii, H.: \emph{Perron's Method for Hamilton-Jacobi
Equations}. Duke Math. J., \textbf{55}(2) (1987): 369-384.

\bibitem{LP90}  Lamberton, D., Pag\`{e}s, G.: \emph{Sur l'approximation des
reduites}. Ann. Inst. Henri Poincar\'{e}, \textbf{26}(2) (1990): 331--355.

\bibitem{MR 90}  Ma, M., Rockner, M.: ``\emph{The Theory of (Non-Symmetric)
Dirichlet Forms and Markov Processes}''. (1990), Springer Verlag, Berlin.

\bibitem{merten73}  Mertens, J.F.: \emph{Strongly supermedian functions and
optimal stopping}. Z. Wahrscheinlichkeitstheorie verw. Gebiete \textbf{22 }%
(1972): 45-68.

\bibitem{oksendal2005}  Oksendal, B.; Sulem, A.: \emph{``Applied Stochastic
Control of Jump Diffusions''}. Springer Berlin. (2005).

\bibitem{Prandini06}  Prandini, M.; Hu, J.: \emph{A Stochastic Approximation
Method for Reachability Computations.} In \cite{BL06Hyb}: 107-139.

\bibitem{Shir76}  Shiryayev, A.N.: \emph{``Optimal Stopping Rules''}.
Springer Verlag (1976).

\bibitem{Wei99}  Wei, S.: Mosco Convergence of Quasi-Regular Dirichlet
Forms. Acta Math. Applic. Sinica \textbf{15} (3): 225-232.
\end{thebibliography}

\end{document}
