\newcommand{\figref}[1]{(Figure~\ref{#1})}     % Com par?nteses
%% bare_conf.tex
%% V1.3
%% 2007/01/11
%% by Michael Shell
%% See:
%% http://www.michaelshell.org/
%% for current contact information.
%%
%% This is a skeleton file demonstrating the use of IEEEtran.cls
%% (requires IEEEtran.cls version 1.7 or later) with an IEEE conference paper.
%%
%% Support sites:
%% http://www.michaelshell.org/tex/ieeetran/
%% http://www.ctan.org/tex-archive/macros/latex/contrib/IEEEtran/
%% and
%% http://www.ieee.org/

%%*************************************************************************
%% Legal Notice:
%% This code is offered as-is without any warranty either expressed or
%% implied; without even the implied warranty of MERCHANTABILITY or
%% FITNESS FOR A PARTICULAR PURPOSE! 
%% User assumes all risk.
%% In no event shall IEEE or any contributor to this code be liable for
%% any damages or losses, including, but not limited to, incidental,
%% consequential, or any other damages, resulting from the use or misuse
%% of any information contained here.
%%
%% All comments are the opinions of their respective authors and are not
%% necessarily endorsed by the IEEE.
%%
%% This work is distributed under the LaTeX Project Public License (LPPL)
%% ( http://www.latex-project.org/ ) version 1.3, and may be freely used,
%% distributed and modified. A copy of the LPPL, version 1.3, is included
%% in the base LaTeX documentation of all distributions of LaTeX released
%% 2003/12/01 or later.
%% Retain all contribution notices and credits.
%% ** Modified files should be clearly indicated as such, including  **
%% ** renaming them and changing author support contact information. **
%%
%% File list of work: IEEEtran.cls, IEEEtran_HOWTO.pdf, bare_adv.tex,
%%                    bare_conf.tex, bare_jrnl.tex, bare_jrnl_compsoc.tex
%%*************************************************************************

% *** Authors should verify (and, if needed, correct) their LaTeX system  ***
% *** with the testflow diagnostic prior to trusting their LaTeX platform ***
% *** with production work. IEEE's font choices can trigger bugs that do  ***
% *** not appear when using other class files.                            ***
% The testflow support page is at:
% http://www.michaelshell.org/tex/testflow/



% Note that the a4paper option is mainly intended so that authors in
% countries using A4 can easily print to A4 and see how their papers will
% look in print - the typesetting of the document will not typically be
% affected with changes in paper size (but the bottom and side margins will).
% Use the testflow package mentioned above to verify correct handling of
% both paper sizes by the user's LaTeX system.
%
% Also note that the "draftcls" or "draftclsnofoot", not "draft", option
% should be used if it is desired that the figures are to be displayed in
% draft mode.
%
\documentclass[conference]{IEEEtran}
% Add the compsoc option for Computer Society conferences.
%
% If IEEEtran.cls has not been installed into the LaTeX system files,
% manually specify the path to it like:
% \documentclass[conference]{../sty/IEEEtran}





% Some very useful LaTeX packages include:
% (uncomment the ones you want to load)


% *** MISC UTILITY PACKAGES ***
%
%\usepackage{ifpdf}
% Heiko Oberdiek's ifpdf.sty is very useful if you need conditional
% compilation based on whether the output is pdf or dvi.
% usage:
% \ifpdf
%   % pdf code
% \else
%   % dvi code
% \fi
% The latest version of ifpdf.sty can be obtained from:
% http://www.ctan.org/tex-archive/macros/latex/contrib/oberdiek/
% Also, note that IEEEtran.cls V1.7 and later provides a builtin
% \ifCLASSINFOpdf conditional that works the same way.
% When switching from latex to pdflatex and vice-versa, the compiler may
% have to be run twice to clear warning/error messages.






% *** CITATION PACKAGES ***
%
\usepackage{cite}
% cite.sty was written by Donald Arseneau
% V1.6 and later of IEEEtran pre-defines the format of the cite.sty package
% \cite{} output to follow that of IEEE. Loading the cite package will
% result in citation numbers being automatically sorted and properly
% "compressed/ranged". e.g., [1], [9], [2], [7], [5], [6] without using
% cite.sty will become [1], [2], [5]--[7], [9] using cite.sty. cite.sty's
% \cite will automatically add leading space, if needed. Use cite.sty's
% noadjust option (cite.sty V3.8 and later) if you want to turn this off.
% cite.sty is already installed on most LaTeX systems. Be sure and use
% version 4.0 (2003-05-27) and later if using hyperref.sty. cite.sty does
% not currently provide for hyperlinked citations.
% The latest version can be obtained at:
% http://www.ctan.org/tex-archive/macros/latex/contrib/cite/
% The documentation is contained in the cite.sty file itself.






% *** GRAPHICS RELATED PACKAGES ***
%
\ifCLASSINFOpdf
 \usepackage[pdftex]{graphicx}
  % declare the path(s) where your graphic files are
  % \graphicspath{{../pdf/}{../jpeg/}}
  % and their extensions so you won't have to specify these with
  % every instance of \includegraphics
   \DeclareGraphicsExtensions{.pdf,.jpeg,.png}
\else
  % or other class option (dvipsone, dvipdf, if not using dvips). graphicx
  % will default to the driver specified in the system graphics.cfg if no
  % driver is specified.
  % \usepackage[dvips]{graphicx}
  % declare the path(s) where your graphic files are
  % \graphicspath{{../eps/}}
  % and their extensions so you won't have to specify these with
  % every instance of \includegraphics
  % \DeclareGraphicsExtensions{.eps}
\fi
% graphicx was written by David Carlisle and Sebastian Rahtz. It is
% required if you want graphics, photos, etc. graphicx.sty is already
% installed on most LaTeX systems. The latest version and documentation can
% be obtained at: 
% http://www.ctan.org/tex-archive/macros/latex/required/graphics/
% Another good source of documentation is "Using Imported Graphics in
% LaTeX2e" by Keith Reckdahl which can be found as epslatex.ps or
% epslatex.pdf at: http://www.ctan.org/tex-archive/info/
%
% latex, and pdflatex in dvi mode, support graphics in encapsulated
% postscript (.eps) format. pdflatex in pdf mode supports graphics
% in .pdf, .jpeg, .png and .mps (metapost) formats. Users should ensure
% that all non-photo figures use a vector format (.eps, .pdf, .mps) and
% not a bitmapped formats (.jpeg, .png). IEEE frowns on bitmapped formats
% which can result in "jaggedy"/blurry rendering of lines and letters as
% well as large increases in file sizes.
%
% You can find documentation about the pdfTeX application at:
% http://www.tug.org/applications/pdftex





% *** MATH PACKAGES ***
%
%\usepackage[cmex10]{amsmath}
% A popular package from the American Mathematical Society that provides
% many useful and powerful commands for dealing with mathematics. If using
% it, be sure to load this package with the cmex10 option to ensure that
% only type 1 fonts will utilized at all point sizes. Without this option,
% it is possible that some math symbols, particularly those within
% footnotes, will be rendered in bitmap form which will result in a
% document that can not be IEEE Xplore compliant!
%
% Also, note that the amsmath package sets \interdisplaylinepenalty to 10000
% thus preventing page breaks from occurring within multiline equations. Use:
%\interdisplaylinepenalty=2500
% after loading amsmath to restore such page breaks as IEEEtran.cls normally
% does. amsmath.sty is already installed on most LaTeX systems. The latest
% version and documentation can be obtained at:
% http://www.ctan.org/tex-archive/macros/latex/required/amslatex/math/





% *** SPECIALIZED LIST PACKAGES ***
%
%\usepackage{algorithmic}
% algorithmic.sty was written by Peter Williams and Rogerio Brito.
% This package provides an algorithmic environment fo describing algorithms.
% You can use the algorithmic environment in-text or within a figure
% environment to provide for a floating algorithm. Do NOT use the algorithm
% floating environment provided by algorithm.sty (by the same authors) or
% algorithm2e.sty (by Christophe Fiorio) as IEEE does not use dedicated
% algorithm float types and packages that provide these will not provide
% correct IEEE style captions. The latest version and documentation of
% algorithmic.sty can be obtained at:
% http://www.ctan.org/tex-archive/macros/latex/contrib/algorithms/
% There is also a support site at:
% http://algorithms.berlios.de/index.html
% Also of interest may be the (relatively newer and more customizable)
% algorithmicx.sty package by Szasz Janos:
% http://www.ctan.org/tex-archive/macros/latex/contrib/algorithmicx/




% *** ALIGNMENT PACKAGES ***
%
%\usepackage{array}
% Frank Mittelbach's and David Carlisle's array.sty patches and improves
% the standard LaTeX2e array and tabular environments to provide better
% appearance and additional user controls. As the default LaTeX2e table
% generation code is lacking to the point of almost being broken with
% respect to the quality of the end results, all users are strongly
% advised to use an enhanced (at the very least that provided by array.sty)
% set of table tools. array.sty is already installed on most systems. The
% latest version and documentation can be obtained at:
% http://www.ctan.org/tex-archive/macros/latex/required/tools/


%\usepackage{mdwmath}
%\usepackage{mdwtab}
% Also highly recommended is Mark Wooding's extremely powerful MDW tools,
% especially mdwmath.sty and mdwtab.sty which are used to format equations
% and tables, respectively. The MDWtools set is already installed on most
% LaTeX systems. The lastest version and documentation is available at:
% http://www.ctan.org/tex-archive/macros/latex/contrib/mdwtools/


% IEEEtran contains the IEEEeqnarray family of commands that can be used to
% generate multiline equations as well as matrices, tables, etc., of high
% quality.


%\usepackage{eqparbox}
% Also of notable interest is Scott Pakin's eqparbox package for creating
% (automatically sized) equal width boxes - aka "natural width parboxes".
% Available at:
% http://www.ctan.org/tex-archive/macros/latex/contrib/eqparbox/





% *** SUBFIGURE PACKAGES ***
%\usepackage[tight,footnotesize]{subfigure}
% subfigure.sty was written by Steven Douglas Cochran. This package makes it
% easy to put subfigures in your figures. e.g., "Figure 1a and 1b". For IEEE
% work, it is a good idea to load it with the tight package option to reduce
% the amount of white space around the subfigures. subfigure.sty is already
% installed on most LaTeX systems. The latest version and documentation can
% be obtained at:
% http://www.ctan.org/tex-archive/obsolete/macros/latex/contrib/subfigure/
% subfigure.sty has been superceeded by subfig.sty.



%\usepackage[caption=false]{caption}
%\usepackage[font=footnotesize]{subfig}
% subfig.sty, also written by Steven Douglas Cochran, is the modern
% replacement for subfigure.sty. However, subfig.sty requires and
% automatically loads Axel Sommerfeldt's caption.sty which will override
% IEEEtran.cls handling of captions and this will result in nonIEEE style
% figure/table captions. To prevent this problem, be sure and preload
% caption.sty with its "caption=false" package option. This is will preserve
% IEEEtran.cls handing of captions. Version 1.3 (2005/06/28) and later 
% (recommended due to many improvements over 1.2) of subfig.sty supports
% the caption=false option directly:
%\usepackage[caption=false,font=footnotesize]{subfig}
%
% The latest version and documentation can be obtained at:
% http://www.ctan.org/tex-archive/macros/latex/contrib/subfig/
% The latest version and documentation of caption.sty can be obtained at:
% http://www.ctan.org/tex-archive/macros/latex/contrib/caption/




% *** FLOAT PACKAGES ***
%
%\usepackage{fixltx2e}
% fixltx2e, the successor to the earlier fix2col.sty, was written by
% Frank Mittelbach and David Carlisle. This package corrects a few problems
% in the LaTeX2e kernel, the most notable of which is that in current
% LaTeX2e releases, the ordering of single and double column floats is not
% guaranteed to be preserved. Thus, an unpatched LaTeX2e can allow a
% single column figure to be placed prior to an earlier double column
% figure. The latest version and documentation can be found at:
% http://www.ctan.org/tex-archive/macros/latex/base/



%\usepackage{stfloats}
% stfloats.sty was written by Sigitas Tolusis. This package gives LaTeX2e
% the ability to do double column floats at the bottom of the page as well
% as the top. (e.g., "\begin{figure*}[!b]" is not normally possible in
% LaTeX2e). It also provides a command:
%\fnbelowfloat
% to enable the placement of footnotes below bottom floats (the standard
% LaTeX2e kernel puts them above bottom floats). This is an invasive package
% which rewrites many portions of the LaTeX2e float routines. It may not work
% with other packages that modify the LaTeX2e float routines. The latest
% version and documentation can be obtained at:
% http://www.ctan.org/tex-archive/macros/latex/contrib/sttools/
% Documentation is contained in the stfloats.sty comments as well as in the
% presfull.pdf file. Do not use the stfloats baselinefloat ability as IEEE
% does not allow \baselineskip to stretch. Authors submitting work to the
% IEEE should note that IEEE rarely uses double column equations and
% that authors should try to avoid such use. Do not be tempted to use the
% cuted.sty or midfloat.sty packages (also by Sigitas Tolusis) as IEEE does
% not format its papers in such ways.





% *** PDF, URL AND HYPERLINK PACKAGES ***
%
%\usepackage{url}
% url.sty was written by Donald Arseneau. It provides better support for
% handling and breaking URLs. url.sty is already installed on most LaTeX
% systems. The latest version can be obtained at:
% http://www.ctan.org/tex-archive/macros/latex/contrib/misc/
% Read the url.sty source comments for usage information. Basically,
% \url{my_url_here}.





% *** Do not adjust lengths that control margins, column widths, etc. ***
% *** Do not use packages that alter fonts (such as pslatex).         ***
% There should be no need to do such things with IEEEtran.cls V1.6 and later.
% (Unless specifically asked to do so by the journal or conference you plan
% to submit to, of course. )


% correct bad hyphenation here
\hyphenation{op-tical net-works semi-conduc-tor}


\begin{document}
%
% paper title
% can use linebreaks \\ within to get better formatting as desired
\title{Modular distributed visualization and collaboration for a real-time 3D visualizer}


% author names and affiliations
% use a multiple column layout for up to three different
% affiliations

% conference papers do not typically use \thanks and this command
% is locked out in conference mode. If really needed, such as for
% the acknowledgment of grants, issue a \IEEEoverridecommandlockouts
% after \documentclass

% for over three affiliations, or if they all won't fit within the width
% of the page, use this alternative format:
% 
\author{\IEEEauthorblockN{Alexandre Valdetaro\IEEEauthorrefmark{1},
Alberto Raposo\IEEEauthorrefmark{2} and
Pablo Elias\IEEEauthorrefmark{3}}
\IEEEauthorblockA{\IEEEauthorrefmark{1}Informatics Department\\
Pontifical Catholic University of Rio de Janeiro\\
Rio de Janeiro, Brazil\\
Email: xvaldetaro@gmail.com}
\IEEEauthorblockA{\IEEEauthorrefmark{2}Informatics Department\\
Pontifical Catholic University of Rio de Janeiro\\
Rio de Janeiro, Brazil\\
Email: abraposo@tecgraf.com.br}
\IEEEauthorblockA{\IEEEauthorrefmark{3}Tecgraf Institute\\
Pontifical Catholic University of Rio de Janeiro\\
Rio de Janeiro, Brazil\\
Email: pelias@tecgraf.puc-rio.br}}
%\IEEEauthorblockA{\IEEEauthorrefmark{3}Starfleet Academy, San Francisco, California 96678-2391\\
%Telephone: (800) 555--1212, Fax: (888) 555--1212}
%\IEEEauthorblockA{\IEEEauthorrefmark{4}Tyrell Inc., 123 Replicant Street, Los Angeles, California 90210--4321}}




% use for special paper notices
%\IEEEspecialpapernotice{(Invited Paper)}




% make the title area
\maketitle


\begin{abstract}
%\boldmath
In this work, we present the design and implementation of distributed visualization and collaboration for an immersive 3D visualizer in a component-oriented fashion. The design follows an MVC approach, isolating all the business objects in the lowest level of the application, making it modular and extensible, therefore providing an easier prototyping of functionality and the isolation of complex business logic algorithms. This design as a solution came from the necessity of an existing visualizer with a monolithic implementation, whose maintainability and improvement are impaired due to the coupling between the business logic and the diverse visualization and distribution algorithms. Our design may be reused as an inspiration for other visualizers that wish to reduce the complexity and cost of the development  of new business functionality. We implemented the designed visualizer, where we verified both the synchronism of the distributed visualization and the consistency of the collaboration among multiple nodes. We also evaluated the performance impact caused by the distributed visualization.
\end{abstract}
% IEEEtran.cls defaults to using nonbold math in the Abstract.
% This preserves the distinction between vectors and scalars. However,
% if the conference you are submitting to favors bold math in the abstract,
% then you can use LaTeX's standard command \boldmath at the very start
% of the abstract to achieve this. Many IEEE journals/conferences frown on
% math in the abstract anyway.

% no keywords




% For peer review papers, you can put extra information on the cover
% page as needed:
% \ifCLASSOPTIONpeerreview
% \begin{center} \bfseries EDICS Category: 3-BBND \end{center}
% \fi
%
% For peerreview papers, this IEEEtran command inserts a page break and
% creates the second title. It will be ignored for other modes.
\IEEEpeerreviewmaketitle



\section{Introduction}
\label{intro}
Immersive applications seek to provide an experience to the user as close to real life as possible. Such experience must re-create sensorial stimuli and perception of space in order to induce the user's brain into believing that the immersive experience is real life. Although an immersive experience can be delivered by any kind of system, Real-Time 3D visualizers may stand out as the key player in the field. A visualizer is an application that enables the user to visually explore a virtual scene. The success of such applications is consequence of many features combined, such as, credible visual input to the user by usage of modern computer graphics techniques, enhanced illusion of visual depth due to stereoscopy, precise and immediate control thanks to real-time reading and processing of user input. Therefore, the development of an application that provides an immersive experience to the user can be very complex because of these many requirements and their implementation in a real-time reactive system.

\subsection{Distributed Visualization}
Distributed visualization is the process of displaying a single virtual scene from multiple views. Such process faces the technical problem of how to render and synchronize so many views and output to different screens. A traditional approach to achieve distributed visualization is through the usage of out-of-the box systems that make use of dedicated hardware. The hardware controls all the visualization devices. Such solution presents, usually, a high cost of deployment and maintenance.

The rendering speed depends roughly on the amount of geometry---the number of vertices of every object in the viewable scene---being processed and the size of the output image. Therefore, rendering multiple views of the same scene gets very expensive in terms of processing, specially if the output images have to be very big for display walls. Consequently, distributing the scene to multiple nodes can be a feasible solution---despite the distribution difficulties. There are many possible designs of a visualizer that provide scene distribution as well as many tools available for it. In the related work section we make a review of the main available tools and their usefulness for our scenario.

\subsection{Collaboration}
Collaborative software is a software that enables multiple users to collaborate to achieve a common goal \cite{ellis1991groupware}. A collaborative visualizer lets the users explore a common scene, and possibly make changes and view other users avatars in the scene. The most common case of a collaborative visualizer is a multi-player game, where the players interact with each other in a shared virtual world. The players can talk to each other, engage non player characters and change the persistent world in many ways. 

\subsection{Real-time Visualizer}
Any real-time application must be reactive, i.e., it must process input from external devices with a delay small enough not to break user immersion. Moreover, the interactive visualization requires that at least a given amount of frames be rendered every second. Thus, efficiency is the utmost requirement when developing a feature in a real time 3D visualizer. 

In order to achieve maximum efficiency in a visualizer's many routines, e.g., rendering, input processing, distribution, data loading and so on, there is a natural tendency of the developers to trade abstractions for low-level APIs in order to have access to every available optimization setting. However, the exposure of every low level API to the business logic developer can greatly increase the complexity, which may lead to an increase in the lifecycle cost \cite{Boe91} and effort \cite{MSPatterns} required to maintain and extend a visualizer with new functionalities. 

To the best of our knowledge, the majority of the 3D visualizers, where efficiency is paramount, are implemented in a monolithic way. There is no clear separation between business, distribution, rendering, architectural elements and so on. This approach can have some advantages because the development of every functionality has access to every low level system, therefore the developer can highly optimize the rendering, distribution and other techniques used in the developed feature. However, from our experience (see section \ref{sec:background}), the constant increase in complexity for the development of simple business/application features, and the tight coupling between the logic layers can seriously impair the maintainability and extensibility of the application. This can lead to a stall in productivity, where developers suffer with the high complexity they need to understand in order to develop a simple feature. 

We believe that the application programmer, who develops the features that aggregate real knowledge value in a software, should be focused on the domain of the application, the business logic, transactions, user interface and any other front end concern. Therefore the application programmer should work with as much high level abstractions as possible, such as frameworks, tools and \emph{middleware} \cite{SAFoundations}. However, one must always consider the trade-offs when abstracting low level systems, as the usage of low level optimizations may be necessary in many cases as explained above

This work is organized as follows: 
In section \ref{sec:background} we show our background and motivation for this work. In section \ref{cha:rel_work} we walk through the available solutions for our distributed visualization and collaboration requirements for an immersive visualizer. In section \ref{cha:design_hadean} we explain our design for a visualizer oriented towards modularity and extensibility. In section \ref{cha:DVCM} we explain our design for the module that provides distributed visualization and collaboration for the visualizer. In section \ref{sec:results} we show the performance of the implemented example system. In section \ref{cha:conclusion} we present our conclusions and thoughts about this work and in the last section we indicate some ideas for future work.

\section{Background and Motivation}
\label{sec:background}
This work is developed as a part of SiVIEP (Integrated Visualization System of Exploration and Production) project. SiVIEP is an immersive scientific visualizer, which supports visualization of reservoirs, geological surfaces, wells, risers, platforms and many other objects from the oil field domain. All of these objects can be visualized together inside one project in a 3D multi-screen stereoscopic setup, controlled by several manipulators and tracking devices. Many supported objects pose already a challenge to be rendered in a single screen due to the number of polygons, simulation data and property visualization. Therefore, complex distribution algorithms must be employed for the system to be able to render all the required objects in a multi-screen stereoscopic setup.

SiVIEP's current production version is a monolithic \emph{C++} application using Qt for user interface and OpenSG \cite{OpenSG} for distributed rendering. There are some bottlenecks with the distribution provided by OpenSG \cite{OpenSG} for our scenario, and these are discussed in the related work section. Also, the complexity of adding new business objects to the current \emph{C++} version is too high due to the tight coupling between many non cohesive parts of the application.

We started this work to provide a solution for these SiVIEP issues with a new design that could isolate the rendering and distribution complexity from the business logic.

\section{Related Work}
\label{cha:rel_work}

A number of generic solutions for distributed visualization have been developed, and are available commercially or open source, e.g., Equalizer \cite{Equalizer}, Chromium \cite{Chromium}, \cite{OpenSG} and VRJuggler \cite{VRJuggler}.  Some of those are completely transparent, some require a certain level of adaptation from the programmers and some force its own programming model. In the collaboration field, most of the state of art solutions are far more complex and generalist than desired for this work. However, some of these works contributed to model our solution and are going to be detailed here.

OpenSG \cite{OpenSG} is a scene graph as much as any non-distributed scene graph. A user can develop using OpenSG without distributed visualization in mind despite being one of the scene graph's major features. The distribution strategy of OpenSG works by distributing the scene graph with all the graphical nodes containing vertices, textures, matrices and other graphical primitives. Therefore all the graphical nodes must be serializable. Making a serializable graphical-node is simple if it is a regular triangle mesh that requires no processing during runtime because the node will be serialized once during pre-processing only. However, many nodes have a constantly varying set of primitives such as: large on-demand loaded files such as terrains, photo-realistic detailed meshes with continuous level-of-detail techniques, simulation data visualization. These variable nodes are very expensive to distribute, as their data is constantly varying and can be of very large size. Taking our oil field visualizer scenario as an example, most of our domain is composed of simulation data, that requires different visualization modes with generated graphical data on demand. Therefore, this graphical distribution approach is not suitable for us.

Chromium \cite{Chromium} works by creating a powerful abstraction of the OpenGL API, that is, it intercepts every call to the API and executes a scalable rendering algorithm to distribute the call among multiple nodes. Such approach is very clean and non-intrusive, but requires the constant distribution of graphical data, which can be a bottleneck if the scene is very diverse so that the GPU memory will need to be updated frequently. Also, this approach does not consider collaboration. In order to provide collaboration with Chromium, a whole separate solution must be implemented.

Equalizer \cite{Equalizer}, Chromium\cite{Chromium} and VRJuggler \cite{VRJuggler} are tailored for parallel and scalable rendering, that is, using multiple nodes for rendering the same scene independently of the number of screens. They all support outputting the rendered scene to multiple screens making distributed visualization feasible. One of these tools stands out as the most complete and non-invasive, the Equalizer \cite{Equalizer}. It is a very powerful and complete tool that forces the programmer to abstract the rendering code from the rest of the application. Thus, it distributes this rendering client for the slave nodes. Such nodes perform rendering tasks controlled by a master node, which is configured by configuration files describing the available resources in the cluster as well as the desired compositing strategies. The application is unchanged for any kind of scalable setup. The rendering in the slave nodes can be configured for many different compositing strategies \cite{sorting}, such as sort-first or tile based, DB based or sort-last, among others \cite{SortStrat}. After one of these strategies is used, the output image is copied to the configured output walls/screens. Such strategy of scalable rendering is very powerful, but unnecessary for a typical case of  multi-screen rendering setup (our whole range of cluster examples fit the common case) where every screen is driven by an individual node of the cluster. Also, if the application is going to be used in single computer workstations to visualize the same projects, then using the scalable rendering system to achieve better system results can be pointless, as the scene needs to be processed by a single machine as well. 

All of the aforementioned solutions provide the distribution on the graphical layer of the application, that is, they distribute graphical code, graphic controlling commands, graphical primitives and so on. However, for a collaborative visualization to be achieved, a distribution of the business domain (not just its graphical representation) is necessary. Therefore, in the mentioned approaches, the collaborative visualization would have to be solved by other means. In this work's solution, we provide a DSO (distributed shared objects) system for our domain data. That is, our domain data is a single shared graph among the nodes. This way, our solution provide collaboration through two-way updates of local changes for every node. Moreover, this same solution will serve as the ground for our multi-screen rendering algorithm to work with, as all the graphical representation can be loaded locally based on the received domain information. 

\section{Design of a Modular Visualizer}
\label{cha:design_hadean}

\subsection{Domains and Entities} The domain of an application represents \emph{business knowledge}---as the \emph{model}---in a \emph{Model View Controller}---MVC--- paradigm. What we have referenced in the previous sections as \emph{business data} or the \emph{business objects layer}, is technically the \emph{application domain}. It is a mirror of the real world, composed of all the real world objects that the application wants to represent. Ideally, there should be a one-to-one correspondence between the domain objects and their real world counterparts. These domain objects are represented by \emph{Entities}. Examples of entities in an oil field visualizer application are risers, wells, reservoirs and so on. We designed the domain structure as a graph and the entities as collections of attributes with no relational structure associated. Furthermore the graph and its entities can be versioned, shared and persisted if needed.

\subsection{Scenes} We define a \emph{scene} as a visualizable collection of entities with behaviors and tags. This definition however, may be considered incorrect because of our definition of domain, as the scene clearly has different properties and elements than a real world object. Moreover a more purist definition could consider the domain itself a scene, which is the case in many visualizers. Nevertheless we regard the scene as business knowledge because they represent snapshots of the same domain in different times, configurations, simulation scenarios and so on. Figure \ref{fig:domain_scene_entity} shows the structuring of our domain regarding scenes. Examples of scenes in an oil field visualizer are: the oil field when exploration began, the oil field in present date, the oil field in a catastrophe simulation scenario, etc .

\begin{figure*}[!t]
\centering
\includegraphics[width=5.5in]{domain_scene_entity.pdf}
\caption{Chart showing the relation between the domain, the scenes and the entities}
\label{fig:domain_scene_entity}
\end{figure*}

Figure \ref{fig:blueprint} shows the layering of the system according to the MVC concept and an overview of the interfaces and interactions among the many modules.

\begin{figure*}[!t]
\centering
\includegraphics[width=5.5in]{blueprint.pdf}
\caption{Blueprint of the the visualizer architecture}
\label{fig:blueprint}
\end{figure*}

\subsection{Domain Model Framework}
\label{sec:interactions}
As mentioned above, the domain should be an object graph. However, this graph is not useful for the rest of the application only by itself. Its state needs to be propagated through the layers above it. Therefore it is necessary to have a graph controlling system that provides controlled access to the domain graph and notifications to everyone that depends on it. We designed this controlling system as a non-intrusive module that keeps track of all the changes in the object graph and provides a fine grained notification system for the rest of the application. We call this system the DMF---Domain Model Framework---\figref{fig:DMF}. Every component that applies some modification in the business object graph needs to certify that these changes will be propagated to everyone else. Hence whenever a change has to be applied to the graph, this change has to go through the DMF. Consequently, every component that needs to display information about a business object needs to be connected to the notification system of the DMF.

The DMF notification system follows the \emph{Observer} pattern \cite{GoF95}. Every element that needs to watch the state of a piece of the domain will be registered as an observer to that piece in the DMF notification system. Accordingly, the DMF will notify that observing element whenever a change is applied to the observed piece of the domain. The notification itself consists of the previous and the current states of the piece. The DMF also provides a global graph access system for the elements that need to change the graph. Notice that in our design, the actual objects from the business object graph does not reside inside the DMF data structures. As we mentioned before, the domain management is non-intrusive. Therefore, the DMF has an internal graph containing meta information about the actual objects and it keeps versioning information, which is necessary for undo/redo operations and is also very important for our distribution module that will be explained in the next section. 

This non-intrusive approach makes the DMF useful for any graph of objects that must be versioned and shared. However, as the actual objects are not inside the DMF, there is the risk of a business object being modified from outside of the DMF interface. If this happens, then there will be an inconsistent state. Therefore, there must be strict guidelines for the developers on the usage of such kind of framework.

\begin{figure*}[!t]
\centering
\includegraphics[width=5.5in]{dmf.pdf}
\caption{DMF architecture overview}
\label{fig:DMF}
\end{figure*}

With these foundations set, a module responsible for displaying the 3D scene can be a simple observer of the current scene and its referred entities, which are all contained by the domain graph. Whenever a new entity reference is added to the current scene, the module searches for a graphical representation for the object and add it to the underlying graphical API. Never should the actor be modified directly, it must always mirror the business object through observation. This way, it becomes easy to switch the graphical implementation of the objects. We may change the whole scene graph implementation without touching the rest of the application.

The UI layer sits above the rest of the application, no module should access the UI. The UI must make sure also that it does not change its own state, which is very common if no special care is taken. E.g., if a button that when pressed automatically changes its image and the button represents a state of a business object, when the state of the object is changed by any other means---be it collaboration, animation, task scheduling and so on---the button will be left in an inconsistent state. Therefore, the developers must make sure that these automatic feedbacks that usual UI frameworks have by default are not enabled. 

\section{Distributed Visualization and Collaboration Module}
\label{cha:DVCM}

\subsection{Domain Distribution}

We intend to seamlessly connect this module to our visualizer, hence transforming its visualization and workflow with only small changes in configuration files. Nevertheless switching from local to distributed may create a high risk for inconsistency if the application is not properly designed. This inconsistency can be noticed when the application state is spread across multiple elements and layers, hence making complicated for one instance of the application to propagate its state consistently to another. However, in our designed visualizer, we concentrate all the application state in a single element---the \emph{domain}---thus alleviating our effort designing distribution. Still, in order to provide seamless integration with the application, this module must distribute the application domain in an agnostic and non-intrusive way.

We followed the same approach as with the DMF design---explained in Section \ref{sec:interactions}---, we created a separate module with its own internal structures that serve only for the module purposes, the actual application domain data is controlled by the application. Hence the module \emph{observes} and \emph{applies} state changes to the application domain, for that purpose it has a dependency to the DMF. Therefore, our domain propagation strategy is clearly a mere distribution of the DMF notification calls, and persisted through the DMF domain control functionality.  

\subsection{Collaboration}

In our shared domain scheme, whenever multiple users are making a collaborative visualization, they can simultaneously change the same data, therefore collisions are prone to happen. In online gaming realm, collisions are a very difficult problem to handle because there can be a massive amount of players interacting with the same data at the same time. Moreover there is usually no action of ``take control of an entity" before altering it in any way, as the players are interacting with a simulated world, and requiring such action may break the immersion. Therefore, the collision problem needs to be handled very fast and smoothly so that all the players involved in the collision feel that the object that triggered the collision is being really shared. Such steep requirements are not the case in the scientific visualizer.

Entities in a scientific visualizer domain are not to be treated as objects in a game. It is possible to apply a system where a user must first take control of an entity in order to apply any changes to it. Such system eliminates the problem of users sharing an object. Still, even if there is no control system, the latency when solving a collision does not need to be so low as in games. Therefore, it is acceptable to have an arbiter node responsible to keep the consistency, and every change applied by every node must be verified by the arbiter first, and only the arbiter and other designated nodes by the arbiter can propagate the changes (see Figure \ref{fig:arbiter_topology}).

\begin{figure}[!t]
\centering
\includegraphics[width=2.5in]{arbiter_topology.pdf}
\caption{Arbiter Topology}
\label{fig:arbiter_topology}
\end{figure}

\subsection{Distributed Visualization}
\label{sec:dv}

Considering that all the domain data is properly distributed as explained in the previous section, achieving distributed visualization becomes a simpler issue. We need to calculate the views for every display output based on a single observer and synchronize the frame displaying. These tasks must be assigned somehow to nodes in the visualization cluster, hence we need to create roles with appropriate responsibilities for the nodes. In order to accomplish distributed visualization in our design, it is imperative that we assign at least master and slave roles. 

The frustum calculation is done based on the output screens. Every node has a number of outputs connected to it and they need to display the current view of the scene on those. The view of the scene is based on a single observer---in a typical scenario---and therefore, this view should be split between the nodes. The master node needs to know the dimensions of the multiple-display environment and slave nodes disposition.

The frame synchronization is very simple. The master node must first command every node to render---with the appropriate frusta transferred. Then, after \emph{every} slave node has finished rendering, the master node commands the slaves to present their rendered frames. If this synchronization strategy is not used, the user immersion can be broken, as there will be different frames being exhibited at the same time.

\subsection{Integrating with the Visualizer}

The Distributed Visualization and Collaboration Module---\emph{DVCM}--- should be a generic DMF observer that propagates local changes to the network and network changes locally. In order to use the module, when assembling the visualizer's components through a startup script, there must be a list of available options in the \emph{DVCM} interface in order to assign its role. These options must cover a series of supported roles: local domain updates to be propagated throughout the network, incoming domain changes are going to be analyzed for collisions and confirmed or simply accepted, passively or actively render and display on the screen, among others. All of these possible setups are going to be covered and shown implementation-wise in the following section.

\section{Results}
\label{sec:results}
We show in this section how the implemented example system behaves under local and distributed visualization. Our goal is to measure how much the distribution and frame display checkpoint impacts the performance. Therefore we registered the frame rates of the visualization while varying the number of nodes in the cluster and which nodes used. As mentioned before, the example system has no multicast implemented, thus all the tests have been executed with only unicast communication between the nodes. We used 4 nodes with similar configuration in our tests:

Our tests are consisted of the visualization of a scene with 1,300,000 triangles with all culling disabled. This scene is just a collection of triangles in a certain position, but emulates a typical scene in our application usage scenario. During this visualization, we recorded the frames per second while the camera followed a predefined path. We executed these tests for a series of combination of nodes as shown in table.

\begin{table*}[!t]
\renewcommand{\arraystretch}{1.3}
\centering
  \begin{tabular}{ | l | c | c |  c | c | c | c | c | c | }
    \hline
   Group & A & B & C & D & A, B & A,B,C & A,C,D & A,B,C,D \\ \hline
	 Average FPS & 430 & 416 & 220 & 216 & 356 & 199 & 195 & 191 \\ \hline
FPS loss & --- & --- & --- & --- & 14.4\% & 11.3\% & 11.5\% & 11.4\% \\ \hline
  \end{tabular}
\caption{Frame rates and performance comparison of different cluster configurations}
\label{tab:fps}
\end{table*}

In table \ref{tab:fps}, we show in the "Average FPS`` row how many frames per second each setup achieved and in the "FPS loss`` row the loss of performance of the given node group when compared to the performance of the worst local node performance of the group. Notice that the first 4 scenarios are of the nodes executing the tests locally. The results show that there is a performance loss of less than 15\% with networking due to latency of the network in any of the cases. Moreover, considering that nodes C and D have a similar performance locally and since a group performance limit is equal the weakest node's local performance, the comparison of performance between groups A,B,C and A,B,C,D is important to show us the raw impact of adding another node to a group, which is a negligible loss of less than 1\%.

As can be seen, although the performance suffers an expected loss with distribution, our system can easily provide interactive frame rates during the visualization of large scenes in a distributed visualization scenario \ref{tab:fps}, which is the usual desired cave disposition and our ultimate goal with the project.

We tested the system in the same 4 node scenario with a collaborative scene following an arbiter topology. The tests successfully worked and there had been no inconsistencies or noticeable performance hindrances as expected.

\section{Conclusion}
\label{cha:conclusion}

In this work, we presented our design of an extensible and modular visualizer as well as the design of a module that provides distributed visualization and collaboration for the visualizer. By following these designs, we implemented an example system and tested the synchronism of the distributed visualization and the consistency of the collaboration among multiple nodes, we also evaluated the impact on performance caused by the distributed visualization.

We discussed first the relevant concepts and problems of designing and implementing a real time immersive visualizer, from which we extracted our main architectural requirements---modularity and extensibility. Following we presented our design for the visualizer inspired by MVC architecture and explained how we try to fulfill the aforementioned requirements. We continued by presenting the design of the module responsible for distributed visualization and collaboration, its interactions with the visualizer and why our described design for the visualizer simplifies its implementation and usage. We concluded with results of the implemented example system.

We expected to indicate how our MVC-inspired design made possible the development of a module that transparently provided distributed visualization and collaboration to a visualizer. Also, how the design enabled the substitution of parts of the system easily. E.g., the scene graph implementation can be switched between a very efficient and licensed per station library for displaying a scene in massive immersive environments and a cheaper licensed library for common desktop usage. These reasons along with others explained in section \ref{intro} are what motivated this work as a solution for our real project. We try to summarize here the key points that we believe we have addressed with our design and can impact overall productivity of the development of real time and efficiency-focused applications.

We addressed complexity with the separation of a system into modules with explicit interfaces, which tend to isolate the low level details of implemented features and create abstractions, which enhance the productivity of the business logic development by reducing the complexity of the system. The isolation of all the distribution code inside our designed module leverages the productivity of the business and graphical developers in our visualizer.

We provided easier prototyping for a product with a long development cycle, which can be extremely complex. Therefore, designing and implementing the system iteratively can leverage productivity and also accommodate late changes in product requirements. By making the application extensible, we give the developers an easy way to prototype new functionality and implementations. Our component based design enabled our system to be distributed after its first version without any significant modifications to it.

We enhanced the flexibility of the system with the possibility of switching between different functionality with no code change, which is important for a software that must assume different roles depending on the scenario---e.g. different scene graphs, distribution schemes and so on. 

Our example system implementation showed that our design worked for the expect scenario. The synchronism among multiple output displays was achieved without any dedicated hardware and also without even the need to use broadcast messages. A loss of less than 15\% for a 4 node setup, which is our common CAVE scenario has been a very good result, confirming our design as a proper solution. Furthermore, the consistency among the scenes when using the software collaboratively was achieved without any noticeable performance hindrances. 

\section{Future Work}
Our distributed visualization strategy is not suitable for scenarios where there is no single node in the cluster with a number of output displays that creates a rendering bottleneck on it, which happens due to the amount of graphical processing on it. However, if desired, the task of rendering can be separated for the task of displaying. By creating this new layer of parallelism, the rendering task can be distributed equally among the nodes independently of the number of output displays connected to each one of them. However, it becomes necessary to recompose the final image based on a given recompositing strategy, which can adds complexity and inefficiency if not necessary.

The consistency among multiple stations when working collaboratively is currently only designed for reliable environments without simultaneous edition of the same entity. Therefore no special collision treatment of time synchronizing strategy is currently needed. However, if the visualizer needs to be deployed in a slow and non reliable network, with dynamic entity behavior, simultaneous edition of the same entity, and so on, the distribution module can be extended to support prediction algorithms, time synchronizing strategies and dynamic behavior descriptions.

% An example of a floating figure using the graphicx package.
% Note that \label must occur AFTER (or within) \caption.
% For figures, \caption should occur after the \includegraphics.
% Note that IEEEtran v1.7 and later has special internal code that
% is designed to preserve the operation of \label within \caption
% even when the captionsoff option is in effect. However, because
% of issues like this, it may be the safest practice to put all your
% \label just after \caption rather than within \caption{}.
%
% Reminder: the "draftcls" or "draftclsnofoot", not "draft", class
% option should be used if it is desired that the figures are to be
% displayed while in draft mode.
%
%\begin{figure}[!t]
%\centering
%\includegraphics[width=2.5in]{myfigure}
% where an .eps filename suffix will be assumed under latex, 
% and a .pdf suffix will be assumed for pdflatex; or what has been declared
% via \DeclareGraphicsExtensions.
%\caption{Simulation Results}
%\label{fig_sim}
%\end{figure}

% Note that IEEE typically puts floats only at the top, even when this
% results in a large percentage of a column being occupied by floats.


% An example of a double column floating figure using two subfigures.
% (The subfig.sty package must be loaded for this to work.)
% The subfigure \label commands are set within each subfloat command, the
% \label for the overall figure must come after \caption.
% \hfil must be used as a separator to get equal spacing.
% The subfigure.sty package works much the same way, except \subfigure is
% used instead of \subfloat.
%
%\begin{figure*}[!t]
%\centerline{\subfloat[Case I]\includegraphics[width=2.5in]{subfigcase1}%
%\label{fig_first_case}}
%\hfil
%\subfloat[Case II]{\includegraphics[width=2.5in]{subfigcase2}%
%\label{fig_second_case}}}
%\caption{Simulation results}
%\label{fig_sim}
%\end{figure*}
%
% Note that often IEEE papers with subfigures do not employ subfigure
% captions (using the optional argument to \subfloat), but instead will
% reference/describe all of them (a), (b), etc., within the main caption.


% An example of a floating table. Note that, for IEEE style tables, the 
% \caption command should come BEFORE the table. Table text will default to
% \footnotesize as IEEE normally uses this smaller font for tables.
% The \label must come after \caption as always.
%
%\begin{table}[!t]
%% increase table row spacing, adjust to taste
%\renewcommand{\arraystretch}{1.3}
% if using array.sty, it might be a good idea to tweak the value of
% \extrarowheight as needed to properly center the text within the cells
%\caption{An Example of a Table}
%\label{table_example}
%\centering
%% Some packages, such as MDW tools, offer better commands for making tables
%% than the plain LaTeX2e tabular which is used here.
%\begin{tabular}{|c||c|}
%\hline
%One & Two\\
%\hline
%Three & Four\\
%\hline
%\end{tabular}
%\end{table}


% Note that IEEE does not put floats in the very first column - or typically
% anywhere on the first page for that matter. Also, in-text middle ("here")
% positioning is not used. Most IEEE journals/conferences use top floats
% exclusively. Note that, LaTeX2e, unlike IEEE journals/conferences, places
% footnotes above bottom floats. This can be corrected via the \fnbelowfloat
% command of the stfloats package.


% conference papers do not normally have an appendix


% use section* for acknowledgement
\section*{Acknowledgment}


This research project is supported by Petrobras. Alberto Raposo (process \#310607/2013-2) thanks CNPq for the support to his research work.





% trigger a \newpage just before the given reference
% number - used to balance the columns on the last page
% adjust value as needed - may need to be readjusted if
% the document is modified later
%\IEEEtriggeratref{8}
% The "triggered" command can be changed if desired:
%\IEEEtriggercmd{\enlargethispage{-5in}}

% references section

% can use a bibliography generated by BibTeX as a .bbl file
% BibTeX documentation can be easily obtained at:
% http://www.ctan.org/tex-archive/biblio/bibtex/contrib/doc/
% The IEEEtran BibTeX style support page is at:
% http://www.michaelshell.org/tex/ieeetran/bibtex/
%\bibliographystyle{IEEEtran}
% argument is your BibTeX string definitions and bibliography database(s)
%\bibliography{IEEEabrv,../bib/paper}
%
% <OR> manually copy in the resultant .bbl file
% set second argument of \begin to the number of references
% (used to reserve space for the reference number labels box)
\bibliographystyle{IEEEtran}
% argument is your BibTeX string definitions and bibliography database(s)
\bibliography{bibli}




% that's all folks
\end{document}


