veil_shmem.c

Go to the documentation of this file.
00001 /**
00002  * @file   veil_shmem.c
00003  * \code
00004  *     Author:       Marc Munro
00005  *     Copyright (c) 2005 - 2010 Marc Munro
00006  *     License:      BSD
00007  * $Id: veil_shmem.c,v 1.8 2008/07/05 00:32:26 bloodnok Exp $
00008  * \endcode
00009  * @brief  
00010  * Functions for dealing with veil shared memory.
00011  *
00012  * This provides dynamic memory allocation, like malloc, from chunks of
00013  * shared memory allocated from the Postgres shared memory pool.  In
00014  * order to be able to reset and reload shared memory structures while
00015  * other backends continue to use the existing structures, a shared
00016  * memory reset creates a new context, or switches to an existing one
00017  * that is no longer in use.  No more than two separate contexts will be
00018  * created.
00019  *
00020  * Each context of veil shared memory is associated with a shared hash,
00021  * which is used to store veil's shared variables.  A specially named
00022  * variable, VEIL_SHMEMCTL appears only in context0 and contains a
00023  * reference to chunk0, and the ShmemCtl structure.  From this structure
00024  * we can identify the current context, the initial chunks for each
00025  * active context, and whether a context switch is in progress. 
00026  * 
00027  * A context switch takes place in 3 steps:
00028  * -  preparation, in which we determine if a context switch is allowed,
00029  *    initialise the new context and record the fact that we are in the
00030  *    process of switching.  All subsequent operations in the current
00031  *    backend will work in the new context, while other backends will
00032  *    continue to use the original context
00033  * -  initialisation of the new context, variables, etc.  This is done
00034  *    by the user-space function veil_init().
00035  * -  switchover, when all other processes gain access to the newly
00036  *    initialised context.  They may continue to use the previous
00037  *    context for the duration of their current transactions.
00038  *
00039   \endverbatim
00040  * To access shared variable "x" in a new session, the following steps
00041  * are taken:
00042  *  - We access the hash "VEIL_SHARED1_nnn" (where nnn is the oid of our
00043  *    database).  This gives us a reference to the ShmemCtl structure.
00044  *    We record hash0 and shared_meminfo on the way.
00045  *  - We access ShemCtl to identify the current hash and current
00046  *    context. 
00047  *  - We look up variable "x" in the current hash, and if we have to
00048  *    allocate space for it, allocate it from the current context.
00049  *
00050  * Note that We use a dynamicall allocated LWLock, VeilLWLock to protect
00051  * our shared control structures.
00052  * 
00053  */
00054 
00055 #include "postgres.h"
00056 #include "utils/hsearch.h"
00057 #include "storage/pg_shmem.h"
00058 #include "storage/shmem.h"
00059 #include "storage/lwlock.h"
00060 #include "access/xact.h"
00061 #include "miscadmin.h"
00062 #include "veil_version.h"
00063 #include "veil_shmem.h"
00064 #include "veil_funcs.h"
00065 
00066 #if PG_VERSION_GE(8, 2)
00067 
00068 /**
00069  * shared_meminfo provides access to the ShmemCtl structure allocated in
00070  * context 0.
00071  */
00072 static ShmemCtl *shared_meminfo = NULL;
00073 
00074 /**
00075  * Whether the current backend is in the process of switching contexts.
00076  * If so, it will be setting up the non-current context in readiness for
00077  * making it available to all other backends.
00078  */
00079 static bool      prepared_for_switch = false;
00080 
00081 /**
00082  * The LWLock that Veil will use for managing concurrent access to
00083  * shared memory.  It is initialised to a lock id that is distinct
00084  * from any tha twill be dynamically allocated.
00085  */
00086 static LWLockId  VeilLWLock = AddinShmemInitLock;
00087 
00088 /**
00089  * The LWLock to be used while initially setting up shared memory and 
00090  * allocating a veil database-specific LWLock.
00091  */
00092 static LWLockId  InitialLWLock = AddinShmemInitLock;
00093 
00094 /** 
00095  * Return the index of the other context from the one supplied.
00096  * 
00097  * @param x the context for which we want the other one.
00098  * 
00099  * @return the opposite context to that of x.
00100  */
00101 #define OTHER_CONTEXT(x)    (x ? 0: 1)
00102 
00103 /**
00104  * Whether Veil has registered for addin shared memory or not.  If not,
00105  * Veil will steal shared memory from postgres with the subsequent risk
00106  * of shared memory exhaustion.  
00107  */
00108 static bool using_registered_shmem = false;
00109 
00110 
00111 /** 
00112  * Veil's startup function.  This should be run when the Veil shared
00113  * library is loaded by postgres.
00114  * 
00115  * If shared_preload_libraries is not defined, Veil may still be run but
00116  * it will steal shared memory from postgres, potentially exhausting it.
00117  * 
00118  * This function is not defined if VEIL_TRIAL is set.  This allows a
00119  * veil_trail shared object to be created separately from the normal
00120  * veil shared object, which may be used without defining
00121  * shared_preload_libraries.
00122  */
00123 #ifndef VEIL_TRIAL
00124 void
00125 _PG_init()
00126 {
00127     int veil_dbs;
00128 
00129     // Define GUCs for veil
00130     veil_config_init(); 
00131     veil_dbs = veil_dbs_in_cluster();
00132 
00133     // Request a Veil-specific shared memory context
00134     RequestAddinShmemSpace(2 * veil_shmem_context_size() * veil_dbs);
00135 
00136     // Request a LWLock for later use by all backends
00137     RequestAddinLWLocks(veil_dbs);
00138 }
00139 #endif
00140 
00141 /** 
00142  * Create/attach to the shared hash identified by hashname.  Return a
00143  * pointer to an HTAB that references the shared hash.  All locking is
00144  * handled by the caller.
00145  * 
00146  * @param hashname 
00147  * 
00148  * @return Pointer to HTAB referencing the shared hash.
00149  */
00150 static HTAB *
00151 create_shared_hash(const char *hashname)
00152 {
00153     HASHCTL  hashctl;
00154     HTAB    *result;
00155     char    *db_hashname;
00156     int      hash_elems = veil_shared_hash_elems();
00157 
00158     // Add the current database oid into the hashname so that it is
00159     // distinct from the shared hash for other databases in the cluster.
00160     db_hashname = (char *) vl_malloc(HASH_KEYLEN);
00161     (void) snprintf(db_hashname, HASH_KEYLEN - 1, "%s_%u", 
00162                     hashname, MyDatabaseId);
00163     hashctl.keysize = HASH_KEYLEN;
00164     hashctl.entrysize = sizeof(VarEntry);
00165 
00166     result = ShmemInitHash(db_hashname, hash_elems,
00167                            hash_elems, &hashctl, HASH_ELEM);
00168     pfree(db_hashname);
00169     return result;
00170 }
00171 
00172 /** 
00173  * Return reference to the HTAB for the shared hash associated with
00174  * context 0.
00175  * 
00176  * @return Pointer to HTAB referencing shared hash for context 0.
00177  */
00178 static HTAB *
00179 get_hash0()
00180 {
00181     static HTAB *hash0 = NULL;
00182 
00183     if (!hash0) {
00184         hash0 = create_shared_hash("VEIL_SHARED1");
00185     }
00186     return hash0;
00187 }
00188 
00189 /** 
00190  * Return reference to the HTAB for the shared hash associated with
00191  * context 1.
00192  * 
00193  * @return Pointer to HTAB referencing shared hash for context 1.
00194  */
00195 static HTAB *
00196 get_hash1()
00197 {
00198     static HTAB *hash1 = NULL;
00199 
00200     if (!hash1) {
00201         hash1 = create_shared_hash("VEIL_SHARED2");
00202     }
00203 
00204     return hash1;
00205 }
00206 
00207 
00208 /** 
00209  * Allocate or attach to, a new chunk of shared memory for a named
00210  * memory context.
00211  * 
00212  * @param name The name
00213  * @param p_found Pointer to boolean that will identify whether this
00214  * chunk has already been initialised.
00215  * 
00216  * @return Pointer to chunk of shared memory.
00217  */
00218 static MemContext *
00219 get_shmem_context(char   *name,
00220                   size_t  size,
00221                   bool   *p_found)
00222 {
00223     int         i;
00224     MemContext *context;
00225     char       *uniqname  = (char *) vl_malloc(strlen(name) + 16);
00226     int         max_dbs = veil_dbs_in_cluster();
00227 
00228     for (i = 0; i < max_dbs; i++) {
00229         (void) sprintf(uniqname, "%s_%d", name, i);
00230         context = ShmemInitStruct(uniqname, size, p_found);;
00231         if (!context) {
00232             ereport(ERROR,
00233                     (errcode(ERRCODE_INTERNAL_ERROR),
00234                      errmsg("veil: cannot allocate shared memory(1)")));
00235         }
00236 
00237         if (*p_found) {
00238             // Already exists.  Check database id. 
00239             if (context->db_id == MyDatabaseId) {
00240                 /* This context is the one for the current database, 
00241                  * nothing else to do. */
00242                 return context;
00243             }
00244         }
00245         else {
00246             // We Just allocated our first context
00247             context->db_id = MyDatabaseId;
00248             context->next = sizeof(MemContext);
00249             context->limit = size;
00250             context->lwlock = VeilLWLock;
00251             return context;
00252         }
00253     }
00254 
00255     // We reach this point if no existing contexts are allocated to our
00256     // database.  Now we check those existing contexts to see whether
00257     // they are still in use.  If not, we will redeploy them.
00258 
00259     for (i = 0; i < max_dbs; i++) {
00260         (void) sprintf(uniqname, "%s_%d", name, i);
00261         context = ShmemInitStruct(uniqname, size, p_found);;
00262 
00263         if (!context) {
00264             ereport(ERROR,
00265                     (errcode(ERRCODE_INTERNAL_ERROR),
00266                      errmsg("veil: cannot allocate shared memory(2)")));
00267         }
00268 
00269         if (*p_found) {
00270             // Is this context for a still existant database?
00271             if (!vl_db_exists(context->db_id)) {
00272                 // We can re-use this context.
00273                 context->db_id = MyDatabaseId;
00274                 context->next = sizeof(MemContext);
00275                 context->limit = size;
00276 
00277                 *p_found = false;  // Tell the caller that init is required
00278                 return context;
00279             }
00280         }
00281         else {
00282             // We didn't find an unused context, so now we have created 
00283             // a new one.
00284 
00285             context->db_id = MyDatabaseId;
00286             context->next = sizeof(MemContext);
00287             context->limit = size;
00288             return context;
00289         }
00290     }
00291     ereport(ERROR,
00292             (errcode(ERRCODE_INTERNAL_ERROR),
00293              errmsg("veil: no more shared memory contexts allowed")));
00294 }
00295 
00296 // Forward ref, required by next function.
00297 static void shmalloc_init();
00298 
00299 /** 
00300  * Return the id (index) of the current context for this session 
00301  * 
00302  * @return The current context id
00303  */
00304 static int
00305 get_cur_context_id()
00306 {
00307     static initialised = false;
00308     int    context;
00309 
00310     if (!initialised) {
00311         shmalloc_init();
00312         initialised = true;
00313     }
00314         
00315     context = shared_meminfo->current_context;
00316     if (prepared_for_switch) {
00317         context = OTHER_CONTEXT(context);
00318     }
00319     else {
00320         // Check for the default context being for a later transaction
00321         // than current and, if so, use the other one.
00322         if (TransactionIdPrecedes(GetCurrentTransactionId(), 
00323                                   shared_meminfo->xid[context]))
00324         {
00325             context = OTHER_CONTEXT(context);
00326         }
00327     }
00328 
00329     return context;
00330 }
00331 
00332 /** 
00333  * Return pointer to shared memory allocated for the current context.
00334  * 
00335  * @return The current context. 
00336  */
00337 static MemContext *
00338 get_cur_context()
00339 {
00340     int context;
00341     context = get_cur_context_id();
00342     return shared_meminfo->context[context];
00343 }
00344 
00345 /** 
00346  * Dynamically allocate a piece of shared memory from the current
00347  * context, doing no locking.
00348  * 
00349  * @param context The context in which we are operating
00350  * @param size The size of the requested piece of memory.
00351  * 
00352  * @return Pointer to dynamically allocated memory.
00353  */
00354 void *
00355 do_vl_shmalloc(MemContext *context,
00356                size_t size)
00357 {
00358     void *result;
00359     size_t amount = (size_t) MAXALIGN(size);
00360 
00361     if ((amount + context->next) <= context->limit) {
00362         result = (void *) context + context->next;
00363         context->next += amount;
00364     }
00365     else {
00366         ereport(ERROR,
00367                 (ERROR,
00368                  (errcode(ERRCODE_INTERNAL_ERROR),
00369                   errmsg("veil: out of shared memory"))));
00370     }
00371     return result;
00372 }
00373 
00374 /** 
00375  * Dynamically allocate a piece of shared memory from the current context. 
00376  * 
00377  * @param size The size of the requested piece of memory.
00378  * 
00379  * @return Pointer to dynamically allocated memory.
00380  */
00381 void *
00382 vl_shmalloc(size_t size)
00383 {
00384     MemContext *context;
00385     void       *result;
00386 
00387     context = get_cur_context();
00388 
00389     LWLockAcquire(VeilLWLock, LW_EXCLUSIVE);
00390     result = do_vl_shmalloc(context, size);
00391     LWLockRelease(VeilLWLock);
00392 
00393     return result;
00394 }
00395 
00396 /** 
00397  * Free a piece of shared memory within the current context.  Currently
00398  * this does nothing as implementation of freeing of shared memory has
00399  * been deferred.
00400  * 
00401  * @param mem Pointer to the memory to be freed.
00402  * 
00403  */
00404 void
00405 vl_free(void *mem)
00406 {
00407     return;
00408 }
00409 
00410 
00411 /** 
00412  * Attach to, creating and initialising as necessary, the shared memory
00413  * control structure.  Record this for the session in shared_meminfo.
00414  */
00415 static void
00416 shmalloc_init()
00417 {
00418     if (!shared_meminfo) {
00419         VarEntry   *var;
00420         MemContext *context0;
00421         MemContext *context1;
00422         bool        found = false;
00423         size_t      allocated;
00424         HTAB       *hash0;
00425         HTAB       *hash1;
00426         size_t      size;
00427 
00428         size = veil_shmem_context_size();
00429 
00430         LWLockAcquire(InitialLWLock, LW_EXCLUSIVE);
00431         context0 = get_shmem_context("VEIL_SHMEM0", size, &found);
00432 
00433         if (found) {
00434             shared_meminfo = context0->memctl;
00435             VeilLWLock = shared_meminfo->veil_lwlock;
00436             // By aquiring and releasing this lock, we ensure that Veil
00437             // shared memory has been fully initialised, by a process
00438             // following the else clause of this code path.
00439             LWLockAcquire(VeilLWLock, LW_EXCLUSIVE);
00440             LWLockRelease(InitialLWLock);
00441             LWLockRelease(VeilLWLock);
00442         }
00443         else {
00444             // Do minimum amount of initialisation while holding
00445             // the initial lock.  We don't want to do anything that
00446             // may cause other locks to be aquired as this could lead
00447             // to deadlock with other add-ins.  Instead, we aquire the
00448             // Veil-specific lock before finishing the initialisation.
00449 
00450             shared_meminfo = do_vl_shmalloc(context0, sizeof(ShmemCtl));
00451 
00452             if (context0->lwlock != InitialLWLock) {
00453                 // Re-use the LWLock previously allocated to this memory 
00454                 // context
00455                 VeilLWLock = context0->lwlock;
00456             }
00457             else {
00458                  // Allocate new LWLock for this new shared memory context
00459                 VeilLWLock = LWLockAssign(); 
00460             }
00461             // Record the lock id in context0 (for possible re-use if
00462             // the current database is dropped and a new veil-using
00463             // database created), and in the shared_meminfo struct
00464             context0->lwlock = VeilLWLock;
00465             shared_meminfo->veil_lwlock = VeilLWLock;
00466             
00467             // Exchange the initial lock for our Veil-specific one.
00468             LWLockAcquire(VeilLWLock, LW_EXCLUSIVE);
00469             LWLockRelease(InitialLWLock);
00470     
00471             // Now do the rest of the Veil shared memory initialisation
00472 
00473             // Set up the other memory context
00474             context1 = get_shmem_context("VEIL_SHMEM1", size, &found);
00475             
00476             // Record location of shmemctl structure in each context
00477             context0->memctl = shared_meminfo;
00478             context1->memctl = shared_meminfo;
00479 
00480             // Finish initialising the shmemctl structure
00481             shared_meminfo->type = OBJ_SHMEMCTL;
00482             shared_meminfo->current_context = 0;
00483             shared_meminfo->total_allocated[0] = size;
00484             shared_meminfo->total_allocated[1] = size;
00485             shared_meminfo->switching = false;
00486             shared_meminfo->context[0] = context0;
00487             shared_meminfo->context[1] = context1;
00488             shared_meminfo->xid[0] = GetCurrentTransactionId();
00489             shared_meminfo->xid[1] = shared_meminfo->xid[0];
00490             shared_meminfo->initialised = true;
00491 
00492             // Set up both shared hashes
00493             hash0 = get_hash0();
00494             hash1 = get_hash1();
00495 
00496             // Record the shmemctl structure in hash0
00497             var = (VarEntry *) hash_search(hash0, (void *) "VEIL_SHMEMCTL",
00498                                            HASH_ENTER, &found);
00499             // elog(WARNING, "SHMEMCTL FOUND - 1 = %d", found);
00500 
00501             var->obj = (Object *) shared_meminfo;
00502             var->shared = true;
00503 
00504             var = (VarEntry *) hash_search(hash0, (void *) "VEIL_SHMEMCTL",
00505                                            HASH_ENTER, &found);
00506             // elog(WARNING, "SHMEMCTL FOUND - 2 = %d", found);
00507 
00508             LWLockRelease(VeilLWLock);
00509         }
00510     }
00511 }
00512 
00513 /** 
00514  * Return the shared hash for the current context.
00515  * 
00516  * @return Pointer to the HTAB for the current context's shared hash.
00517  */
00518 HTAB *
00519 vl_get_shared_hash()
00520 {
00521     int context;
00522     HTAB *hash;
00523     static initialised = false;
00524 
00525     if (!initialised) {
00526         (void) get_cur_context();  // Ensure shared memory is set up.
00527         initialised = true;
00528     }
00529 
00530     context = get_cur_context_id();
00531 
00532     if (context == 0) {
00533         hash = get_hash0();
00534     }
00535     else {
00536         hash = get_hash1();
00537     }
00538     
00539     return hash;
00540 }
00541 
00542 /** 
00543  * Reset one of the shared hashes.  This is one of the final steps in a
00544  * context switch.
00545  * 
00546  * @return hash The shared hash that is to be reset.
00547  */
00548 static void
00549 clear_hash(HTAB *hash)
00550 {
00551     static HASH_SEQ_STATUS status;
00552     VarEntry *var;
00553 
00554     hash_seq_init(&status, hash);
00555     while (var = hash_seq_search(&status)) {
00556         if (strncmp("VEIL_SHMEMCTL", var->key, strlen("VEIL_SHMEMCTL")) != 0) {
00557             (void) hash_search(hash, var->key, HASH_REMOVE, NULL);
00558         }
00559     }
00560 }
00561 
00562 /** 
00563  * Prepare for a switch to the alternate context.  Switching will
00564  * only be allowed if there are no transactions that may still be using
00565  * the context to which we are switching, and there is no other
00566  * process attempting the switch.
00567  * 
00568  * @return true if the switch preparation was successful.
00569  */
00570 bool
00571 vl_prepare_context_switch()
00572 {
00573     int   context_curidx;
00574     int   context_newidx;
00575     HTAB *hash0 = get_hash0(); // We must not attempt to create hashes on
00576     HTAB *hash1 = get_hash1(); // the fly below as they also acquire the lock
00577     TransactionId oldest_xid;
00578     MemContext *context;
00579 
00580     (void) get_cur_context();  // Ensure shared memory is set up
00581 
00582     LWLockAcquire(VeilLWLock, LW_EXCLUSIVE);
00583 
00584     if (shared_meminfo->switching) {
00585         // Another process is performing the switch
00586         LWLockRelease(VeilLWLock);
00587         return false;
00588     }
00589 
00590     shared_meminfo->switching = true;
00591 
00592     // We have claimed the switch.  If we decide that we cannot proceed,
00593     // we will return it to its previous state.
00594 
00595     context_curidx = shared_meminfo->current_context;
00596     context_newidx = OTHER_CONTEXT(context_curidx);
00597 
00598     // In case the alternate context has been used before, we must
00599     // clear it.
00600 
00601     oldest_xid = GetOldestXmin(false);
00602     if (TransactionIdPrecedes(oldest_xid, 
00603                               shared_meminfo->xid[context_curidx])) 
00604     {
00605         // There is a transaction running that precedes the time of
00606         // the last context switch.  That transaction may still be
00607         // using the chunk to which we wish to switch.  We cannot
00608         // allow the switch.
00609         shared_meminfo->switching = false;
00610         LWLockRelease(VeilLWLock);
00611         return false;
00612     }
00613     else {
00614         // It looks like we can safely make the switch.  Reset the
00615         // new context, and make it the current context for this
00616         // session only.
00617         context = shared_meminfo->context[context_newidx];
00618         context->next = sizeof(MemContext);
00619 
00620         // If we are switching to context 0, reset the next field of
00621         // the first chunk to leave space for the ShmemCtl struct.
00622         if (context_newidx == 0) {
00623             context->next += sizeof(ShmemCtl);
00624             clear_hash(hash0);
00625         }
00626         else {
00627             clear_hash(hash1);
00628         }
00629     }
00630 
00631     LWLockRelease(VeilLWLock);
00632     prepared_for_switch = true;
00633     return true;
00634 }
00635 
00636 /** 
00637  * Complete the context switch started by vl_prepare_context_switch().
00638  * Raise an ERROR if the context switch cannot be completed.
00639  * 
00640  * @return true if the context switch is successfully completed.
00641  */
00642 bool
00643 vl_complete_context_switch()
00644 {
00645     int  context_curidx;
00646     int  context_newidx;
00647 
00648     if (!prepared_for_switch) {
00649         ereport(ERROR,
00650                 (errcode(ERRCODE_INTERNAL_ERROR),
00651                  errmsg("failed to complete context switch"),
00652                  errdetail("Not prepared for switch - "
00653                            "invalid state for operation")));
00654     }
00655 
00656     LWLockAcquire(VeilLWLock, LW_EXCLUSIVE);
00657     context_curidx = shared_meminfo->current_context;
00658     context_newidx = OTHER_CONTEXT(context_curidx);
00659 
00660     if (!shared_meminfo->switching) {
00661         // We do not claim to be switching.  We should.
00662         LWLockRelease(VeilLWLock);
00663 
00664         ereport(ERROR,
00665                 (errcode(ERRCODE_INTERNAL_ERROR),
00666                  errmsg("failed to complete context switch"),
00667                  errdetail("Session does not have switching set to true- "
00668                            "invalid state for operation")));
00669     }
00670 
00671     shared_meminfo->switching = false;
00672     shared_meminfo->current_context = context_newidx;
00673     shared_meminfo->xid[context_newidx] = GetCurrentTransactionId();
00674     LWLockRelease(VeilLWLock);
00675     prepared_for_switch = false;
00676     return true;
00677 }
00678 
00679 /** 
00680  * In desparation, if we are unable to complete a context switch, we
00681  * should use this function.
00682  */
00683 void
00684 vl_force_context_switch()
00685 {
00686     int  context_curidx;
00687     int  context_newidx;
00688     MemContext *context;
00689     HTAB *hash0 = get_hash0();
00690     HTAB *hash1 = get_hash1();
00691 
00692     (void) get_cur_context();
00693 
00694     LWLockAcquire(VeilLWLock, LW_EXCLUSIVE);
00695 
00696     context_curidx = shared_meminfo->current_context;
00697     context_newidx = OTHER_CONTEXT(context_curidx);
00698 
00699     // Clear the alternate context.
00700 
00701     context = shared_meminfo->context[context_newidx];
00702     context->next = sizeof(MemContext);
00703     
00704     // If we are switching to context 0, reset the next field of
00705     // the first chunk to leave space for the ShmemCtl struct.
00706     if (context_newidx == 0) {
00707         context->next += sizeof(ShmemCtl);
00708         clear_hash(hash0);
00709     }
00710     else {
00711         clear_hash(hash1);
00712     }
00713     
00714     shared_meminfo->switching = false;
00715     shared_meminfo->current_context = context_newidx;
00716     shared_meminfo->xid[context_newidx] = GetCurrentTransactionId();
00717     shared_meminfo->xid[0] = GetCurrentTransactionId();
00718     LWLockRelease(VeilLWLock);
00719     prepared_for_switch = false;
00720 }
00721 
00722 #else
00723 #include "veil_shmem_pre82.inc"
00724 
00725 #endif

Generated on Fri Mar 12 08:38:37 2010 for Veil by  doxygen 1.5.6