veil_shmem.c

Go to the documentation of this file.
00001 /**
00002  * @file   veil_shmem.c
00003  * \code
00004  *     Author:       Marc Munro
00005  *     Copyright (c) 2005, 2006 Marc Munro
00006  *     License:      BSD
00007  * $Id: veil_shmem.c,v 1.7 2007/07/31 22:18:27 bloodnok Exp $
00008  * \endcode
00009  * @brief  
00010  * Functions for dealing with veil shared memory.
00011  *
00012  * This provides dynamic memory allocation, like malloc, from chunks of
00013  * shared memory allocated from the Postgres shared memory pool.  In
00014  * order to be able to reset and reload shared memory structures while
00015  * other backends continue to use the existing structures, a shared
00016  * memory reset creates a new context, or switches to an existing one
00017  * that is no longer in use.  No more than two separate contexts will be
00018  * created.
00019  *
00020  * Each context of veil shared memory is associated with a shared hash,
00021  * which is used to store veil's shared variables.  A specially named
00022  * variable, VEIL_SHMEMCTL appears only in context0 and contains a
00023  * reference to chunk0, and the ShmemCtl structure.  From this structure
00024  * we can identify the current context, the initial chunks for each
00025  * active context, and whether a context switch is in progress. 
00026  * 
00027  * A context switch takes place in 3 steps:
00028  * -  preparation, in which we determine if a context switch is allowed,
00029  *    initialise the new context and record the fact that we are in the
00030  *    process of switching.  All subsequent operations in the current
00031  *    backend will work in the new context, while other backends will
00032  *    continue to use the original context
00033  * -  initialisation of the new context, variables, etc.  This is done
00034  *    by the user-space function veil_init().
00035  * -  switchover, when all other processes gain access to the newly
00036  *    initialised context.  They may continue to use the previous
00037  *    context for the duration of their current transactions.
00038  *
00039   \endverbatim
00040  * To access shared variable "x" in a new session, the following steps
00041  * are taken:
00042  *  - We access the hash "VEIL_SHARED1_nnn" (where nnn is the oid of our
00043  *    database).  This gives us a reference to the ShmemCtl structure.
00044  *    We record hash0 and shared_meminfo on the way.
00045  *  - We access ShemCtl to identify the current hash and current
00046  *    context. 
00047  *  - We look up variable "x" in the current hash, and if we have to
00048  *    allocate space for it, allocate it from the current context.
00049  *
00050  * Note that We use a dynamicall allocated LWLock, VeilLWLock to protect
00051  * our shared control structures.
00052  * 
00053  */
00054 
00055 #include "postgres.h"
00056 #include "utils/hsearch.h"
00057 #include "storage/pg_shmem.h"
00058 #include "storage/shmem.h"
00059 #include "storage/lwlock.h"
00060 #include "access/xact.h"
00061 #include "miscadmin.h"
00062 #include "veil_version.h"
00063 #include "veil_shmem.h"
00064 #include "veil_funcs.h"
00065 
00066 #if PG_VERSION_GE(8, 2)
00067 
00068 /**
00069  * shared_meminfo provides access to the ShmemCtl structure allocated in
00070  * context 0.
00071  */
00072 static ShmemCtl *shared_meminfo = NULL;
00073 
00074 /**
00075  * Whether the current backend is in the process of switching contexts.
00076  * If so, it will be setting up the non-current context in readiness for
00077  * making it available to all other backends.
00078  */
00079 static bool      prepared_for_switch = false;
00080 
00081 /**
00082  * The LWLock that Veil will use for managing concurrent access to
00083  * shared memory.  It is initialised to a lock id that is distinct
00084  * from any tha twill be dynamically allocated.
00085  */
00086 static LWLockId  VeilLWLock = AddinShmemInitLock;
00087 
00088 /**
00089  * The LWLock to be used while initially setting up shared memory and 
00090  * allocating a veil database-specific LWLock.
00091  */
00092 static LWLockId  InitialLWLock = AddinShmemInitLock;
00093 
00094 /** 
00095  * Return the index of the other context from the one supplied.
00096  * 
00097  * @param x the context for which we want the other one.
00098  * 
00099  * @return the opposite context to that of x.
00100  */
00101 #define OTHER_CONTEXT(x)    (x ? 0: 1)
00102 
00103 /**
00104  * Whether Veil has registered for addin shared memory or not.  If not,
00105  * Veil will steal shared memory from postgres with the subsequent risk
00106  * of shared memory exhaustion.  
00107  */
00108 static bool using_registered_shmem = false;
00109 
00110 
00111 /** 
00112  * Veil's startup function.  This should be run when the Veil shared
00113  * library is loaded by postgres.
00114  * 
00115  * If shared_preload_libraries is not defined, Veil may still be run but
00116  * it will steal shared memory from postgres, potentially exhausting it.
00117  * 
00118  * This function is not defined if VEIL_TRIAL is set.  This allows a
00119  * veil_trail shared object to be created separately from the normal
00120  * veil shared object, which may be used without defining
00121  * shared_preload_libraries.
00122  */
00123 #ifndef VEIL_TRIAL
00124 void
00125 _PG_init()
00126 {
00127     int veil_dbs = veil_dbs_in_cluster();
00128 
00129     // Define GUCs for veil
00130     veil_config_init(); 
00131 
00132     // Request a Veil-specific shared memory context
00133     RequestAddinShmemSpace(2 * veil_shmem_context_size() * veil_dbs);
00134 
00135     // Request a LWLock for later use by all backends
00136     RequestAddinLWLocks(veil_dbs);
00137 }
00138 #endif
00139 
00140 /** 
00141  * Create/attach to the shared hash identified by hashname.  Return a
00142  * pointer to an HTAB that references the shared hash.  All locking is
00143  * handled by the caller.
00144  * 
00145  * @param hashname 
00146  * 
00147  * @return Pointer to HTAB referencing the shared hash.
00148  */
00149 static HTAB *
00150 create_shared_hash(const char *hashname)
00151 {
00152     HASHCTL  hashctl;
00153     HTAB    *result;
00154     char    *db_hashname;
00155     int      hash_elems = veil_shared_hash_elems();
00156 
00157     // Add the current database oid into the hashname so that it is
00158     // distinct from the shared hash for other databases in the cluster.
00159     db_hashname = (char *) vl_malloc(HASH_KEYLEN);
00160     (void) snprintf(db_hashname, HASH_KEYLEN - 1, "%s_%u", 
00161                     hashname, MyDatabaseId);
00162     hashctl.keysize = HASH_KEYLEN;
00163     hashctl.entrysize = sizeof(VarEntry);
00164 
00165     result = ShmemInitHash(db_hashname, hash_elems,
00166                            hash_elems, &hashctl, HASH_ELEM);
00167     pfree(db_hashname);
00168     return result;
00169 }
00170 
00171 /** 
00172  * Return reference to the HTAB for the shared hash associated with
00173  * context 0.
00174  * 
00175  * @return Pointer to HTAB referencing shared hash for context 0.
00176  */
00177 static HTAB *
00178 get_hash0()
00179 {
00180     static HTAB *hash0 = NULL;
00181 
00182     if (!hash0) {
00183         hash0 = create_shared_hash("VEIL_SHARED1");
00184     }
00185     return hash0;
00186 }
00187 
00188 /** 
00189  * Return reference to the HTAB for the shared hash associated with
00190  * context 1.
00191  * 
00192  * @return Pointer to HTAB referencing shared hash for context 1.
00193  */
00194 static HTAB *
00195 get_hash1()
00196 {
00197     static HTAB *hash1 = NULL;
00198 
00199     if (!hash1) {
00200         hash1 = create_shared_hash("VEIL_SHARED2");
00201     }
00202 
00203     return hash1;
00204 }
00205 
00206 /** 
00207  * Allocate or attach to, a new chunk of shared memory for a named
00208  * memory context.
00209  * 
00210  * @param name The name
00211  * @param p_found Pointer to boolean that will identify whether this
00212  * chunk has already been initialised.
00213  * 
00214  * @return Pointer to chunk of shared memory.
00215  */
00216 static MemContext *
00217 get_shmem_context(char   *name,
00218                   size_t  size,
00219                   bool   *p_found)
00220 {
00221     int         i;
00222     MemContext *context;
00223     char       *uniqname  = (char *) vl_malloc(strlen(name) + 16);
00224     int         max_dbs = veil_dbs_in_cluster();
00225 
00226     for (i = 0; i < max_dbs; i++) {
00227         (void) sprintf(uniqname, "%s_%d", name, i);
00228         context = ShmemInitStruct(uniqname, size, p_found);;
00229         if (!context) {
00230             ereport(ERROR,
00231                     (errcode(ERRCODE_INTERNAL_ERROR),
00232                      errmsg("veil: cannot allocate shared memory(1)")));
00233         }
00234 
00235         if (*p_found) {
00236             // Already exists.  Check database id. 
00237             if (context->db_id == MyDatabaseId) {
00238                 /* This context is the one for the current database, 
00239                  * nothing else to do. */
00240                 return context;
00241             }
00242         }
00243         else {
00244             // We Just allocated our first context
00245             context->db_id = MyDatabaseId;
00246             context->next = sizeof(MemContext);
00247             context->limit = size;
00248             context->lwlock = VeilLWLock;
00249             return context;
00250         }
00251     }
00252 
00253     // We reach this point if no existing contexts are allocated to our
00254     // database.  Now we check those existing contexts to see whether
00255     // they are still in use.  If not, we will redeploy them.
00256 
00257     for (i = 0; i < max_dbs; i++) {
00258         (void) sprintf(uniqname, "%s_%d", name, i);
00259         context = ShmemInitStruct(uniqname, size, p_found);;
00260 
00261         if (!context) {
00262             ereport(ERROR,
00263                     (errcode(ERRCODE_INTERNAL_ERROR),
00264                      errmsg("veil: cannot allocate shared memory(2)")));
00265         }
00266 
00267         if (*p_found) {
00268             // Is this context for a still existant database?
00269             if (!vl_db_exists(context->db_id)) {
00270                 // We can re-use this context.
00271                 context->db_id = MyDatabaseId;
00272                 context->next = sizeof(MemContext);
00273                 context->limit = size;
00274 
00275                 *p_found = false;  // Tell the caller that init is required
00276                 return context;
00277             }
00278         }
00279         else {
00280             // We didn't find an unused context, so now we have created 
00281             // a new one.
00282 
00283             context->db_id = MyDatabaseId;
00284             context->next = sizeof(MemContext);
00285             context->limit = size;
00286             return context;
00287         }
00288     }
00289     ereport(ERROR,
00290             (errcode(ERRCODE_INTERNAL_ERROR),
00291              errmsg("veil: no more shared memory contexts allowed")));
00292 }
00293 
00294 // Forward ref, required by next function.
00295 static void shmalloc_init();
00296 
00297 /** 
00298  * Return the id (index) of the current context for this session 
00299  * 
00300  * @return The current context id
00301  */
00302 static int
00303 get_cur_context_id()
00304 {
00305     static initialised = false;
00306     int    context;
00307 
00308     if (!initialised) {
00309         shmalloc_init();
00310         initialised = true;
00311     }
00312         
00313     context = shared_meminfo->current_context;
00314     if (prepared_for_switch) {
00315         context = OTHER_CONTEXT(context);
00316     }
00317     else {
00318         // Check for the default context being for a later transaction
00319         // than current and, if so, use the other one.
00320         if (TransactionIdPrecedes(GetCurrentTransactionId(), 
00321                                   shared_meminfo->xid[context]))
00322         {
00323             context = OTHER_CONTEXT(context);
00324         }
00325     }
00326 
00327     return context;
00328 }
00329 
00330 /** 
00331  * Return pointer to shared memory allocated for the current context.
00332  * 
00333  * @return The current context. 
00334  */
00335 static MemContext *
00336 get_cur_context()
00337 {
00338     int context;
00339     context = get_cur_context_id();
00340     return shared_meminfo->context[context];
00341 }
00342 
00343 /** 
00344  * Dynamically allocate a piece of shared memory from the current
00345  * context, doing no locking.
00346  * 
00347  * @param context The context in which we are operating
00348  * @param size The size of the requested piece of memory.
00349  * 
00350  * @return Pointer to dynamically allocated memory.
00351  */
00352 void *
00353 do_vl_shmalloc(MemContext *context,
00354                size_t size)
00355 {
00356     void *result;
00357     size_t amount = (size_t) MAXALIGN(size);
00358 
00359     if ((amount + context->next) <= context->limit) {
00360         result = context + context->next;
00361         context->next += amount;
00362     }
00363     else {
00364         ereport(ERROR,
00365                 (ERROR,
00366                  (errcode(ERRCODE_INTERNAL_ERROR),
00367                   errmsg("veil: out of shared memory"))));
00368     }
00369     return result;
00370 }
00371 
00372 /** 
00373  * Dynamically allocate a piece of shared memory from the current context. 
00374  * 
00375  * @param size The size of the requested piece of memory.
00376  * 
00377  * @return Pointer to dynamically allocated memory.
00378  */
00379 void *
00380 vl_shmalloc(size_t size)
00381 {
00382     MemContext *context;
00383     void       *result;
00384 
00385     context = get_cur_context();
00386 
00387     LWLockAcquire(VeilLWLock, LW_EXCLUSIVE);
00388     result = do_vl_shmalloc(context, size);
00389     LWLockRelease(VeilLWLock);
00390 
00391     return result;
00392 }
00393 
00394 /** 
00395  * Free a piece of shared memory within the current context.  Currently
00396  * this does nothing as implementation of freeing of shared memory has
00397  * been deferred.
00398  * 
00399  * @param mem Pointer to the memory to be freed.
00400  * 
00401  */
00402 void
00403 vl_free(void *mem)
00404 {
00405     return;
00406 }
00407 
00408 
00409 /** 
00410  * Attach to, creating and initialising as necessary, the shared memory
00411  * control structure.  Record this for the session in shared_meminfo.
00412  */
00413 static void
00414 shmalloc_init()
00415 {
00416     if (!shared_meminfo) {
00417         VarEntry   *var;
00418         MemContext *context0;
00419         MemContext *context1;
00420         bool        found = false;
00421         size_t      allocated;
00422         HTAB       *hash0;
00423         HTAB       *hash1;
00424         size_t      size;
00425 
00426         size = veil_shmem_context_size();
00427 
00428         LWLockAcquire(InitialLWLock, LW_EXCLUSIVE);
00429         context0 = get_shmem_context("VEIL_SHMEM0", size, &found);
00430 
00431         if (found) {
00432             shared_meminfo = context0->memctl;
00433             VeilLWLock = shared_meminfo->veil_lwlock;
00434             // By aquiring and releasing this lock, we ensure that Veil
00435             // shared memory has been fully initialised, by a process
00436             // following the else clause of this code path.
00437             LWLockAcquire(VeilLWLock, LW_EXCLUSIVE);
00438             LWLockRelease(InitialLWLock);
00439             LWLockRelease(VeilLWLock);
00440         }
00441         else {
00442             // Do minimum amount of initialisation while holding
00443             // the initial lock.  We don't want to do anything that
00444             // may cause other locks to be aquired as this could lead
00445             // to deadlock with other add-ins.  Instead, we aquire the
00446             // Veil-specific lock before finishing the initialisation.
00447 
00448             shared_meminfo = do_vl_shmalloc(context0, sizeof(ShmemCtl));
00449 
00450             if (context0->lwlock != InitialLWLock) {
00451                 // Re-use the LWLock previously allocated to this memory 
00452                 // context
00453                 VeilLWLock = context0->lwlock;
00454             }
00455             else {
00456                  // Allocate new LWLock for this new shared memory context
00457                 VeilLWLock = LWLockAssign(); 
00458             }
00459             // Record the lock id in context0 (for possible re-use if
00460             // the current database is dropped and a new veil-using
00461             // database created), and in the shared_meminfo struct
00462             context0->lwlock = VeilLWLock;
00463             shared_meminfo->veil_lwlock = VeilLWLock;
00464             
00465             // Exchange the initial lock for our Veil-specific one.
00466             LWLockAcquire(VeilLWLock, LW_EXCLUSIVE);
00467             LWLockRelease(InitialLWLock);
00468     
00469             // Now do the rest of the Veil shared memory initialisation
00470 
00471             // Set up the other memory context
00472             context1 = get_shmem_context("VEIL_SHMEM1", size, &found);
00473             
00474             // Record location of shmemctl structure in each context
00475             context0->memctl = shared_meminfo;
00476             context1->memctl = shared_meminfo;
00477 
00478             // Finish initialising the shmemctl structure
00479             shared_meminfo->type = OBJ_SHMEMCTL;
00480             shared_meminfo->current_context = 0;
00481             shared_meminfo->total_allocated[0] = size;
00482             shared_meminfo->total_allocated[1] = size;
00483             shared_meminfo->switching = false;
00484             shared_meminfo->context[0] = context0;
00485             shared_meminfo->context[1] = context1;
00486             shared_meminfo->xid[0] = GetCurrentTransactionId();
00487             shared_meminfo->xid[1] = shared_meminfo->xid[0];
00488             shared_meminfo->initialised = true;
00489 
00490             // Set up both shared hashes
00491             hash0 = get_hash0();
00492             hash1 = get_hash1();
00493 
00494             // Record the shmemctl structure in hash0
00495             var = (VarEntry *) hash_search(hash0, (void *) "VEIL_SHMEMCTL",
00496                                            HASH_ENTER, &found);
00497             // elog(WARNING, "SHMEMCTL FOUND - 1 = %d", found);
00498 
00499             var->obj = (Object *) shared_meminfo;
00500             var->shared = true;
00501 
00502             var = (VarEntry *) hash_search(hash0, (void *) "VEIL_SHMEMCTL",
00503                                            HASH_ENTER, &found);
00504             // elog(WARNING, "SHMEMCTL FOUND - 2 = %d", found);
00505 
00506             LWLockRelease(VeilLWLock);
00507         }
00508     }
00509 }
00510 
00511 /** 
00512  * Return the shared hash for the current context.
00513  * 
00514  * @return Pointer to the HTAB for the current context's shared hash.
00515  */
00516 HTAB *
00517 vl_get_shared_hash()
00518 {
00519     int context;
00520     HTAB *hash;
00521     static initialised = false;
00522 
00523     if (!initialised) {
00524         (void) get_cur_context();  // Ensure shared memory is set up.
00525         initialised = true;
00526     }
00527 
00528     context = get_cur_context_id();
00529 
00530     if (context == 0) {
00531         hash = get_hash0();
00532     }
00533     else {
00534         hash = get_hash1();
00535     }
00536     
00537     return hash;
00538 }
00539 
00540 /** 
00541  * Reset one of the shared hashes.  This is one of the final steps in a
00542  * context switch.
00543  * 
00544  * @return hash The shared hash that is to be reset.
00545  */
00546 static void
00547 clear_hash(HTAB *hash)
00548 {
00549     static HASH_SEQ_STATUS status;
00550     VarEntry *var;
00551 
00552     hash_seq_init(&status, hash);
00553     while (var = hash_seq_search(&status)) {
00554         if (strncmp("VEIL_SHMEMCTL", var->key, strlen("VEIL_SHMEMCTL")) != 0) {
00555             (void) hash_search(hash, var->key, HASH_REMOVE, NULL);
00556         }
00557     }
00558 }
00559 
00560 /** 
00561  * Prepare for a switch to the alternate context.  Switching will
00562  * only be allowed if there are no transactions that may still be using
00563  * the context to which we are switching, and there is no other
00564  * process attempting the switch.
00565  * 
00566  * @return true if the switch preparation was successful.
00567  */
00568 bool
00569 vl_prepare_context_switch()
00570 {
00571     int   context_curidx;
00572     int   context_newidx;
00573     HTAB *hash0 = get_hash0(); // We must not attempt to create hashes on
00574     HTAB *hash1 = get_hash1(); // the fly below as they also acquire the lock
00575     TransactionId oldest_xid;
00576     MemContext *context;
00577 
00578     (void) get_cur_context();  // Ensure shared memory is set up
00579 
00580     LWLockAcquire(VeilLWLock, LW_EXCLUSIVE);
00581 
00582     if (shared_meminfo->switching) {
00583         // Another process is performing the switch
00584         LWLockRelease(VeilLWLock);
00585         return false;
00586     }
00587 
00588     shared_meminfo->switching = true;
00589 
00590     // We have claimed the switch.  If we decide that we cannot proceed,
00591     // we will return it to its previous state.
00592 
00593     context_curidx = shared_meminfo->current_context;
00594     context_newidx = OTHER_CONTEXT(context_curidx);
00595 
00596     // In case the alternate context has been used before, we must
00597     // clear it.
00598 
00599     oldest_xid = GetOldestXmin(false);
00600     if (TransactionIdPrecedes(oldest_xid, 
00601                               shared_meminfo->xid[context_curidx])) 
00602     {
00603         // There is a transaction running that precedes the time of
00604         // the last context switch.  That transaction may still be
00605         // using the chunk to which we wish to switch.  We cannot
00606         // allow the switch.
00607         shared_meminfo->switching = false;
00608         LWLockRelease(VeilLWLock);
00609         return false;
00610     }
00611     else {
00612         // It looks like we can safely make the switch.  Reset the
00613         // new context, and make it the current context for this
00614         // session only.
00615         context = shared_meminfo->context[context_newidx];
00616         context->next = sizeof(MemContext);
00617 
00618         // If we are switching to context 0, reset the next field of
00619         // the first chunk to leave space for the ShmemCtl struct.
00620         if (context_newidx == 0) {
00621             context->next += sizeof(ShmemCtl);
00622             clear_hash(hash0);
00623         }
00624         else {
00625             clear_hash(hash1);
00626         }
00627     }
00628 
00629     LWLockRelease(VeilLWLock);
00630     prepared_for_switch = true;
00631     return true;
00632 }
00633 
00634 /** 
00635  * Complete the context switch started by vl_prepare_context_switch().
00636  * Raise an ERROR if the context switch cannot be completed.
00637  * 
00638  * @return true if the context switch is successfully completed.
00639  */
00640 bool
00641 vl_complete_context_switch()
00642 {
00643     int  context_curidx;
00644     int  context_newidx;
00645 
00646     if (!prepared_for_switch) {
00647         ereport(ERROR,
00648                 (errcode(ERRCODE_INTERNAL_ERROR),
00649                  errmsg("failed to complete context switch"),
00650                  errdetail("Not prepared for switch - "
00651                            "invalid state for operation")));
00652     }
00653 
00654     LWLockAcquire(VeilLWLock, LW_EXCLUSIVE);
00655     context_curidx = shared_meminfo->current_context;
00656     context_newidx = OTHER_CONTEXT(context_curidx);
00657 
00658     if (!shared_meminfo->switching) {
00659         // We do not claim to be switching.  We should.
00660         LWLockRelease(VeilLWLock);
00661 
00662         ereport(ERROR,
00663                 (errcode(ERRCODE_INTERNAL_ERROR),
00664                  errmsg("failed to complete context switch"),
00665                  errdetail("Session does not have switching set to true- "
00666                            "invalid state for operation")));
00667     }
00668 
00669     shared_meminfo->switching = false;
00670     shared_meminfo->current_context = context_newidx;
00671     shared_meminfo->xid[context_newidx] = GetCurrentTransactionId();
00672     LWLockRelease(VeilLWLock);
00673     prepared_for_switch = false;
00674     return true;
00675 }
00676 
00677 /** 
00678  * In desparation, if we are unable to complete a context switch, we
00679  * should use this function.
00680  */
00681 void
00682 vl_force_context_switch()
00683 {
00684     int  context_curidx;
00685     int  context_newidx;
00686     MemContext *context;
00687     HTAB *hash0 = get_hash0();
00688     HTAB *hash1 = get_hash1();
00689 
00690     (void) get_cur_context();
00691 
00692     LWLockAcquire(VeilLWLock, LW_EXCLUSIVE);
00693 
00694     context_curidx = shared_meminfo->current_context;
00695     context_newidx = OTHER_CONTEXT(context_curidx);
00696 
00697     // Clear the alternate context.
00698 
00699     context = shared_meminfo->context[context_newidx];
00700     context->next = sizeof(MemContext);
00701     
00702     // If we are switching to context 0, reset the next field of
00703     // the first chunk to leave space for the ShmemCtl struct.
00704     if (context_newidx == 0) {
00705         context->next += sizeof(ShmemCtl);
00706         clear_hash(hash0);
00707     }
00708     else {
00709         clear_hash(hash1);
00710     }
00711     
00712     shared_meminfo->switching = false;
00713     shared_meminfo->current_context = context_newidx;
00714     shared_meminfo->xid[context_newidx] = GetCurrentTransactionId();
00715     shared_meminfo->xid[0] = GetCurrentTransactionId();
00716     LWLockRelease(VeilLWLock);
00717     prepared_for_switch = false;
00718 }
00719 
00720 #else
00721 #include "veil_shmem_pre82.inc"
00722 
00723 #endif

Generated on Tue Mar 11 10:20:03 2008 for Veil by  doxygen 1.5.4