This file is indexed.

/usr/include/GAServices.h is in libglobalarrays-dev 5.4~beta~r10636+dfsg-5.

This file is owned by root:root, with mode 0o644.

The actual contents of the file can be viewed below.

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
#ifndef _GA_SERVICES_H
#define _GA_SERVICES_H

namespace GA {

class GlobalArray;

/**              
 * Global Arrays Services class.
 * 
 * @author Manoj Kumar Krishnan, PNNL. 
 * @deprecated Please use the same functions found within the GA namespace.
 * 
 * Collecting the global information: who am I, and how many processors 
 * are being used. Initialize the communication library (either MPI or 
 * TCSMSG) and Global array. Allocate momory to be used by GA by calling MA
 * and create global arrays.
 */
class GAServices {

public:
  /**  
   * Null-constructor. The component won't really be 'alive'
   * much at all until after setServices is called on it.
   */
  GAServices();
  
  /** Destructor. */
  ~GAServices();
  
  /**  
   * Creates an ndim-dimensional array using the regular distribution model 
   * and returns integer handle representing the array. 
   *
   * The array can be distributed evenly or not. The control over the 
   * distribution is accomplished by specifying chunk (block) size for all or 
   * some of array dimensions.
   *
   * For example, for a 2-dimensional array, setting chunk[0]=dim[0] gives 
   * distribution by vertical strips (chunk[0]*dims[0]); 
   * setting chunk[1]=dim[1] gives distribution by horizontal strips 
   * (chunk[1]*dims[1]). Actual chunks will be modified so that they are at 
   * least the size of the minimum and each process has either zero or one 
   * chunk. Specifying chunk[i] as <1 will cause that dimension to be 
   * distributed evenly. 
   *
   * As a convenience, when chunk is specified as NULL, the entire array is 
   * distributed evenly.
   *
   * This is a collective operation. 
   *
   * @param[in] type        data type(MT_F_DBL,MT_F_INT,MT_F_DCPL)
   * @param[in] ndim        number of array dimensions
   * @param[in] dims[ndim]  array of dimensions
   * @param[in] arrayname   a unique character string
   * @param[in] chunk[ndim] array of chunks, each element specifies 
   *                        minimum size that given dimensions should be
   *                        chunked up into
   *
   * @return pointer to GlobalArray object created; NULL if it fails
   */
  GlobalArray * createGA(int type, int ndim, int dims[], char *arrayname, 
          int chunk[]);
  
  /**
   * Creates an array by following the user-specified distribution and 
   * returns integer handle representing the array. 
   *
   * The distribution is specified as a Cartesian product of distributions 
   * for each dimension. The array indices start at 0. For example, the 
   * following figure demonstrates distribution of a 2-dimensional array 8x10 
   * on 6 (or more) processors. nblock[2]={3,2}, the size of map array is s=5 
   * and array map contains the following elements map={0,2,6, 0, 5}. The 
   * distribution is nonuniform because, P1 and P4 get 20 elements each and 
   * processors P0,P2,P3, and P5 only 10 elements each. 
   *        
   * <TABLE>
   * <TR> <TD>5</TD>  <TD>5</TD>  </TR>
   * <TR> <TD>P0</TD> <TD>P3</TD> <TD>2</TD> </TR>
   * <TR> <TD>P1</TD> <TD>P4</TD> <TD>4</TD> </TR>
   * <TR> <TD>P2</TD> <TD>P5</TD> <TD>2</TD> </TR>
   * </TABLE>
   *
   * This is a collective operation. 
   *
   * @param[in] arrayname   a unique character string
   * @param[in] type        MA data type (MT_F_DBL,MT_F_INT,MT_F_DCPL)
   * @param[in] ndim        number of array dimensions
   * @param[in] dims        array of dimension values
   * @param[in] block       [ndim] no. of blocks each dimension is divided into
   * @param[in] maps        [s] starting index for for each block;
   *                        the size s is a sum all elements of nblock array
   *
   * @return pointer to GlobalArray object created; NULL if it fails
   */
  GlobalArray * createGA(int type, int ndim, int dims[], char *arrayname, 
          int block[], int maps[]);
  
  /**
   * Creates a new array by applying all the properties of another existing 
   * array.
   *
   * This is a collective operation. 
   *
   * @param[in] arrayname a character string
   * @param[in] g_b       integer handle for reference array
   *
   * @return pointer to GlobalArray object created; NULL if it fails
   */
  GlobalArray * createGA(const GlobalArray *g_b, char *arrayname);
  
  /**
   * Creates a new array by applying all the properties of another existing 
   * array.
   *
   * This is a collective operation. 
   *
   * @param[in] g_b integer handle for reference array
   *
   * @return pointer to GlobalArray object created; NULL if it fails
   */
  GlobalArray * createGA(const GlobalArray &g_b);
  
  /**
   * Creates a 10x10 global array of type "double"(default).
   *
   * This is a collective operation. 
   *
   * @return pointer to GlobalArray object created; NULL if it fails
   */
  GlobalArray * createGA();  
  
  /**
   * Creates an ndim-dimensional array with a layer of ghost cells around 
   * the visible data on each processor using the regular distribution 
   * model and returns an integer handle representing the array. 
   * The array can be distributed evenly or not evenly. The control over 
   * the distribution is accomplished by specifying chunk (block) size for 
   * all or some of the array dimensions. For example, for a 2-dimensional 
   * array, setting chunk(1)=dim(1) gives distribution by vertical strips 
   * (chunk(1)*dims(1)); setting chunk(2)=dim(2) gives distribution by 
   * horizontal strips (chunk(2)*dims(2)). Actual chunks will be modified 
   * so that they are at least the size of the minimum and each process 
   * has either zero or one chunk. Specifying chunk(i) as <1 will cause
   * that dimension (i-th) to be distributed evenly. The  width of the 
   * ghost cell layer in each dimension is specified using the array 
   * width().  The local data of the global array residing on each 
   * processor will have a layer width[n] ghosts cells wide on either 
   * side of the visible data along the dimension n. 
   *
   * This is a collective operation. 
   * 
   * @param[in] array_name a unique character string
   * @param[in] type       data type (MT_DBL,MT_INT,MT_DCPL)
   * @param[in] ndim       number of array dimensions
   * @param[in] dims       [ndim] array of dimensions
   * @param[in] width      [ndim] array of ghost cell widths
   * @param[in] chunk      [ndim] array of chunks, each element specifies
   *                       minimum size that given dimensions should be
   *                       chunked up into
   *
   * @returns pointer to GlobalArray object created; NULL if it fails
   */
  GlobalArray * createGA_Ghosts(int type, int ndim, int dims[], 
          int width[], char *array_name, int chunk[]);
  
  /**
   * Creates an array with ghost cells by following the user-specified 
   * distribution and returns integer handle representing the array. 
   * The distribution is specified as a Cartesian product of distributions 
   * for each dimension. For example, the following figure demonstrates 
   * distribution of a 2-dimensional array 8x10 on 6 (or more) processors. 
   * nblock(2)={3,2}, the size of map array is s=5 and array map contains 
   * the following elements map={1,3,7, 1, 6}. The distribution is 
   * nonuniform because, P1 and P4 get 20 elements each and processors 
   * P0,P2,P3, and P5 only 10 elements each. 
   *
   * <TABLE>
   * <TR> <TD>5</TD>  <TD>5</TD>  </TR>
   * <TR> <TD>P0</TD> <TD>P3</TD> <TD>2</TD> </TR>
   * <TR> <TD>P1</TD> <TD>P4</TD> <TD>4</TD> </TR>
   * <TR> <TD>P2</TD> <TD>P5</TD> <TD>2</TD> </TR>
   * </TABLE>
   *
   * The array width[] is used to control the width of the ghost cell 
   * boundary around the visible data on each processor. The local data 
   * of the global array residing on each processor will have a layer 
   * width[n] ghosts cells wide on either side of the visible data along 
   * the dimension n. 
   *
   * This is a collective operation. 
   *
   * @param[in] array_name a unique character string
   * @param[in] type       data type (MT_DBL,MT_INT,MT_DCPL)
   * @param[in] ndim       number of array dimensions
   * @param[in] dims       [ndim] array of dimensions
   * @param[in] width      [ndim] array of ghost cell widths
   * @param[in] nblock     [ndim] no. of blocks each dimension is divided into
   * @param[in] map        [s] starting index for for each block;
   *                       the size s is a sum of all elements of nblock array
   *
   * @return pointer to GlobalArray object created; NULL if it fails
   */
  GlobalArray * createGA_Ghosts(int type, int ndim, int dims[], 
          int width[], char *array_name, int map[], 
          int nblock[]);
  
  /**
   * Broadcast from process root to all other processes a message of 
   * length lenbuf. This is operation is provided only for convenience 
   * purposes: it is available regardless of the message-passing library 
   * that GA is running with. 
   *
   * This is a collective operation. 
   *
   * @param[in]     lenbuf length of buffer
   * @param[in,out] buf    [lenbuf] data
   * @param[in]     root   root process
   */
  void brdcst(void *buf, int lenbuf, int root);
  
  /**
   * Returns the current value of the internal debug flag.
   *
   * This is a local operation.
   *
   * @return 0 if the debug flag is false, 1 if it is true.
   */
  int getDebug();
  
  /**
   * This functions returns the total number of nodes that the program is 
   * running on.
   *
   * On SMP architectures, this will be less than or equal to the total
   * number of processors. 
   *
   * This is a  local operation. 
   *
   * @return the number of nodes the program is running on
   */
  int clusterNnodes();
  
  /**  
   * This function returns the node ID of the process.
   *
   * On SMP architectures with more than one processor per node, several
   * processes may return the same node id. 
   *
   * This is a  local operation. 
   *
   * @return the node ID of the process
   */
  int clusterNodeid();
  
  /**  
   * This function returns the cluster node ID of the specified process.
   *
   * On SMP architectures with more than one processor per node, several
   * processes may return the same node id. 
   *
   * This is a  local operation. 
   *
   * @return the cluster node ID of the specified process
   */
  int clusterProcNodeid(int iproc);
  
  /**
   * This function returns the number of processors available on node inode. 
   *
   * This is a  local operation. 
   *
   * @param[in] inode
   *
   * @return the number of processors available on the given node
   */
  int clusterNprocs(int inode);
  
  /**
   * This function returns the processor id associated with node inode and 
   * the local processor id iproc.
   *
   * If node inode has N processors, then the value of iproc lies between
   * 0 and N-1. 
   *
   * This is a  local operation. 
   *
   * @param[in] inode
   * @param[in] iproc
   *
   * @return the processor ID associated with the given node and local processor
   * ID
   */
  int clusterProcid(int inode, int iproc);
  
  /**
   * Creates a set containing the number of mutexes.
   *
   * Mutex is a simple synchronization object used to protect Critical
   * Sections. Only one set of mutexes can exist at a time. Array of mutexes
   * can be created and destroyed as many times as needed. 
   * Mutexes are numbered: 0, ..., number -1.
   *
   * This is a collective operation. 
   *
   * @param[in] number of mutexes in mutex array
   *
   * @return 0 if the operation succeeded or 1 when failed. 
   */
  int createMutexes(int number);

  /**
   * Remove a user defined data type from GA
   *
   * @param[in] type - user defined data type
   *
   * @return 0 is operation is successful
   *         -2 if type not registered
   *         -1 if type reserved
   */
  int deregisterType(int type);
  
  /** 
   * Destroys the set of mutexes created with ga_create_mutexes.
   *
   * This is a collective operation. 
   *
   * @return 0 if the operation succeeded or 1 when failed. 
   */
  int destroyMutexes();
  
  /**
   * Double Global OPeration. 
   *
   * X(1:N) is a vector present on each process. DGOP 'sums' elements of 
   * X accross all nodes using the commutative operator OP. The result is 
   * broadcast to all nodes. Supported operations include '+', '*', 'max', 
   * 'min', 'absmax', 'absmin'. The use of lowerecase for operators is 
   * necessary. This is operation is provided only for convenience purposes: 
   * it is available regardless of the message-passing library that GA is 
   * running with.
   *
   * This is a collective operation. 
   *
   * @param[in]     n  number of elements
   * @param[in,out] x  [n] array of elements
   * @param[in]     op operator
   */
  void dgop(double x[], int n, char *op);
  
  /**
   * Creates a new array by applying all the properties of another existing 
   * array.
   *
   * This is a collective operation. 
   *
   * @param[in] array_name a character string
   * @param[in] g_a        integer handle for reference array
   *
   * @return array handle; a non-zero array handle means the call was succesful. 
   */
  int duplicate(int g_a, char* array_name);
  
  /**
   * To be called in case of an error.
   *
   * Print an error message and an integer value that represents error code.
   * Releases some system resources. 
   * This is the required way of aborting the program execution. 
   *
   * This operation is local. 
   *
   * @param[in] message string to print
   * @param[in] code    code to print
   */
  void error(const char *message, int code);
  
  /**
   * Blocks the calling process until all the data transfers corresponding to 
   * GA operations called after ga_init_fence complete.
   *
   * For example, since ga_put might return before the data reaches the final
   * destination, ga_init_fence and ga_fence allow process to wait until the
   * data tranfer is fully completed: 
   *
   * @code
   *   ga_init_fence();
   *   ga_put(g_a, ...);
   *   ga_fence();
   * @endcode
   * 
   * ga_fence must be called after ga_init_fence. A barrier, ga_sync, assures 
   * completion of all data transfers and implicitly cancels all outstanding
   * ga_init_fence calls. ga_init_fence and ga_fence must be used in pairs, 
   * multiple calls to ga_fence require the same number of corresponding
   * ga_init_fence calls. ga_init_fence/ga_fence pairs can be nested. 
   * 
   * ga_fence works for multiple GA operations. For example: 
   * 
   * @code
   *   ga_init_fence();
   *   ga_put(g_a, ...);
   *   ga_scatter(g_a, ...);
   *   ga_put(g_b, ...);
   *   ga_fence();
   * @endcode
   *
   * The calling process will be blocked until data movements initiated by 
   * two calls to ga_put and one ga_scatter complete. 
   */
  void fence();

  /**
   * Integer Global OPeration.
   *
   * The integer version of ga_dgop described above, also include the bitwise OR
   * operation.  This is operation is provided only for convenience purposes: it
   * is available regardless of the message-passing library that GA is running
   * with.
   *
   * This is a collective operation. 
   *
   * @param[in]     n  number of elements
   * @param[in,out] x  [n] array of elements
   * @param[in]     op operator
   */
  void gop(int x[], int n, char *op);
  
  /**
   * Long Global OPeration. 
   *
   * X(1:N) is a vector present on each process. LGOP 'sums' elements of 
   * X accross all nodes using the commutative operator OP. The result is 
   * broadcast to all nodes. Supported operations include '+', '*', 'max', 
   * 'min', 'absmax', 'absmin'. The use of lowerecase for operators is 
   * necessary. This is operation is provided only for convenience purposes: 
   * it is available regardless of the message-passing library that GA is 
   * running with.
   *
   * This is a collective operation. 
   *
   * @param[in]     n  number of elements
   * @param[in,out] x  [n] array of elements
   * @param[in]     op operator
   */
  void gop(long x[], int n, char *op);
  
  /**
   * Float Global OPeration. 
   *
   * X(1:N) is a vector present on each process. FGOP 'sums' elements of 
   * X accross all nodes using the commutative operator OP. The result is 
   * broadcast to all nodes. Supported operations include '+', '*', 'max', 
   * 'min', 'absmax', 'absmin'. The use of lowerecase for operators is 
   * necessary. This is operation is provided only for convenience purposes: 
   * it is available regardless of the message-passing library that GA is 
   * running with.
   *
   * This is a collective operation. 
   *
   * @param[in]     n  number of elements
   * @param[in,out] x  [n] array of elements
   * @param[in]     op operator
   */
  void gop(float x[], int n, char *op);
  
  /**
   * Double Global OPeration. 
   *
   * X(1:N) is a vector present on each process. DGOP 'sums' elements of 
   * X accross all nodes using the commutative operator OP. The result is 
   * broadcast to all nodes. Supported operations include '+', '*', 'max', 
   * 'min', 'absmax', 'absmin'. The use of lowerecase for operators is 
   * necessary. This is operation is provided only for convenience purposes: 
   * it is available regardless of the message-passing library that GA is 
   * running with.
   *
   * This is a collective operation. 
   *
   * @param[in]     n  number of elements
   * @param[in,out] x  [n] array of elements
   * @param[in]     op operator
   */
  void gop(double x[], int n, char *op);
  
  /**
   * Integer Global OPeration.
   *
   * The integer (more precisely long) version of ga_dgop described above,
   * also include the bitwise OR operation. 
   * This is operation is provided only for convenience purposes: it is 
   * available regardless of the message-passing library that GA is running 
   * with.
   *
   * This is a collective operation. 
   *
   * @param[in]     n  number of elements
   * @param[in,out] x  [n] array of elements
   * @param[in]     op operator
   */
  void igop(int x[], int n, char *op);
  
  /**
   * Initializes tracing of completion status of data movement operations. 
   *
   * This operation is local. 
   */
  void initFence();
  
  /**
   * Returns amount of memory (in bytes) used in the allocated global 
   * arrays on the calling processor.
   *
   * This operation is local. 
   *
   * @return amount of memory (in bytes) used in the allocated global arrays on
   *         the calling processor
   */
  size_t inquireMemory();
  
  /**
   * Long Global OPeration. 
   *
   * X(1:N) is a vector present on each process. LGOP 'sums' elements of 
   * X accross all nodes using the commutative operator OP. The result is 
   * broadcast to all nodes. Supported operations include '+', '*', 'max', 
   * 'min', 'absmax', 'absmin'. The use of lowerecase for operators is 
   * necessary. This is operation is provided only for convenience purposes: 
   * it is available regardless of the message-passing library that GA is 
   * running with.
   *
   * This is a collective operation. 
   *
   * @param[in]     n  number of elements
   * @param[in,out] x  [n] array of elements
   * @param[in]     op operator
   */
  void lgop(long x[], int n, char *op);
  
  /**
   * Locks a mutex object identified by the mutex number. It is a fatal 
   * error for a process to attempt to lock a mutex which was already 
   * locked by this process. 
   *
   * @param[in] mutex object id
   */
  void lock(int mutex);
  
  /**
   * Mask the intrinsic sync operations during collective calls.
   *
   * GA Collective calls has Sync calls at the begining and ending of
   * of the call. Sometimes there may be some redundacy in sync calls, which
   * can be avoided by masking the sync operations. 
   *
   * Setting the parameters as zero will mask (disable) the call. Any non-zero 
   * value will enable the call. Initially these params are set to non-zero 
   * value.
   *
   * @param[in] first masks the sync at the begining of the collective call.
   * @param[in] last  masks the sync at the end of the collective call.
   */
  void maskSync(int first, int last);
  
  /**
   * If GA_uses_ma returns true, then GA_Memory_avail returns the 
   * lesser of the amount available under the GA limit and the amount 
   * available from MA (according to ma_inquire_avail operation). 
   * If no GA limit has been set, it returns what MA says is available. 
   * If ( ! GA_Uses_ma() && ! GA_Memory_limited() ) returns < 0, indicating 
   * that the bound on currently available memory cannot be determined. 
   *
   * This operation is local. 
   *
   * @return amount of memory (in bytes) left for allocation of new 
   *         global arrays on the calling processor. 
   *
   */
  int memoryAvailable() ;
  
  /**
   * Indicates if limit is set on memory usage in Global Arrays on the 
   * calling processor.
   * 
   * This operation is local. 
   *
   * @return 1 means "yes", "0" means "no".
   */
  int memoryLimited();
  
  /**
   * Force completion of a nonblocking operation locally.
   *
   * Waiting on a nonblocking put or an accumulate operation assures that data
   * was injected into the network and the user buffer can be now be reused.
   * Completing a get operation assures data has arrived into the user memory
   * and is ready for use. Wait operation ensures only local completion. Unlike
   * their blocking counterparts, the nonblocking operations are not ordered
   * with respect to the destination. Performance being one reason, the other
   * reason is that by ensuring ordering we incur additional and possibly
   * unnecessary overhead on applications that do not require their operations
   * to be ordered. For cases where ordering is necessary, it can be done by
   * calling a fence operation.  The fence operation is provided to the user to
   * confirm remote completion if needed.
   *
   * This is a local operation.
   *
   * @param[in] nbhandle nonblocking handle
   */
  void nbWait(GANbhdl *nbhandle);
  
  /**
   * Returns the GA process id (0, ..., ga_Nnodes()-1) of the requesting 
   * compute process.
   *
   * This operation is local. 
   *
   * @return the GA process ID of the requesting process
   */
  int nodeid();
  
  /**
   * Returns the number of the GA compute (user) processes. 
   *
   * This operation is local. 
   *
   * @return the number of GA processes
   */
  int nodes();
  
  /** 
   * Print statistical information on GA use.
   *
   * This non-collective (MIMD) operation prints information about:
   *   - number of calls to
   *     - create
   *     - duplicate
   *     - destroy
   *     - get
   *     - put
   *     - scatter
   *     - gather
   *     - read_and_inc operations
   *   - total amount of data moved in the primitive operations
   *   - amount of data moved in the primitive operations to logicaly remote
   *     locations
   *   - maximum memory consumption in global arrays
   *   - number of requests serviced in the interrupt-driven implementations
   *     by the calling process.
   *
   * This operation is local. 
   */
  void printStats();

  /**
   * Add a user defined data type to GA
   *
   * @param[in] size - size (in bytes) of user defined data type
   *
   * @return  handle for new data type
   */
  int registerType(size_t size);
  
  /**
   * This function sets an internal flag in the GA library to either true or
   * false.
   *
   * The value of this flag can be recovered at any time using the
   * getDebug function. The flag is set to false when the the GA library
   * is initialized. This can be useful in a number of debugging situations,
   * especially when examining the behavior of routines that are called in
   * multiple locations in a code. 
   *
   * This is a local operation.
   *
   * @param[in] dbg value to set internal flag
   */
  void setDebug(int dbg);
  
  /**
   * Sets the amount of memory to be used (in bytes) per process.
   * 
   * This is a local operation. 
   *
   * @param[in] limit the amount of memory in bytes per process
   */
  void setMemoryLimit(size_t limit);
  
  /** 
   * Prints info about allocated arrays. 
   *
   * @param[in] verbose if true print distribution info
   */
  void summarize(int verbose);
  
  /**
   * Synchronize processes (a barrier) and ensure that all GA operations 
   * completed. 
   *
   * This is a collective operation. 
   */
  void sync();
  
  /**
   * Unlocks a mutex object identified by the mutex number.
   *
   * It is a fatal error for a process to attempt to unlock a mutex which has
   * not been locked by this process. 
   *
   * @param[in] mutex object id
   */
  void unlock(int mutex);
  
  /**
   * Returns whether memory comes from internal or external allocator.
   *
   * This operation is local. 
   *
   * @return "1" if memory comes from MA;
   *         "0" if memory comes from another source e.g. System V shared memory
   */
  int usesMA();
  
  /**
   * Returns whether GA is using Fortran indexing.
   *
   * @return "1" if uses fortran API, else returns "0"
   */
  int usesFAPI();
  
  /**
   * This function return a wall (or elapsed) time on the calling processor.
   *
   * Returns time in seconds representing elapsed wall-clock time
   * since an arbitrary time in the past. Example:
   *
   * @code
   *     double starttime, endtime;
   *     starttime = GA::SERVICES.wtime();
   *     // {{.... code snippet to be timed ....}}
   *     endtime   = GA::SERVICES.wtime();
   *     printf("Time taken = %lf seconds\n", endtime-starttime);
   * @endcode
   *
   * This is a local operation.
   *
   * @note This function is only available in release 4.1 or greater.
   */
  double wtime();
};

extern GAServices SERVICES;

}

#endif /* _GA_SERVICES_H */