This file is indexed.

/usr/include/trilinos/Teuchos_Comm.hpp is in libtrilinos-teuchos-dev 12.12.1-5.

This file is owned by root:root, with mode 0o644.

The actual contents of the file can be viewed below.

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
// @HEADER
// ***********************************************************************
//
//                    Teuchos: Common Tools Package
//                 Copyright (2004) Sandia Corporation
//
// Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
// license for use of this work by or on behalf of the U.S. Government.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// 1. Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
//
// 3. Neither the name of the Corporation nor the names of the
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// Questions? Contact Michael A. Heroux (maherou@sandia.gov)
//
// ***********************************************************************
// @HEADER

#ifndef TEUCHOS_COMM_HPP
#define TEUCHOS_COMM_HPP

#include "Teuchos_ReductionOp.hpp"
#include "Teuchos_ArrayRCP.hpp"


namespace Teuchos {

/// \class CommStatus
/// \brief Encapsulation of the result of a receive (blocking or nonblocking).
///
/// An instance of this class encapsulates the result of a receive.
/// (An MPI implementation would wrap MPI_Status.)  You can query it
/// for information like the rank of the process that sent you the
/// message.  (This is useful if your receive specified a negative
/// source rank, indicating that you would accept a message from any
/// process in the communicator.)
///
/// \tparam OrdinalType The same template parameter as Comm.  Only use
///   \c int here.  We only make this a template class for
///   compatibility with Comm.
///
/// \note For now, this class only exposes the rank of the process
///   that sent the message (the "source rank") and its tag.  Later,
///   we might expose other fields of MPI_Status in this interface.
///   For now, you can attempt a dynamic cast to MpiCommStatus to
///   access all three fields (MPI_SOURCE, MPI_TAG, and MPI_ERROR).
template<class OrdinalType>
class CommStatus {
public:
  //! Destructor (declared virtual for memory safety)
  virtual ~CommStatus() {}

  //! The source rank that sent the message.
  virtual OrdinalType getSourceRank () = 0;

  //! The tag of the received message.
  virtual OrdinalType getTag () = 0;
};

// Forward declaration for CommRequest::wait.
template<class OrdinalType>
class Comm;

/// \class CommRequest
/// \brief Encapsulation of a pending nonblocking communication operation.
/// \tparam OrdinalType Same as the template parameter of Comm.
///
/// An instance of (a subclass of) this class represents a nonblocking
/// communication operation, such as a nonblocking send, receive, or
/// collective.  To wait on the communication operation, you may give
/// the CommRequest to functions like wait() or waitAll() (which may
/// be found in Teuchos_CommHelpers.hpp).  Here is an example of how
/// to use wait().
/// \code
/// const int sourceRank = ...; // Rank of the sending process.
/// RCP<const Comm<int> > comm = ...; // The communicator.
/// ArrayRCP<double> buf (...); // Buffer for incoming data.
/// RCP<CommRequest<int> > req = ireceive (comm, buf, sourceRank);
///
/// // ... Do some other things ...
///
/// // Wait on the request.  This blocks on the sending process.
/// // When it finishes, it invalidates the req reference, and
/// // returns a status (which wraps MPI_Status in an MPI
/// // implementation).
/// RCP<CommStatus<int> > status = wait (comm, ptr (&req));
/// \endcode
///
/// This object's destructor cancels the request without
/// communication.  If you wish, you may rely on this behavior for
/// speculative communication.  For example:
/// \code
/// const int sourceRank = ...; // Rank of the sending process.
/// RCP<const Comm<int> > comm = ...; // The communicator.
/// ArrayRCP<double> buf (...); // Buffer for incoming data.
/// RCP<CommRequest<int> > req = ireceive (comm, buf, sourceRank);
///
/// // ... Do some other things ...
/// // ... Find out we didn't need to receive data ...
///
/// // This cancels the request.  We could also just let
/// // the one reference to the request fall out of scope.
/// req = null;
/// \endcode
///
/// \note To implementers: The MPI implementation of this class
///   (MpiCommRequest) wraps MPI_Request.  The MPI version of
///   waitAll() will need to unpack the array of wrapped requests, and
///   then pack up the resulting MPI_Request after waiting on them.
///   It would be preferable to have a class \c CommRequests that
///   encapsulates a set of requests, so that you can avoid this
///   unpacking and packing.
template<class OrdinalType>
class CommRequest : public Teuchos::Describable {
public:
  /// \brief Destructor; cancels the request if it is still pending.
  ///
  /// Canceling a communication request must always be a local
  /// operation.  An MPI implementation may achieve this by first
  /// calling MPI_Cancel to cancel the request, then calling MPI_Wait
  /// (which behaves as a local operation for a canceled request) to
  /// complete the canceled request (as required by the MPI standard).
  virtual ~CommRequest() {}

  //! Wait on this request (a blocking operation).
  virtual RCP<CommStatus<OrdinalType> > wait () = 0;
};

/// \class Comm
/// \brief Abstract interface for distributed-memory communication.
/// \tparam Ordinal Type of indices used for communication.
///
/// \section Teuchos_Comm_What What is Comm?
///
/// This class is Teuchos' interface to distributed-memory
/// communication between one or more parallel processes.  It presents
/// an interface very much like that of MPI (the Message Passing
/// Interface).  Teuchos provides two implementations of Comm:
/// - An MPI (Message Passing Interface) implementation, MpiComm
/// - A "serial" implementation, SerialComm, that only has one process
///
/// Comm is an abstract interface.  You cannot create a Comm directly.
/// You have to create one of the subclasses.  The normal way to
/// handle a Comm is to pass it around using RCP (a reference-counted
/// "smart" pointer).  For example:
/// \code
/// // Make a Comm.  This one happens to wrap MPI_COMM_WORLD.
/// RCP<const Comm<int> > comm = rcp (new MpiComm (MPI_COMM_WORLD));
/// // Equivalent of MPI_Comm_rank
/// const int myRank = comm->getRank ();
/// // Equivalent of MPI_Comm_size
/// const int numProcs = comm->getSize ();
/// // Equivalent of MPI_Comm_barrier
/// comm->barrier ();
/// \endcode
///
/// Comm's communication methods that actually send or receive data
/// accept that data as an array of \c char.  You should never call
/// these methods directly.  Instead, you should use the nonmember
/// "helper" functions in Teuchos_CommHelpers.hpp.  These methods are
/// templated on the \c Packet type, that is, the type of data you
/// want to send or receive.  See the example below.
///
/// \section Teuchos_Comm_Handle Treat <tt>RCP<const Comm<int> ></tt> like an opaque handle
///
/// You should consider an <tt>RCP<const Comm<int> ></tt> as
/// equivalent to the MPI_Comm opaque handle, except that the RCP also
/// does reference counting to ensure memory safety when using the
/// same communicator in different parts of the code.  That is,
/// copying the RCP does not create a new communicator; the following
/// two codes do about the same thing, except with a different syntax
/// (and reference counting in the second code).
///
/// Raw MPI_Comm handles:
/// \code
/// MPI_Comm comm = ...;
/// // sameComm is THE SAME COMMUNICATOR as comm.
/// MPI_Comm sameComm = comm;
/// \endcode
///
/// Reference-counted pointers to Comm:
/// \code
/// RCP<const Comm<int> > comm = ...;
/// // *sameComm is THE SAME COMMUNICATOR as *comm.
/// RCP<const Comm<int> > sameComm = comm;
/// \endcode
///
/// If you want to make a "new communicator" rather than just "copy
/// the handle," you should call the duplicate() method.  This has the
/// same behavior as MPI_Comm_dup (which see).
///
/// The "reference counting" feature means that the subclass of Comm
/// will take care of freeing the underlying MPI_Comm (and any other
/// data structures it may use) by calling MPI_Comm_free if necessary,
/// once the reference count of the RCP goes to zero.
///
/// \warning Do <i>not</i> pass around subclasses of Comm by value!
///   Comm or its subclasses by themselves do not have handle
///   semantics.  Their copy constructors likely do not behave as you
///   would expect if the classes had handle semantics.
///
/// \section Teuchos_Comm_How How do I make a Comm?
///
/// Comm works whether or not you have build Trilinos with MPI
/// support.  If you want to make a "default" Comm that is the
/// equivalent of MPI_COMM_WORLD, but you don't know if your Trilinos
/// with MPI enabled, you may use GlobalMPISession to call MPI_Init if
/// necessary, and DefaultComm to "get a default communicator."  For
/// example:
/// int main (int argc, char* argv[]) {
///   using Teuchos::Comm;
///   using Teuchos::DefaultComm;
///   using Teuchos::RCP;
///
///   // This replaces the call to MPI_Init.  If you didn't
///   // build with MPI, this doesn't call MPI functions.
///   Teuchos::GlobalMPISesssion session (&argc, &argv, NULL);
///   // comm is the equivalent of MPI_COMM_WORLD.
///   RCP<const Comm<int> > comm = DefaultComm<int>::getComm ();
///
///   // ... use comm in your code as you would use MPI_COMM_WORLD ...
///
///   // We don't need to call MPI_Finalize, since the
///   // destructor of GlobalMPISession does that for us.
///   return EXIT_SUCCESS;
/// }
/// \endcode
/// This code works whether or not you built Trilinos with MPI
/// support.  It is not necessary to use GlobalMPISession, but it's
/// useful so you don't have to remember to call MPI_Finalize.  If you
/// don't want to use GlobalMPISession, you can still call
/// <tt>DefaultComm<int>::getComm()</tt>, though you must have called
/// MPI_Init first if you build Trilinos with MPI support.
/// Furthermore, if you know MPI is present, you don't need to use
/// DefaultComm.  You may simply pass MPI_COMM_WORLD directly to
/// MpiComm, like this:
/// \code
/// RCP<const Comm<int> > comm = rcp (new MpiComm (MPI_COMM_WORLD));
/// \endcode
/// You may also pass an arbitrary MPI_Comm directly into MpiComm's
/// constructor, though you are responsible for freeing it after use
/// (via MPI_Comm_free) if necessary.  You may automate the freeing
/// of your MPI_Comm by using OpaqueWrapper (which see).
///
/// \section Teuchos_Comm_Use How do I use Comm?
///
/// As we mentioned above, for communication of data with Comm, you
/// you should use the nonmember "helper" functions in
/// Teuchos_CommHelpers.hpp.  These methods are templated on the
/// <tt>Packet</tt> type, that is, the type of data you want to send
/// or receive.  For example, suppose you have two processes (with
/// ranks 0 and 1, respectively), and you want to send an array of
/// 10 <tt>double</tt> from Process 0 to Process 1.  Both processes have
/// defined <tt>RCP<const Comm<int> > comm</tt> as above.  Here is the
/// code on Process 0:
/// \code
/// const int count = 10; // Send 10 doubles
/// double values[10] = ...;
/// const int destinationRank = 1; // Send to Process 1
/// // You may be able to omit the template arguments of 'send' here.
/// Teuchos::send<int, double> (*comm, 10, values, destinationRank);
/// \endcode
/// Here is the code on Process 1:
/// \code
/// const int count = 10; // Receive 10 doubles
/// double values[10]; // Will be overwritten by receive
/// const int sourceRank = 0; // Receive from Process 0
/// // You may be able to omit the template arguments of 'receive' here.
/// Teuchos::receive<int, double> (*comm, sourceRank, 10, values);
/// \endcode
/// Please refer to the documentation in Teuchos_CommHelpers.hpp for
/// more details.
///
/// \section Teuchos_Comm_Former Former documentation
///
/// This interface is templated on the ordinal type but only deals with buffers
/// of untyped data represented as arrays <tt>char</tt> type. All reduction
/// operations that are initiated by the concreate communicator object are
/// performed by user-defined <tt>ReductOpBase</tt> objects.  It is the
/// responsibility of the <tt>ReductOpBase</tt> object to know what the currect
/// data type is, to perform casts or serializations/unserializations to and
/// from <tt>char[]</tt> buffers, and to know how to reduce the objects
/// correctly.  It is strictly up to the client to correctly convert data types
/// to <tt>char[]</tt> arrays but there is a great deal of helper code to make
/// this easy and safe.
template<typename Ordinal>
class Comm : virtual public Describable {
public:
  /// \brief The current tag.
  ///
  /// \warning This method is ONLY for use by Teuchos developers.
  ///   Users should not depend on the interface of this method.
  ///   It may change or disappear at any time without warning.
  virtual int getTag () const = 0;

  //! @name Destructor
  //@{

  //! Destructor, declared virtual for safety of derived classes.
  virtual ~Comm() {}
  //@}

  //! @name Query functions
  //@{

  /** \brief Returns the rank of this process.
   *
   * <b>Postconditions:</b><ul>
   * <li><tt>0 <= return && return < this->getSize()</tt>
   * </ul>
   */
  virtual int getRank() const = 0;

  /** \brief Returns the number of processes that make up this communicator.
   *
   * <b>Postconditions:</b><ul>
   * <li><tt>return > 0</tt>
   * </ul>
   */
  virtual int getSize() const = 0;

  //@}

  //! @name Collective Operations
  //@{

  /** \brief Pause every process in <tt>*this</tt> communicator until all the
   * processes reach this point.
   */
  virtual void barrier() const = 0;

  /** \brief Broadcast values from the root process to the slave processes.
   *
   * \param rootRank [in] The rank of the root process.
   *
   * \param count [in] The number of bytes in <tt>buffer[]</tt>.
   *
   * \param buffer [in/out] Array (length <tt>bytes</tt>) of packed data.
   * Must be set on input on the root processes with rank <tt>root</tt>.  On
   * output, each processs, including the root process contains the data.
   *
   * <b>Preconditions:</b><ul>
   * <li><tt>0 <= rootRank && rootRank < this->getSize()</tt>
   * </ul>
   */
  virtual void broadcast(
    const int rootRank, const Ordinal bytes, char buffer[]
    ) const = 0;

  //! Gather values from all processes to the root process.
  virtual void
  gather (const Ordinal sendBytes, const char sendBuffer[],
          const Ordinal recvBytes, char recvBuffer[],
          const int root) const = 0;

  /** \brief Gather values from each process to collect on all processes.
   *
   * \param sendBytes [in] Number of entires in <tt>sendBuffer[]</tt> on
   * input.
   *
   * \param sendBuffer [in] Array (length <tt>sendBytes</tt>) of data being
   * sent from each process.
   *
   * \param recvBytes [in] Number of entires in <tt>recvBuffer[]</tt> which
   * must be equal to <tt>sendBytes*this->getSize()</tt>.  This field is just
   * here for debug checking.
   *
   * \param recvBuffer [out] Array (length <tt>recvBytes</tt>) of all of the
   * entires sent from each processes.  Specifically,
   * <tt>recvBuffer[sendBytes*j+i]</tt>, for <tt>j=0...this->getSize()-1</tt>
   * and <tt>i=0...sendBytes-1</tt>, is the entry <tt>sendBuffer[i]</tt> from
   * process with rank <tt>j</tt>.
   *
   * <b>Preconditions:</b><ul>
   * <li><tt>recvBytes==sendBytes*this->getSize()</tt>
   * </ul>
   */
  virtual void gatherAll(
    const Ordinal sendBytes, const char sendBuffer[]
    ,const Ordinal recvBytes, char recvBuffer[]
    ) const = 0;

  /** \brief Global reduction.
   *
   * \param reductOp [in] The user-defined reduction operation
   *
   * \param bytes [in] The length of the buffers <tt>sendBuffer[]</tt> and
   * <tt>globalReducts[]</tt>.
   *
   * \param sendBuffer [in] Array (length <tt>bytes</tt>) of the data
   * contributed from each process.
   *
   * \param globalReducts [out] Array (length <tt>bytes</tt>) of the global
   * reduction from each process.
   */
  virtual void reduceAll(
    const ValueTypeReductionOp<Ordinal,char> &reductOp
    ,const Ordinal bytes, const char sendBuffer[], char globalReducts[]
    ) const = 0;

  /** \brief Scan reduction.
   *
   * \param reductOp [in] The user-defined reduction operation
   *
   * \param bytes [in] The length of the buffers <tt>sendBuffer[]</tt> and
   * <tt>scanReducts[]</tt>.
   *
   * \param sendBuffer [in] Array (length <tt>bytes</tt>) of the data
   * contributed from each process.
   *
   * \param scanReducts [out] Array (length <tt>bytes</tt>) of the reduction
   * up to and including this process.
   */
        virtual void scan(
    const ValueTypeReductionOp<Ordinal,char> &reductOp
    ,const Ordinal bytes, const char sendBuffer[], char scanReducts[]
    ) const = 0;

  //! @name Blocking Point-to-Point Operations
  //@{

  /** \brief Possibly blocking send of data from this process to another process.
   *
   * This routine does not return until you can reuse the send buffer.
   * Whether this routine blocks depends on whether the MPI
   * implementation buffers.
   *
   * \param bytes [in] The number of bytes of data being passed between
   * processes.
   *
   * \param sendBuffer [in] Array (length <tt>bytes</tt>) of data being sent
   * from this process.  This buffer can be immediately destroyed or reused as
   * soon as the function exits (that is why this function is "blocking").
   *
   * \param destRank [in] The rank of the process to receive the data.
   *
   * <b>Preconditions:</b><ul>
   * <li><tt>0 <= destRank && destRank < this->getSize()</tt>
   * <li><tt>destRank != this->getRank()</tt>
   * </ul>
   */
  virtual void send(
    const Ordinal bytes, const char sendBuffer[], const int destRank
    ) const = 0;

  //! Variant of send() that takes a tag.
  virtual void
  send (const Ordinal bytes,
        const char sendBuffer[],
        const int destRank,
        const int tag) const = 0;

  /** \brief Always blocking send of data from this process to another process.
   *
   * This routine blocks until the matching receive posts.  After it
   * returns, you are allowed to reuse the send buffer.
   *
   * \param bytes [in] The number of bytes of data being passed between
   * processes.
   *
   * \param sendBuffer [in] Array (length <tt>bytes</tt>) of data being sent
   * from this process.  This buffer can be immediately destroyed or reused as
   * soon as the function exits (that is why this function is "blocking").
   *
   * \param destRank [in] The rank of the process to receive the data.
   *
   * <b>Preconditions:</b><ul>
   * <li><tt>0 <= destRank && destRank < this->getSize()</tt>
   * <li><tt>destRank != this->getRank()</tt>
   * </ul>
   */
  virtual void ssend(
    const Ordinal bytes, const char sendBuffer[], const int destRank
    ) const = 0;

  //! Variant of ssend() that takes a message tag.
  virtual void
  ssend (const Ordinal bytes,
         const char sendBuffer[],
         const int destRank,
         const int tag) const = 0;

  /** \brief Blocking receive of data from this process to another process.
   *
   * \param sourceRank [in] The rank of the process to receive the data from.
   * If <tt>sourceRank < 0</tt> then data will be received from any process.
   *
   * \param bytes [in] The number of bytes of data being passed between
   * processes.
   *
   * \param recvBuffer [out] Array (length <tt>bytes</tt>) of data being
   * received from this process.  This buffer can be immediately used to
   * access the data as soon as the function exits (that is why this function
   * is "blocking").
   *
   * <b>Preconditions:</b><ul>
   * <li>[<tt>sourceRank >= 0] <tt>sourceRank < this->getSize()</tt>
   * <li><tt>sourceRank != this->getRank()</tt>
   * </ul>
   *
   * \return Returns the senders rank.
   */
  virtual int receive(
    const int sourceRank, const Ordinal bytes, char recvBuffer[]
    ) const = 0;


  /** \brief Ready send of data from this process to another process.
   *
   * \param sendBuffer [in] The data to be sent.
   *
   * \param destRank [in] The rank of the process to receive the data.
   *
   * <b>Preconditions:</b><ul>
   * <li><tt>0 <= destRank && destRank < this->getSize()</tt>
   * <li><tt>destRank != this->getRank()</tt>
   * </ul>
   */
  virtual void readySend(
    const ArrayView<const char> &sendBuffer,
    const int destRank
    ) const = 0;

  //! Variant of readySend() that accepts a message tag.
  virtual void
  readySend (const Ordinal bytes,
             const char sendBuffer[],
             const int destRank,
             const int tag) const = 0;

  //@}
  //! @name Non-blocking Point-to-Point Operations
  //@{

  /** \brief Non-blocking send.
   *
   * \param sendBuffer [in] The data buffer to be sent.
   *
   * \param destRank [in] The rank of the process to receive the data.
   *
   * <b>Preconditions:</b><ul>
   * <li><tt>0 <= destRank && destRank < this->getSize()</tt>
   * <li><tt>destRank != this->getRank()</tt>
   * </ul>
   */
  virtual RCP<CommRequest<Ordinal> > isend(
    const ArrayView<const char> &sendBuffer,
    const int destRank
    ) const = 0;

  //! Variant of isend() that takes a tag.
  virtual RCP<CommRequest<Ordinal> >
  isend (const ArrayView<const char> &sendBuffer,
         const int destRank,
         const int tag) const = 0;

  /** \brief Non-blocking receive.
   *
   * \param recvBuffer [out] The location for storing the received data.
   *
   * \param sourceRank [in] The rank of the process to receive the data from.
   * If <tt>sourceRank < 0</tt> then data will be received from any process.
   *
   * <b>Preconditions:</b><ul>
   * <li>[<tt>sourceRank >= 0] <tt>sourceRank < this->getSize()</tt>
   * <li><tt>sourceRank != this->getRank()</tt>
   * </ul>
   *
   * \return Returns the senders rank.
   */
  virtual RCP<CommRequest<Ordinal> > ireceive(
    const ArrayView<char> &recvBuffer,
    const int sourceRank
    ) const = 0;

  //! Variant of ireceive that takes a tag.
  virtual RCP<CommRequest<Ordinal> >
  ireceive (const ArrayView<char> &recvBuffer,
            const int sourceRank,
            const int tag) const = 0;

  /** \brief Wait on a set of communication requests.
   *
   * <b>Preconditions:</b><ul>
   * <li> <tt>requests.size() > 0</tt>
   * </ul>
   *
   * <b>Postconditions:</b><ul>
   * <li> <tt>is_null(request[i]))</tt> for <tt>i=0...requests.size()-1</tt>
   * </ul>
   */
  virtual void waitAll(
    const ArrayView<RCP<CommRequest<Ordinal> > > &requests
    ) const = 0;

  /// \brief Wait on communication requests, and return their statuses.
  ///
  /// \pre requests.size() == statuses.size()
  ///
  /// \pre For i in 0, 1, ..., requests.size()-1, requests[i] is
  ///   either null or requests[i] was returned by an ireceive() or
  ///   isend().
  ///
  /// \post For i in 0, 1, ..., requests.size()-1,
  ///   requests[i].is_null() is true.
  ///
  /// \param requests [in/out] On input: the requests on which to
  ///   wait.  On output: all set to null.
  ///
  /// \param statuses [out] The status results of waiting on the
  ///   requests.
  virtual void
  waitAll (const ArrayView<RCP<CommRequest<Ordinal> > >& requests,
           const ArrayView<RCP<CommStatus<Ordinal> > >& statuses) const = 0;

  /// \brief Wait on a single communication request, and return its status.
  ///
  /// \param request [in/out] On input: request is not null, and
  /// <tt>*request</tt> is either null (in which case this function
  /// does nothing and returns null) or an RCP of a valid CommRequest
  /// instance representing an outstanding communication request.  On
  /// output: If the communication request completed successfully, we
  /// set <tt>*request</tt> to null, indicating that the request has
  /// completed.  (This helps prevent common bugs like trying to
  /// complete the same request twice.)
  ///
  /// \return If *request is null, this method returns null.
  /// Otherwise this method returns a CommStatus instance representing
  /// the result of completing the request.  In the case of a
  /// nonblocking receive request, you can query the CommStatus
  /// instance for the process ID of the sending process.  (This is
  /// useful for receiving from any process via \c MPI_ANY_SOURCE.)
  ///
  /// \pre <tt>!is_null(request)</tt> (that is, the Ptr is not null).
  /// \post <tt>is_null(*request)</tt> (that is, the RCP is null).
  ///
  /// This function blocks until the communication operation
  /// associated with the CommRequest object has completed.
  virtual RCP<CommStatus<Ordinal> >
  wait (const Ptr<RCP<CommRequest<Ordinal> > >& request) const = 0;

  //@}

  //! @name Subcommunicator Operations
  //@{

  /**
   * \brief Duplicate this communicator.
   *
   * Make a copy of this communicator with a duplicate communication
   * space.  Note that the returned communicator has the same
   * properties (including process ranks, attributes and topologies)
   * as this communicator, but is distinct from the original.
   * "Distinct" means that if you send a message on the original
   * communicator, you can't receive it on the new one, and vice
   * versa.  The new communicator represents a separate message space.
   * This has the same semantics as MPI_Comm_dup.  (In fact, the
   * subclass MpiComm implements this using MPI_Comm_dup.)
   *
   * Most users don't want to do this.  The duplicate() method returns
   * a <i>new communicator</i>.  In MPI terms, it is a <i>different
   * MPI_Comm</i>.  If you want a shallow copy of the handle, you
   * should pass the <tt>Comm<Ordinal><tt> around by const pointer,
   * like this:
   * \code
   * RCP<const Comm<int> > comm = ...; // my original communicator
   * // ... do some stuff with comm ...
   * // Make a shallow copy.
   * RCP<const Comm<int> > diffHandleSameComm = comm;
   * // ... do some stuff with diffHandleSameComm ...
   * \endcode
   * This behaves the same as the following "raw MPI" code:
   * \code
   * MPI_Comm comm = ...; // my original communicator
   * // ... do some stuff with comm ...
   * // Make a shallow copy.
   * MPI_Comm diffHandleSameComm = comm;
   * // ... do some stuff with diffHandleSameComm ...
   * \endcode
   * The subclass of Comm ensures that the "raw" MPI handle is freed
   * only after the last reference to it by a subclass instance
   * disappears.  (It does reference counting underneath.)
   *
   * Please, please do not invoke the copy constructor or assignment
   * operator of Comm.  Of course it's not legal to do that anyway,
   * because Comm is pure virtual.  However, even if you could do it,
   * you must never do this!  For example, do <i>not</i> do this:
   * \code
   * RCP<const Comm<int> > comm = ...; // my original communicator
   * // ... do some stuff with comm ...
   * // DO NOT DO THIS, EVER!!!  THIS IS VERY BAD!!!
   * RCP<const Comm<int> > badComm (new Comm<int> (*comm));
   * \endcode
   * and do <i>not</i> do this:
   * \code
   * RCP<const Comm<int> > comm = ...; // my original communicator
   * // ... do some stuff with comm ...
   * // DO NOT DO THIS, EITHER!!!  THIS IS JUST AS BAD!!!
   * RCP<const Comm<int> > badComm = rcp (new Comm<int> (*comm));
   * \endcode
   * This is bad because it ignores the subclass' data.  Depending on
   * the subclass of Comm that you are actually using, it may be
   * appropriate to invoke the copy constructor or assignment operator
   * of the specific subclass, but <i>never</i> those of Comm itself.
   *
   * Users are not responsible for freeing the returned communicator.
   * The destructor of the subclass of Comm handles that itself.
   *
   * In an MPI implementation, the returned communicator is created
   * using MPI_Comm_dup, with the resulting semantic implications.
   *
   * \return A new communicator.
   */
  virtual RCP< Comm > duplicate() const = 0;

  /**
   * \brief Split a communicator into subcommunicators based on color
   * and key.
   *
   * Partition this communicator into multiple disjoint groups, and
   * return the communicator corresponding to the group to which this
   * process belongs.  There will be as many groups as there are
   * globally many distinct values for the <tt>color</tt> parameter.
   * Within each subset of the partition, the ranks will be ordered
   * according to the key value each process passed for the
   * <tt>key</tt> parameter. If multiple processes pass the same value
   * for <tt>key</tt>, then they will be ordered according to their
   * rank in the original communicator.  To return a valid
   * communicator, this function requires a nonnegative value for
   * <tt>color</tt>.  If <tt>color</tt> is negative, this method will
   * return a null communicator.
   *
   * This method must be called as a collective on all processes in
   * this communicator.  That is, if this method is called at all, it
   * must be called on all processes in the communicator.
   *
   * Users are not responsible for freeing the returned communicator.
   * The destructor of the subclass of Comm handles that itself.
   *
   * In an MPI implementation, the returned communicator is created
   * using MPI_Comm_split, with the resulting semantic implications.
   *
   * \param color [in] An integer representing the color for the local
   *   rank.  In the MPI implementation, if this is negative,
   *   MPI_Comm_split gets <tt>MPI_UNDEFINED</tt> as the color.
   *
   * \param key [in] A key value to order processes of the same color.
   *   In the MPI implementation, this is passed directly to
   *   MPI_Comm_split.
   *
   * \return A partitioned communicator.
   */
  virtual RCP<Comm> split (const int color, const int key) const = 0;

  /**
   * \brief Create a subcommunicator containing the specified processes.
   *
   * Create and return a subcommunicator of this communicator.  The
   * subcommunicator contains the processes in this communicator with
   * the given ranks, in which they are listed in the input vector.
   * Processes whose ranks are not included in the input vector will
   * be given a null communicator.
   *
   * This method must be called as a collective on all processes in
   * this communicator.  That is, if this method is called at all, it
   * must be called on all processes in the communicator.
   *
   * Users are not responsible for freeing the returned communicator.
   * The destructor of the subclass of Comm handles that itself.
   *
   * In an MPI implementation, the subcommunicator is created using
   * MPI_Comm_create, with the resulting semantic implications.
   *
   * \param ranks The ranks of the processes to include in the subcommunicator.
   * \return The subcommunicator.
   */
  virtual RCP<Comm>
  createSubcommunicator (const ArrayView<const int>& ranks) const = 0;
  //@}

}; // class Comm

} // namespace Teuchos

#endif // TEUCHOS_COMM_HPP