libvtk6-dev 6.3.0+dfsg1-5 / usr / include / vtk-6.3 / vtkPStructuredGridConnectivity.h

/*=========================================================================

 Program:   Visualization Toolkit
 Module:    vtkPStructuredGridConnectivity.h

 Copyright (c) Ken Martin, Will Schroeder, Bill Lorensen
 All rights reserved.
 See Copyright.txt or http://www.kitware.com/Copyright.htm for details.

 This software is distributed WITHOUT ANY WARRANTY; without even
 the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
 PURPOSE.  See the above copyright notice for more information.

 =========================================================================*/
// .NAME vtkPStructuredGridConnectivity.h -- Constructs grid connectivity
//
// .SECTION Description
//  vtkPStructuredGridConnectivity inherits from vtkStructuredGridConnectivity
//  and implements functionality to compute the neighboring topology within a
//  single, partitioned and distributed structured grid dataset.
//
// .SECTION Caveats
//  Initialize(), ComputeNeighbors() and CreateGhostLayers() are collective
//  operations, every process must call that method.
//
// .SECTION See Also
//  vtkStructuredGridConnectivity vtkGhostArray

#ifndef VTKPSTRUCTUREDGRIDCONNECTIVITY_H_
#define VTKPSTRUCTUREDGRIDCONNECTIVITY_H_

// VTK include directives
#include "vtkFiltersParallelGeometryModule.h" // For export macro
#include "vtkStructuredGridConnectivity.h"
#include "vtkMPICommunicator.h" // Needed for vtkMPICommunicator::Request

// C++ include directives
#include <vector> // For STL vector

// Forward declarations
class vtkMultiProcessController;
class vtkMPIController;
class vtkMultiProcessStream;
//class vtkMPICommunicator::Request;

class VTKFILTERSPARALLELGEOMETRY_EXPORT vtkPStructuredGridConnectivity :
  public vtkStructuredGridConnectivity
{
public:
  static vtkPStructuredGridConnectivity* New();
  vtkTypeMacro(vtkPStructuredGridConnectivity,vtkStructuredGridConnectivity);
  void PrintSelf(ostream& os, vtkIndent indent );

  // Description:
  // Set & Get the process controller
  vtkSetMacro( Controller,vtkMultiProcessController* );
  vtkGetMacro( Controller,vtkMultiProcessController* );

  // Description:
  // Sets the total number of domains distributed among processors
  void SetNumberOfGrids( const unsigned int N );

  // Description:
  // See vtkStructuredGridConnectivity::RegisterGrid
  void RegisterGrid( const int gridID, int extents[6],
      vtkUnsignedCharArray* nodesGhostArray,
      vtkUnsignedCharArray* cellGhostArray,
      vtkPointData* pointData,
      vtkCellData* cellData,
      vtkPoints* gridNodes );

  // Description:
  // Returns the number of local grids registers by the process that owns
  // the current vtkPStructuredGridConnectivity instance
  int GetNumberOfLocalGrids()
    { return static_cast<int>(this->GridIds.size()); };

  // Description:
  // Returns the rank of the given gridID. A nominal value of -1 for the
  // return value of this method indicates that possibly ComputeNeighbors
  // has not been called and consequently the GridRanks vector has not been
  // populated yet.
  int GetGridRank( const int gridID );

  // Description:
  // Returns true iff the grid is remote, otherwise false.
  bool IsGridRemote( const int gridID );

  // Description:
  // Returns true iff the grid corresponding to the given gridID is local.
  bool IsGridLocal( const int gridID );

  // Description:
  // Initializes this instance of vtkPStructuredGridConnectivity, essentially,
  // the acquires the local process ID from the registered controller. If a
  // controller is not registered, the global controller is set.
  void Initialize();

  // Description:
  // Computes the neighboring topology of a distributed structured grid
  // data-set.
  // See vtkStructuredGridConnectivity::ComputeNeighbors
  void ComputeNeighbors();

  // Description:
  // Creates ghost layers on the grids owned by this process using data from
  // both local and remote block neighbors.
  virtual void CreateGhostLayers( const int N=1 );

protected:
  vtkPStructuredGridConnectivity();
  ~vtkPStructuredGridConnectivity();

  vtkMultiProcessController *Controller;
  int Rank;
  bool Initialized;

  // BTX
  std::vector< int > GridRanks; // Corresponding rank for each grid
  std::vector< int > GridIds;   // List of GridIds, owned by this process

  // Data structures to store the remote ghost data of each grid for each one
  // of its neighbors. The first index is the global grid index. The second
  // is the neighbor index.
  std::vector< std::vector< vtkPoints* > >    RemotePoints;
  std::vector< std::vector< vtkPointData* > > RemotePointData;
  std::vector< std::vector< vtkCellData* > >  RemoteCellData;

  // Data structures to store the send/receive buffer sizes and corresponding
  // persistent buffers. The first index is the global grid index. The second
  // index is the neighbor index for the given grid.
  std::vector< std::vector< unsigned int > > SendBufferSizes;
  std::vector< std::vector< unsigned int > > RcvBufferSizes;
  std::vector< std::vector< unsigned char* > > SendBuffers;
  std::vector< std::vector< unsigned char* > > RcvBuffers;

  int TotalNumberOfSends;
  int TotalNumberOfRcvs;
  int TotalNumberOfMsgs;

  // Array of MPI requests
  vtkMPICommunicator::Request *MPIRequests;
  // ETX

  // Description:
  // Returns true if the two extents are equal, otherwise false.
  bool GridExtentsAreEqual( int rhs[6], int lhs[6] );

  // Description:
  // Returns true iff the grid corresponding to the given ID has point data.
  bool HasPointData(const int gridIdx);

  // Description:
  // Returns true iff the grid corresponding to the given ID has cell data.
  bool HasCellData(const int gridIdx);

  // Description:
  // Returns true iff the grid corresponding to the given ID has points.
  bool HasPoints(const int gridIdx);

  // Description:
  // Sets all message counters to 0.
  void InitializeMessageCounters();

  // Description:
  // Clears all internal VTK data-structures that are used to store the remote
  // ghost data.
  void ClearRemoteData();

  // Description:
  // Clears all raw send/rcv buffers
  void ClearRawBuffers();

  // Description:
  // Registers a remote grid with the given grid Id, structured extents and
  // process.
  void RegisterRemoteGrid( const int gridID, int extents[6], int process );

  // Description:
  // This method transfers all the remote neighbor data to the ghosted grid
  // instance of the grid corresponding to the given grid index.
  void TransferRemoteNeighborData(
      const int gridIdx,const int nei,const vtkStructuredNeighbor& Neighbor );

  // Description:
  // This method transfers the fields (point data and cell data) to the ghost
  // extents from the neighboring grids of the grid corresponding to the given
  // gridID.
  virtual void TransferGhostDataFromNeighbors(const int gridID);

  // Description:
  // Helper method to pack all the ghost data into send buffers.
  void PackGhostData();

  // Description:
  // Helper method to unpack the raw ghost data from the receive buffers in
  // to the VTK remote point data-structures.
  void UnpackGhostData();

  // Description:
  // Helper method to deserialize the buffer sizes coming from the given
  // process.
  void DeserializeBufferSizesForProcess(
      int *buffersizes, vtkIdType N, const int processId );

  // Description:
  // Helper method to serialize the buffer sizes for the grids of this process
  // to neighboring grids.
  void SerializeBufferSizes(int *&sizesbuf, vtkIdType &N);

  // Description:
  // Helper method to exchange buffer sizes.Each process sends the send buffer
  // size of each grid to each of its neighbors.
  void ExchangeBufferSizes();

  // Description:
  // Helper method for exchanging ghost data. It loops through all the grids,
  // and for each neighbor it constructs the corresponding send buffer.
  // size and posts a non-blocking receive.
  void ExchangeGhostDataInit();

  // Description:
  // Helper method to communicate ghost data. Loops through all the neighbors
  // and for every remote neighbor posts a non-blocking receive.
  void PostReceives();

  // Description:
  // Helper method to communicate ghost data. Loops through the neighbors and
  // for every remote neighbor posts a non-blocking send.
  void PostSends();

  // Description:
  // Helper method for exchanging ghost data. It loops through all the grids
  // and for each neighbor of each grid it serializes the data in the send
  // extent and posts a non-blocking send.
  void CommunicateGhostData();

  // Description:
  // Helper method for exchanging ghost data. It blocks until all requests
  // are complete (via a waitAll) and then de-serializes the receive buffers
  // to form the corresponding remote data-structures.
  void ExchangeGhostDataPost();

  // Description:
  // Exchanges ghost data of the grids owned by this process.
  void ExchangeGhostData();

  // Description:
  // Helper method to serialize the ghost points to send to a remote process.
  // Called from SerializeGhostData.
  void SerializeGhostPoints(
      const int gridIdx, int ext[6], vtkMultiProcessStream& bytestream );

  // Description:
  // Serializes the data array into a bytestream.
  void SerializeDataArray(
      vtkDataArray *dataArray, vtkMultiProcessStream& bytestream );

  // Description:
  // Helper method to serialize field data. Called from
  // SerializeGhostPointData and SerializeGhostCellData.
  void SerializeFieldData(
      int sourceExtent[6], int targetExtent[6], vtkFieldData *fieldData,
      vtkMultiProcessStream& bytestream );

  // Description:
  // Helper method to serialize ghost point data. Called from
  // SerializeGhostData.
  void SerializeGhostPointData(
      const int gridIdx, int ext[6], vtkMultiProcessStream& bytestream );

  // Description:
  // Helper method to serialize ghost cell data. Called from
  // SerializeGhostData.
  void SerializeGhostCellData(
      const int gridIdx, int ext[6], vtkMultiProcessStream& bytestream );

  // Description:
  // Helper method to de-serialize the ghost points received from a remote
  // process. Called from DeserializeGhostData.
  void DeserializeGhostPoints(
      const int gridIdx, const int nei,
      int ext[6], vtkMultiProcessStream& bytestream );

  // Description:
  // Helper method to deserialize the data array from a bytestream.
  void DeserializeDataArray(
      vtkDataArray *&dataArray,const int dataType,
      const int numberOfTuples, const int numberOfComponents,
      vtkMultiProcessStream& bytestream );

  // Description:
  // Helper method to de-serialize field data. Called from
  // DeserializeGhostPointData and DeserializeGhostCellData.
  void DeserializeFieldData(
      int ext[6], vtkFieldData *fieldData,
      vtkMultiProcessStream &bytestream );

  // Description:
  // Helper method to de-serialize the ghost point data received from a
  // remote process. Called from DeserializeGhostData.
  void DeserializeGhostPointData(
      const int gridIdx, const int nei,
      int ext[6], vtkMultiProcessStream& bytestream );

  // Description:
  // Helper method to de-serialize the ghost cell data received from a remote
  // process. Called from DeserializeGhostCellData.
  void DeserializeGhostCellData(
      const int gridIdx, const int nei,
      int ext[6], vtkMultiProcessStream& bytestream );

  // Description:
  // Given a grid ID and the corresponding send extent, this method serializes
  // the grid and data within the given extent. Upon return, the buffer is
  // allocated and contains the data in raw form, ready to be sent. Called
  // from vtkPStructuredGridConnectivity::PackGhostData().
  void SerializeGhostData(
      const int sndGridID, const int rcvGrid, int sndext[6],
      unsigned char*& buffer, unsigned int &size);

  // Description:
  // Given the raw buffer consisting of ghost data, this method deserializes
  // the object and returns the gridID and rcvext of the grid.
  void DeserializeGhostData(
      const int gridID, const int neiListID,
      const int neiGridIdx, int rcvext[6],
      unsigned char *buffer, unsigned int size );

  // Description:
  // Exchanges the grid extents among all processes and fully populates the
  // GridExtents vector.
  void ExchangeGridExtents();

  // Description:
  // Serializes the grid extents and information in a buffer to send over MPI
  // The data is serialized as follows: ID imin imax jmin jmax kmin kmax
  void SerializeGridExtents( int *&sndbuffer, vtkIdType &N );

  // Description:
  // Deserializes the received grid extent information to the GridExtents
  // internal data-structures.
  void DeserializeGridExtentForProcess(
      int *rcvbuffer, vtkIdType &N, const int processId );

private:
  vtkPStructuredGridConnectivity(const vtkPStructuredGridConnectivity& ); // Not implemented
  void operator=(const vtkPStructuredGridConnectivity& ); // Not implemented
};

//=============================================================================
//  INLINE METHODS
//=============================================================================


inline bool vtkPStructuredGridConnectivity::GridExtentsAreEqual(
    int rhs[6], int lhs[6] )
{
  for( int i=0; i < 6; ++i )
    {
    if( rhs[i] != lhs[i] )
      {
      return false;
      }
    }
  return true;
}

//------------------------------------------------------------------------------
inline bool vtkPStructuredGridConnectivity::HasPointData(const int gridIdx)
{
  // Sanity check
  assert("pre: grid index is out-of-bounds!" &&
         (gridIdx >= 0) && (gridIdx < static_cast<int>(this->NumberOfGrids)));

  if( (this->GridPointData[gridIdx] != NULL) &&
      (this->GridPointData[gridIdx]->GetNumberOfArrays() > 0) )
    {
    return true;
    }
  return false;
}

//------------------------------------------------------------------------------
inline bool vtkPStructuredGridConnectivity::HasCellData(const int gridIdx)
{
  // Sanity check
  assert("pre: grid index is out-of-bounds!" &&
         (gridIdx >= 0) && (gridIdx < static_cast<int>(this->NumberOfGrids)));

  if( (this->GridCellData[gridIdx] != NULL) &&
      (this->GridCellData[gridIdx]->GetNumberOfArrays( ) > 0) )
    {
    return true;
    }
  return false;
}

//------------------------------------------------------------------------------
inline bool vtkPStructuredGridConnectivity::HasPoints(const int gridIdx)
{
  // Sanity check
  assert("pre: grid index is out-of-bounds!" &&
         (gridIdx >= 0) && (gridIdx < static_cast<int>(this->NumberOfGrids)));

  if( this->GridPoints[gridIdx] != NULL )
    {
    return true;
    }
  return false;
}

//------------------------------------------------------------------------------
inline void vtkPStructuredGridConnectivity::InitializeMessageCounters()
{
  this->TotalNumberOfMsgs=this->TotalNumberOfRcvs=this->TotalNumberOfSends=0;
}

//------------------------------------------------------------------------------
inline void vtkPStructuredGridConnectivity::ClearRawBuffers()
{
  this->SendBufferSizes.clear();
  this->RcvBufferSizes.clear();

  // STEP 0: Clear send buffers
  for( unsigned int i=0; i < this->SendBuffers.size(); ++i )
    {
    for( unsigned int j=0; j < this->SendBuffers[i].size(); ++j )
      {
      delete [] this->SendBuffers[i][j];
      } // END for all neighbors
    this->SendBuffers[i].clear();
    } // END for all grids
  this->SendBuffers.clear();

  // STEP 1: Clear rcv buffers
  for( unsigned int i=0; i < this->RcvBuffers.size(); ++i )
    {
    for( unsigned int j=0; j < this->RcvBuffers[i].size(); ++j )
      {
      delete [] this->RcvBuffers[i][j];
      } // END for all neighbors
    this->RcvBuffers[i].clear();
    } // END for all grids
  this->RcvBuffers.clear();
}

//------------------------------------------------------------------------------
inline void vtkPStructuredGridConnectivity::ClearRemoteData()
{
  // STEP 0: Clear remote points
  for( unsigned int i=0; i < this->RemotePoints.size(); ++i )
    {
    for( unsigned int j=0; j < this->RemotePoints[i].size(); ++j )
      {
      if( this->RemotePoints[ i ][ j ] != NULL )
        {
        this->RemotePoints[ i ][ j ]->Delete();
        }
      } // END for all j
    this->RemotePoints[ i ].clear();
    } // END for all i
  this->RemotePoints.clear();

  // STEP 1: Clear remote point data
  for( unsigned int i=0; i < this->RemotePointData.size(); ++i )
    {
    for( unsigned int j=0; j < this->RemotePointData[i].size(); ++j )
      {
      if( this->RemotePointData[ i ][ j ] != NULL )
        {
        this->RemotePointData[ i ][ j ]->Delete();
        }
      } // END for all j
    this->RemotePointData[ i ].clear();
    } // END for all i
  this->RemotePointData.clear();

  // STEP 2: Clear remote cell data
  for( unsigned int i=0; i < this->RemoteCellData.size(); ++i )
    {
    for( unsigned int j=0; j < this->RemoteCellData[i].size(); ++j )
      {
      if( this->RemoteCellData[ i ][ j ] != NULL )
        {
        this->RemoteCellData[ i ][ j ]->Delete();
        }
      } // END for all j
    this->RemoteCellData[ i ].clear();
    }
  this->RemoteCellData.clear();
}

//------------------------------------------------------------------------------
inline bool vtkPStructuredGridConnectivity::IsGridRemote(const int gridID )
{
  return( !this->IsGridLocal(gridID) );
}

//------------------------------------------------------------------------------
inline bool vtkPStructuredGridConnectivity::IsGridLocal(const int gridID)
{
  assert( "pre: Instance has not been intialized!" && this->Initialized );
  assert( "pre: gridID is out-of-bounds" &&
          (gridID >= 0) && (gridID < static_cast<int>(this->NumberOfGrids) ) );
  assert( "pre: GridRanks is not properly allocated" &&
          this->NumberOfGrids == this->GridRanks.size() );
  return( (this->GridRanks[ gridID ] == this->Rank) );
}

//------------------------------------------------------------------------------
inline int vtkPStructuredGridConnectivity::GetGridRank( const int gridID )
{
  assert( "pre: Instance has not been intialized!" && this->Initialized );
  assert( "pre: gridID out-of-bounds!" &&
          (gridID >= 0  && gridID < static_cast<int>(this->NumberOfGrids)));
  return( this->GridRanks[ gridID ] );
}
#endif /* VTKPSTRUCTUREDGRIDCONNECTIVITY_H_ */