VTK  9.3.1
vtkPPixelTransfer.h
Go to the documentation of this file.
1 // SPDX-FileCopyrightText: Copyright (c) Ken Martin, Will Schroeder, Bill Lorensen
2 // SPDX-License-Identifier: BSD-3-Clause
17 #ifndef vtkPPixelTransfer_h
18 #define vtkPPixelTransfer_h
19 
20 #include "vtkMPI.h" // for mpi
21 #include "vtkMPIPixelTT.h" // for type traits
22 #include "vtkMPIPixelView.h" // for mpi subarrays
23 #include "vtkPixelExtent.h" // for pixel extent
24 #include "vtkPixelTransfer.h"
25 #include "vtkRenderingParallelLICModule.h" // for export
26 #include "vtkSetGet.h" // for macros
27 
28 // included vtkSystemIncludes.h in the base class.
29 #include <cstring> // for memcpy
30 #include <iostream> // for ostream
31 #include <vector> // for vector
32 
33 // #define vtkPPixelTransferDEBUG
34 
35 VTK_ABI_NAMESPACE_BEGIN
36 class VTKRENDERINGPARALLELLIC_EXPORT vtkPPixelTransfer : public vtkPixelTransfer
37 {
38 public:
40  : SrcRank(0)
41  , DestRank(0)
42  , UseBlockingSend(0)
43  , UseBlockingRecv(0)
44  {
45  }
46 
51  vtkPPixelTransfer(int srcRank, const vtkPixelExtent& srcWholeExt, const vtkPixelExtent& srcExt,
52  int destRank, const vtkPixelExtent& destWholeExt, const vtkPixelExtent& destExt, int id = 0)
53  : Id(id)
54  , SrcRank(srcRank)
55  , SrcWholeExt(srcWholeExt)
56  , SrcExt(srcExt)
57  , DestRank(destRank)
58  , DestWholeExt(destWholeExt)
59  , DestExt(destExt)
60  , UseBlockingSend(0)
61  , UseBlockingRecv(0)
62  {
63  }
64 
69  vtkPPixelTransfer(int srcRank, const vtkPixelExtent& srcWholeExt, const vtkPixelExtent& targetExt,
70  int destRank, const vtkPixelExtent& destWholeExt, int id)
71  : Id(id)
72  , SrcRank(srcRank)
73  , SrcWholeExt(srcWholeExt)
74  , SrcExt(targetExt)
75  , DestRank(destRank)
76  , DestWholeExt(destWholeExt)
77  , DestExt(targetExt)
78  , UseBlockingSend(0)
79  , UseBlockingRecv(0)
80  {
81  }
82 
87  vtkPPixelTransfer(int srcRank, int destRank, const vtkPixelExtent& wholeExt,
88  const vtkPixelExtent& targetExt, int id = 0)
89  : Id(id)
90  , SrcRank(srcRank)
91  , SrcWholeExt(wholeExt)
92  , SrcExt(targetExt)
93  , DestRank(destRank)
94  , DestWholeExt(wholeExt)
95  , DestExt(targetExt)
96  , UseBlockingSend(0)
97  , UseBlockingRecv(0)
98  {
99  }
100 
105  vtkPPixelTransfer(int srcRank, int destRank, const vtkPixelExtent& ext, int id = 0)
106  : Id(id)
107  , SrcRank(srcRank)
108  , SrcWholeExt(ext)
109  , SrcExt(ext)
110  , DestRank(destRank)
111  , DestWholeExt(ext)
112  , DestExt(ext)
113  , UseBlockingSend(0)
114  , UseBlockingRecv(0)
115  {
116  }
117 
122  vtkPPixelTransfer(int srcRank, const vtkPixelExtent& srcWholeExt, int destRank,
123  const vtkPixelExtent& destWholeExt, int id = 0)
124  : Id(id)
125  , SrcRank(srcRank)
126  , SrcWholeExt(srcWholeExt)
127  , SrcExt(srcWholeExt)
128  , DestRank(destRank)
129  , DestWholeExt(destWholeExt)
130  , DestExt(destWholeExt)
131  , UseBlockingSend(0)
132  , UseBlockingRecv(0)
133  {
134  }
135 
141  vtkPPixelTransfer(const vtkPixelExtent& srcWholeExt, const vtkPixelExtent& srcExt,
142  const vtkPixelExtent& destWholeExt, const vtkPixelExtent& destExt)
143  : Id(0)
144  , SrcRank(0)
145  , SrcWholeExt(srcWholeExt)
146  , SrcExt(srcExt)
147  , DestRank(0)
148  , DestWholeExt(destWholeExt)
149  , DestExt(destExt)
150  , UseBlockingSend(0)
151  , UseBlockingRecv(0)
152  {
153  }
154 
155  ~vtkPPixelTransfer() = default;
156 
161  void SetSourceRank(int rank) { this->SrcRank = rank; }
162 
163  int GetSourceRank() const { return this->SrcRank; }
164 
165  void SetDestinationRank(int rank) { this->DestRank = rank; }
166 
167  int GetDestinationRank() const { return this->DestRank; }
168 
174  bool Sender(int rank) const { return (this->SrcRank == rank); }
175  bool Receiver(int rank) const { return (this->DestRank == rank); }
176  bool Local(int rank) const { return (this->Sender(rank) && this->Receiver(rank)); }
177 
182  void SetSourceWholeExtent(vtkPixelExtent& srcExt) { this->SrcWholeExt = srcExt; }
183 
184  vtkPixelExtent& GetSourceWholeExtent() { return this->SrcWholeExt; }
185 
186  const vtkPixelExtent& GetSourceWholeExtent() const { return this->SrcWholeExt; }
187 
192  void SetSourceExtent(vtkPixelExtent& srcExt) { this->SrcExt = srcExt; }
193 
194  vtkPixelExtent& GetSourceExtent() { return this->SrcExt; }
195 
196  const vtkPixelExtent& GetSourceExtent() const { return this->SrcExt; }
197 
202  void SetDestinationWholeExtent(vtkPixelExtent& destExt) { this->DestWholeExt = destExt; }
203 
204  vtkPixelExtent& GetDestinationWholeExtent() { return this->DestWholeExt; }
205 
206  const vtkPixelExtent& GetDestinationWholeExtent() const { return this->DestWholeExt; }
207 
212  void SetDestinationExtent(vtkPixelExtent& destExt) { this->DestExt = destExt; }
213 
214  vtkPixelExtent& GetDestinationExtent() { return this->DestExt; }
215 
216  const vtkPixelExtent& GetDestinationExtent() const { return this->DestExt; }
217 
221  void SetTransactionId(int id) { this->Id = id; }
222 
223  int GetTransactionId() const { return this->Id; }
224 
228  void SetUseBlockingSend(int val) { this->UseBlockingSend = val; }
229 
230  int GetUseBlockingSend() const { return this->UseBlockingSend; }
231 
232  void SetUseBlockingRecv(int val) { this->UseBlockingRecv = val; }
233 
234  int GetUseBlockingRecv() const { return this->UseBlockingRecv; }
235 
239  template <typename SOURCE_TYPE, typename DEST_TYPE>
240  int Execute(MPI_Comm comm, int rank, int nComps, SOURCE_TYPE* srcData, DEST_TYPE* destData,
241  std::vector<MPI_Request>& reqs, std::deque<MPI_Datatype>& types, int tag);
242 
247  int Execute(MPI_Comm comm, int rank, int nComps, int srcType, void* srcData, int destType,
248  void* destData, std::vector<MPI_Request>& reqs, std::deque<MPI_Datatype>& types, int tag);
249 
253  int Blit(int nComps, int srcType, void* srcData, int destType, void* destData);
254 
255 private:
256  // distpatch helper for vtk data type enum
257  template <typename SOURCE_TYPE>
258  int Execute(MPI_Comm comm, int rank, int nComps, SOURCE_TYPE* srcData, int destType,
259  void* destData, std::vector<MPI_Request>& reqs, std::deque<MPI_Datatype>& types, int tag);
260 
261  int Id; // transaction id
262  int SrcRank; // rank who owns source memory
263  vtkPixelExtent SrcWholeExt; // source extent
264  vtkPixelExtent SrcExt; // source subset to transfer
265  int DestRank; // rank who owns destination memory
266  vtkPixelExtent DestWholeExt; // destination extent
267  vtkPixelExtent DestExt; // destination subset
268  int UseBlockingSend; // controls for non-blocking comm
269  int UseBlockingRecv;
270 };
271 
272 //-----------------------------------------------------------------------------
273 template <typename SOURCE_TYPE>
274 int vtkPPixelTransfer::Execute(MPI_Comm comm, int rank, int nComps, SOURCE_TYPE* srcData,
275  int destType, void* destData, std::vector<MPI_Request>& reqs, std::deque<MPI_Datatype>& types,
276  int tag)
277 {
278  // second layer of dispatch
279  switch (destType)
280  {
281  vtkTemplateMacro(
282  return this->Execute(comm, rank, nComps, srcData, (VTK_TT*)destData, reqs, types, tag));
283  }
284  return 0;
285 }
286 
287 //-----------------------------------------------------------------------------
288 template <typename SOURCE_TYPE, typename DEST_TYPE>
289 int vtkPPixelTransfer::Execute(MPI_Comm comm, int rank, int nComps, SOURCE_TYPE* srcData,
290  DEST_TYPE* destData, std::vector<MPI_Request>& reqs, std::deque<MPI_Datatype>& types, int tag)
291 {
292  int iErr = 0;
293  if ((comm == MPI_COMM_NULL) || (this->Local(rank)))
294  {
295  // transaction is local, bypass mpi in favor of memcpy
296  return vtkPixelTransfer::Blit(this->SrcWholeExt, this->SrcExt, this->DestWholeExt,
297  this->DestExt, nComps, srcData, nComps, destData);
298  }
299 
300  if (rank == this->DestRank)
301  {
302  // use mpi to receive the data
303  if (destData == nullptr)
304  {
305  return -1;
306  }
307 
308  MPI_Datatype subarray;
309  iErr = vtkMPIPixelViewNew<DEST_TYPE>(this->DestWholeExt, this->DestExt, nComps, subarray);
310  if (iErr)
311  {
312  return -4;
313  }
314 
315  if (this->UseBlockingRecv)
316  {
317  MPI_Status stat;
318  iErr = MPI_Recv(destData, 1, subarray, this->SrcRank, tag, comm, &stat);
319  }
320  else
321  {
322  reqs.push_back(MPI_REQUEST_NULL);
323  iErr = MPI_Irecv(destData, 1, subarray, this->SrcRank, tag, comm, &reqs.back());
324  }
325 
326 #define HOLD_RECV_TYPES
327 #ifdef HOLD_RECV_YPES
328  types.push_back(subarray);
329 #else
330  MPI_Type_free(&subarray);
331 #endif
332 
333  if (iErr)
334  {
335  return -5;
336  }
337  }
338 
339  if (rank == this->SrcRank)
340  {
341  // use mpi to send the data
342  if (srcData == nullptr)
343  {
344  return -1;
345  }
346 
347  MPI_Datatype subarray;
348  iErr = vtkMPIPixelViewNew<SOURCE_TYPE>(this->SrcWholeExt, this->SrcExt, nComps, subarray);
349  if (iErr)
350  {
351  return -2;
352  }
353 
354  if (this->UseBlockingSend)
355  {
356  iErr = MPI_Ssend(srcData, 1, subarray, this->DestRank, tag, comm);
357  }
358  else
359  {
360  MPI_Request req;
361  iErr = MPI_Isend(srcData, 1, subarray, this->DestRank, tag, comm, &req);
362 #define SAVE_SEND_REQS
363 #ifdef SAVE_SEND_REQS
364  reqs.push_back(req);
365 #else
366  MPI_Request_free(&req);
367 #endif
368  }
369 
370 #define HOLD_SEND_TYPES
371 #ifdef HOLD_SEND_TYPES
372  types.push_back(subarray);
373 #else
374  MPI_Type_free(&subarray);
375 #endif
376 
377  if (iErr)
378  {
379  return -3;
380  }
381  }
382 
383  return iErr;
384 }
385 
386 VTKRENDERINGPARALLELLIC_EXPORT
387 ostream& operator<<(std::ostream& os, const vtkPPixelTransfer& gt);
388 
389 VTK_ABI_NAMESPACE_END
390 #endif
391 // VTK-HeaderTest-Exclude: vtkPPixelTransfer.h
vtkPixelExtent & GetSourceExtent()
void SetUseBlockingSend(int val)
Enable/diasable non-blocking communication.
vtkPPixelTransfer(int srcRank, int destRank, const vtkPixelExtent &ext, int id=0)
Initialize a transaction from sub extent of source to sub extent of dest, both the whole and the subs...
const vtkPixelExtent & GetDestinationExtent() const
void SetDestinationRank(int rank)
int GetSourceRank() const
void SetSourceWholeExtent(vtkPixelExtent &srcExt)
Set/Get the source extent.
vtkPixelExtent & GetDestinationWholeExtent()
vtkPPixelTransfer(int srcRank, const vtkPixelExtent &srcWholeExt, const vtkPixelExtent &srcExt, int destRank, const vtkPixelExtent &destWholeExt, const vtkPixelExtent &destExt, int id=0)
Initialize a transaction from sub extent of source to sub extent of dest, where the subsets are diffe...
int GetDestinationRank() const
int GetUseBlockingRecv() const
class to handle inter-process communication of pixel data from non-contiguous regions of a shared ind...
vtkPixelExtent & GetSourceWholeExtent()
void SetUseBlockingRecv(int val)
void SetSourceExtent(vtkPixelExtent &srcExt)
Set/Get the source extent.
vtkPPixelTransfer(int srcRank, int destRank, const vtkPixelExtent &wholeExt, const vtkPixelExtent &targetExt, int id=0)
Initialize a transaction from sub extent of source to sub extent of dest, both the whole and the subs...
pixel extents
int GetTransactionId() const
const vtkPixelExtent & GetSourceExtent() const
static int Blit(const vtkPixelExtent &ext, int nComps, int srcType, void *srcData, int destType, void *destData)
for memory to memory transfers.
const vtkPixelExtent & GetSourceWholeExtent() const
VTKRENDERINGPARALLELLIC_EXPORT ostream & operator<<(std::ostream &os, const vtkPPixelTransfer &gt)
vtkPPixelTransfer(int srcRank, const vtkPixelExtent &srcWholeExt, const vtkPixelExtent &targetExt, int destRank, const vtkPixelExtent &destWholeExt, int id)
Initialize a transaction from sub extent of source to sub extent of dest, where the subsets are the s...
vtkPPixelTransfer(int srcRank, const vtkPixelExtent &srcWholeExt, int destRank, const vtkPixelExtent &destWholeExt, int id=0)
Initialize a transaction from whole extent of source to whole extent of dest, where source and destin...
bool Sender(int rank) const
Tests to determine a given rank's role in this transaction.
int Execute(MPI_Comm comm, int rank, int nComps, SOURCE_TYPE *srcData, DEST_TYPE *destData, std::vector< MPI_Request > &reqs, std::deque< MPI_Datatype > &types, int tag)
Transfer data from source to destination.
void SetDestinationWholeExtent(vtkPixelExtent &destExt)
Set/get the destination extent.
vtkPPixelTransfer(const vtkPixelExtent &srcWholeExt, const vtkPixelExtent &srcExt, const vtkPixelExtent &destWholeExt, const vtkPixelExtent &destExt)
Initialize a transaction from sub extent of source to sub extent of dest, where the subsets are diffe...
Representation of a cartesian pixel plane and common operations on it.
void SetSourceRank(int rank)
Set/Get the MPI rank of source and destination processes.
bool Local(int rank) const
int GetUseBlockingSend() const
void SetTransactionId(int id)
Set/get the transaction id.
bool Receiver(int rank) const
void SetDestinationExtent(vtkPixelExtent &destExt)
Set/get the destination extent.
const vtkPixelExtent & GetDestinationWholeExtent() const
vtkPixelExtent & GetDestinationExtent()