linear_algebra_distribution.h
Go to the documentation of this file.
1// LIC// ====================================================================
2// LIC// This file forms part of oomph-lib, the object-oriented,
3// LIC// multi-physics finite-element library, available
4// LIC// at http://www.oomph-lib.org.
5// LIC//
6// LIC// Copyright (C) 2006-2022 Matthias Heil and Andrew Hazel
7// LIC//
8// LIC// This library is free software; you can redistribute it and/or
9// LIC// modify it under the terms of the GNU Lesser General Public
10// LIC// License as published by the Free Software Foundation; either
11// LIC// version 2.1 of the License, or (at your option) any later version.
12// LIC//
13// LIC// This library is distributed in the hope that it will be useful,
14// LIC// but WITHOUT ANY WARRANTY; without even the implied warranty of
15// LIC// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16// LIC// Lesser General Public License for more details.
17// LIC//
18// LIC// You should have received a copy of the GNU Lesser General Public
19// LIC// License along with this library; if not, write to the Free Software
20// LIC// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
21// LIC// 02110-1301 USA.
22// LIC//
23// LIC// The authors may be contacted at oomph-lib@maths.man.ac.uk.
24// LIC//
25// LIC//====================================================================
26#ifndef OOMPH_LINEAR_ALGEBRA_DISTRIBUTION_CLASS_HEADER
27#define OOMPH_LINEAR_ALGEBRA_DISTRIBUTION_CLASS_HEADER
28
29// Config header generated by autoconfig
30#ifdef HAVE_CONFIG_H
31#include <oomph-lib-config.h>
32#endif
33
34// MPI headers
35#ifdef OOMPH_HAS_MPI
36#include "mpi.h"
37#endif
38
39#include <ostream>
40
41// General headers
42#include "Vector.h"
43#include "communicator.h"
44#include "oomph_utilities.h"
45
46
47namespace oomph
48{
49 //=============================================================================
50 /// Describes the distribution of a distributable linear algebra type
51 /// object. Typically this is a container (such as a DoubleVector) or an
52 /// operator (e.g Preconditioner or LinearSolver).
53 /// This object is used in both serial and parallel implementations. In the
54 /// serial context (no MPI) this just contains an integer indicating
55 /// the number of rows.
56 /// In parallel either each processor holds a subset of the set of global
57 /// rows. (each processor contains only a single continuous block of rows -
58 /// parametised with variables denoting the first row and the number of local
59 /// rows) or, all rows are be duplicated across all processors.
60 /// In parallel this object also contains an OomphCommunicator object which
61 /// primarily contains the MPI_Comm communicator associated with this object.
62 //=============================================================================
64 {
65 public:
66 /// Default Constructor - creates a Distribution that has not been
67 /// setup
69
70 /// Constructor. Takes the first_row, nrow_local (both for this
71 /// processor) and nrow as arguments. If nrow is not provided
72 /// or equal to 0 then it will be computed automatically
74 const unsigned& first_row_,
75 const unsigned& n_row_local,
76 const unsigned& n_row = 0)
77 : Comm_pt(0)
78 {
79 this->build(&comm, first_row_, n_row_local, n_row);
80 };
81
82 /// Constructor. Takes the number of global rows and uniformly
83 /// distributes them over the processors if distributed = true (default),
84 /// if distributed = false then every row is duplicated on every processor
86 const unsigned& n_row,
87 const bool& distributed_ = true)
88 : Comm_pt(0)
89 {
90 this->build(&comm, n_row, distributed_);
91 };
92
93 /// Constructor. Takes the first_row, nrow_local (both for this
94 /// processor) and nrow as arguments. If nrow is not provided
95 /// or equal to 0 then it will be computed automatically
97 const unsigned& first_row_,
98 const unsigned& n_row_local,
99 const unsigned& n_row = 0)
100 : Comm_pt(0)
101 {
102 this->build(comm_pt, first_row_, n_row_local, n_row);
103 };
104
105 /// Constructor. Takes the number of global rows and uniformly
106 /// distributes them over the processors if distributed = true (default),
107 /// if distributed = false then every row is duplicated on every processor
109 const unsigned& n_row,
110 const bool& distributed_ = true)
111 : Comm_pt(0)
112 {
113 this->build(comm_pt, n_row, distributed_);
114 };
115
116 /// Copy Constructor.
118 : Comm_pt(0)
119 {
120 this->build(old_dist);
121 }
122
123 /// pointer based copy constructor
125 : Comm_pt(0)
126 {
127 this->build(old_dist_pt);
128 }
129
130 /// Destructor
132 {
133 delete Comm_pt;
134 }
135
136 /// Assignment Operator
138 {
139 this->build(old_dist);
140 }
141
142 /// Sets the distribution. Takes first_row, nrow_local and
143 /// nrow as arguments. If nrow is not provided or equal to
144 /// 0 then it is computed automatically
145 void build(const OomphCommunicator* const comm_pt,
146 const unsigned& first_row,
147 const unsigned& nrow_local,
148 const unsigned& nrow = 0);
149
150 /// Build the LinearAlgebraDistribution. if distributed = true
151 /// (default) then uniformly distribute nrow over all processors where
152 /// processors 0 holds approximately the first nrow/n_proc, processor
153 /// 1 holds the next nrow/n_proc and so on... or if distributed = false
154 /// then every row is held on every processor
155 void build(const OomphCommunicator* const comm_pt,
156 const unsigned& nrow,
157 const bool& distributed = true);
158
159 /// Copy the argument distribution.
160 /// Also a helper method for the =assignment operator and copy constructor
161 void build(const LinearAlgebraDistribution& new_dist);
162
163 /// Copy the argument distribution.
164 /// Also a helper method for the =assignment operator and copy constructor
165 void build(const LinearAlgebraDistribution* new_dist_pt)
166 {
167 this->build(*new_dist_pt);
168 }
169
170 /// clears the distribution
171 void clear()
172 {
173 // delete the communicator
174 delete Comm_pt;
175 Comm_pt = 0;
176
177 // delete first_row and nrow_local
178 First_row.clear();
179 Nrow_local.clear();
180
181 // zero Nrow
182 Nrow = 0;
183 }
184
185 /// access function to the number of global rows.
186 unsigned nrow() const
187 {
188 return Nrow;
189 }
190
191 /// access function for the num of local rows on this processor. If
192 /// no MPI then Nrow is returned.
193 unsigned nrow_local() const
194 {
195 // return the nrow_local
196#ifdef OOMPH_HAS_MPI
197 if (Distributed)
198 {
199#ifdef PARANOID
200 if (Comm_pt == 0)
201 {
202 throw OomphLibError(
203 "LinearAlgebraDistribution has not been built : Comm_pt == 0.",
204 "LinearAlgebraDistribution::nrow_local()",
205 OOMPH_EXCEPTION_LOCATION);
206 }
207#endif
208
209 return Nrow_local[Comm_pt->my_rank()];
210 }
211 else
212 {
213 return Nrow;
214 }
215#else
216 return Nrow;
217#endif
218 }
219
220 /// access function for the num of local rows on this processor. If
221 /// no MPI the nrow is returned
222 unsigned nrow_local(const unsigned& p) const
223 {
224#ifdef PARANOID
225 if (Comm_pt == 0)
226 {
227 throw OomphLibError(
228 "LinearAlgebraDistribution has not been built : Comm_pt == 0.",
229 OOMPH_CURRENT_FUNCTION,
230 OOMPH_EXCEPTION_LOCATION);
231 }
232 if (p >= unsigned(Comm_pt->nproc()))
233 {
234 std::ostringstream error_message;
235 error_message << "Requested nrow_local(" << p
236 << "), but this distribution is defined "
237 << "on " << Comm_pt->nproc() << "processors.";
238 throw OomphLibError(error_message.str(),
239 OOMPH_CURRENT_FUNCTION,
240 OOMPH_EXCEPTION_LOCATION);
241 }
242#endif
243
244 // return the nrow_local
245#ifdef OOMPH_HAS_MPI
246 if (Distributed)
247 {
248 return Nrow_local[p];
249 }
250 else
251 {
252 return Nrow;
253 }
254#else
255 return Nrow;
256#endif
257 }
258
259 /// access function for the first row on this processor. If not
260 /// distributed then this is just zero.
261 unsigned first_row() const
262 {
263 // return the first row
264#ifdef OOMPH_HAS_MPI
265 if (Distributed)
266 {
267#ifdef PARANOID
268 if (Comm_pt == 0)
269 {
270 throw OomphLibError(
271 "LinearAlgebraDistribution has not been built : Comm_pt == 0.",
272 OOMPH_CURRENT_FUNCTION,
273 OOMPH_EXCEPTION_LOCATION);
274 }
275#endif
276
277 return First_row[Comm_pt->my_rank()];
278 }
279 else
280 {
281 return 0;
282 }
283#else
284 return 0;
285#endif
286 }
287
288 /// access function for the first row on the p-th processor
289 unsigned first_row(const unsigned& p) const
290 {
291#ifdef PARANOID
292 if (Comm_pt == 0)
293 {
294 throw OomphLibError(
295 "LinearAlgebraDistribution has not been built : Comm_pt == 0.",
296 OOMPH_CURRENT_FUNCTION,
297 OOMPH_EXCEPTION_LOCATION);
298 }
299 if (p >= unsigned(Comm_pt->nproc()))
300 {
301 std::ostringstream error_message;
302 error_message << "Requested first_row(" << p
303 << "), but this distribution is defined "
304 << "on " << Comm_pt->nproc() << "processors.";
305 throw OomphLibError(error_message.str(),
306 OOMPH_CURRENT_FUNCTION,
307 OOMPH_EXCEPTION_LOCATION);
308 }
309
310#endif
311
312 // return the first row
313#ifdef OOMPH_HAS_MPI
314 if (Distributed)
315 {
316 return First_row[p];
317 }
318 else
319 {
320 return 0;
321 }
322#else
323 return 0;
324#endif
325 }
326
327 /// access function to the distributed - indicates whether the
328 /// distribution is serial or distributed
329 bool distributed() const
330 {
331 return Distributed;
332 }
333
334 /// const access to the communicator pointer
336 {
337 return Comm_pt;
338 }
339
340 /// if the communicator_pt is null then the distribution is not setup then
341 /// false is returned, otherwise return true
342 bool built() const
343 {
344 if (Comm_pt == 0)
345 {
346 return false;
347 }
348 return true;
349 }
350
351 /// == Operator
352 bool operator==(const LinearAlgebraDistribution& other_dist) const;
353
354 /// != operator
355 bool operator!=(const LinearAlgebraDistribution& other_dist) const
356 {
357 return !(*this == other_dist);
358 }
359
360 /// << operator
361 friend std::ostream& operator<<(std::ostream& stream,
363
364 /// return the local index corresponding to the global index
365 unsigned global_to_local_row_map(const unsigned& global_i) const
366 {
367#ifdef PARANOID
368 if (global_i >= Nrow)
369 {
370 throw OomphLibError(
371 "Requested global row outside the number of global rows",
372 OOMPH_CURRENT_FUNCTION,
373 OOMPH_EXCEPTION_LOCATION);
374 }
375#endif
376 int local_i = static_cast<int>(global_i);
377 int p = 0;
378 while ((int)(local_i - (int)nrow_local(p)) >= 0)
379 {
380 local_i -= (int)nrow_local(p);
381 p++;
382 }
383 return (unsigned)local_i;
384 }
385
386 /// return the processor rank of the global row number i
387 unsigned rank_of_global_row(const unsigned i) const
388 {
389 unsigned p = 0;
390 while (i < first_row(p) || i >= first_row(p) + nrow_local(p))
391 {
392 p++;
393 }
394 return p;
395 }
396
397 /// return the nrow_local Vector
399 {
400 return Nrow_local;
401 }
402
403 /// return the first_row Vector
405 {
406 return First_row;
407 }
408
409 private:
410 /// the number of global rows
411 unsigned Nrow;
412
413 /// the number of local rows on the processor
415
416 /// the first row on this processor
418
419 /// flag to indicate whether this distribution describes an object that is
420 /// distributed over the processors of Comm_pt (true) or duplicated over the
421 /// processors of Comm_pt (false)
423
424 /// the pointer to the MPI communicator object in this distribution
426 }; // end of LinearAlgebraDistribution
427
428
429 //=============================================================================
430 /// Base class for any linear algebra object that is distributable.
431 /// Just contains storage for the LinearAlgebraDistribution object and
432 /// access functions
433 //=============================================================================
435 {
436 public:
437 /// Default constructor - create a distribution
439 {
441 }
442
443 /// Broken copy constructor
445 const DistributableLinearAlgebraObject& matrix) = delete;
446
447 /// Broken assignment operator
449
450 /// Destructor
452 {
453 delete Distribution_pt;
454 }
455
456 /// access to the LinearAlgebraDistribution
458 {
459 return Distribution_pt;
460 }
461
462 /// access function to the number of global rows.
463 unsigned nrow() const
464 {
465 return Distribution_pt->nrow();
466 }
467
468 /// access function for the num of local rows on this processor.
469 unsigned nrow_local() const
470 {
471 return Distribution_pt->nrow_local();
472 }
473
474 /// access function for the num of local rows on this processor.
475 unsigned nrow_local(const unsigned& p) const
476 {
477 return Distribution_pt->nrow_local(p);
478 }
479
480 /// access function for the first row on this processor
481 unsigned first_row() const
482 {
483 return Distribution_pt->first_row();
484 }
485
486 /// access function for the first row on this processor
487 unsigned first_row(const unsigned& p) const
488 {
489 return Distribution_pt->first_row(p);
490 }
491
492 /// distribution is serial or distributed
493 bool distributed() const
494 {
496 }
497
498 /// if the communicator_pt is null then the distribution is not setup then
499 /// false is returned, otherwise return true
501 {
502 return Distribution_pt->built();
503 }
504
505 /// setup the distribution of this distributable linear algebra
506 /// object
508 {
509 Distribution_pt->build(dist_pt);
510 }
511
512 /// setup the distribution of this distributable linear algebra
513 /// object
515 {
516 Distribution_pt->build(dist);
517 }
518
519 protected:
520 /// clear the distribution of this distributable linear algebra
521 /// object
523 {
525 }
526
527 private:
528 /// the LinearAlgebraDistribution object
530 }; // end of DistributableLinearAlgebraObject
531
532 //=============================================================================
533 /// Namespace for helper functions for LinearAlgebraDistributions
534 //=============================================================================
535 namespace LinearAlgebraDistributionHelpers
536 {
537 /// Takes a vector of LinearAlgebraDistribution objects and
538 /// concatenates them such that the nrow_local of the out_distribution
539 /// is the sum of the nrow_local of all the in_distributions and the number
540 /// of global rows of the out_distribution is the sum of the number of
541 /// global rows of all the in_distributions. This results in a permutation
542 /// of the rows in the out_distribution. Think of this in terms of
543 /// DoubleVectors, if we have DoubleVectors with distributions A and B,
544 /// distributed across two processors (p0 and p1), A: [a0] (on p0) B:
545 /// [b0] (on p0)
546 /// [a1] (on p1) [b1] (on P1),
547 ///
548 /// then the out_distribution is
549 /// [a0 (on p0)
550 /// b0] (on p0)
551 /// [a1 (on p1)
552 /// b1] (on p1),
553 ///
554 /// as opposed to
555 /// [a0 (on p0)
556 /// a1] (on p0)
557 /// [b0 (on p1)
558 /// b1] (on p1).
559 ///
560 /// Note (1): The out_distribution may not be uniformly distributed even
561 /// if the in_distributions are uniform distributions.
562 /// Try this out with two distributions of global rows 3 and 5, uniformly
563 /// distributed across two processors. Compare this against a distribution
564 /// of global row 8 distributed across two processors.
565 ///
566 /// Note (2): There is no equivalent function which takes a Vector of
567 /// LinearAlgebraDistribution objects (as opposed to pointers), there should
568 /// not be one since we do not want to invoke the assignment operator when
569 /// creating the Vector of LinearAlgebraDistribution objects.
570 void concatenate(
571 const Vector<LinearAlgebraDistribution*>& in_distribution_pt,
572 LinearAlgebraDistribution& out_distribution);
573 } // namespace LinearAlgebraDistributionHelpers
574} // namespace oomph
575#endif
cstr elem_len * i
Definition: cfortran.h:603
Base class for any linear algebra object that is distributable. Just contains storage for the LinearA...
unsigned nrow_local(const unsigned &p) const
access function for the num of local rows on this processor.
void clear_distribution()
clear the distribution of this distributable linear algebra object
unsigned first_row(const unsigned &p) const
access function for the first row on this processor
bool distributed() const
distribution is serial or distributed
DistributableLinearAlgebraObject(const DistributableLinearAlgebraObject &matrix)=delete
Broken copy constructor.
unsigned nrow() const
access function to the number of global rows.
bool distribution_built() const
if the communicator_pt is null then the distribution is not setup then false is returned,...
LinearAlgebraDistribution * Distribution_pt
the LinearAlgebraDistribution object
unsigned nrow_local() const
access function for the num of local rows on this processor.
unsigned first_row() const
access function for the first row on this processor
LinearAlgebraDistribution * distribution_pt() const
access to the LinearAlgebraDistribution
void build_distribution(const LinearAlgebraDistribution *const dist_pt)
setup the distribution of this distributable linear algebra object
DistributableLinearAlgebraObject()
Default constructor - create a distribution.
void operator=(const DistributableLinearAlgebraObject &)=delete
Broken assignment operator.
void build_distribution(const LinearAlgebraDistribution &dist)
setup the distribution of this distributable linear algebra object
Describes the distribution of a distributable linear algebra type object. Typically this is a contain...
void operator=(const LinearAlgebraDistribution &old_dist)
Assignment Operator.
bool distributed() const
access function to the distributed - indicates whether the distribution is serial or distributed
unsigned first_row(const unsigned &p) const
access function for the first row on the p-th processor
LinearAlgebraDistribution(const OomphCommunicator *const comm_pt, const unsigned &first_row_, const unsigned &n_row_local, const unsigned &n_row=0)
Constructor. Takes the first_row, nrow_local (both for this processor) and nrow as arguments....
unsigned first_row() const
access function for the first row on this processor. If not distributed then this is just zero.
OomphCommunicator * communicator_pt() const
const access to the communicator pointer
bool operator==(const LinearAlgebraDistribution &other_dist) const
== Operator
Vector< unsigned > Nrow_local
the number of local rows on the processor
void build(const OomphCommunicator *const comm_pt, const unsigned &first_row, const unsigned &nrow_local, const unsigned &nrow=0)
Sets the distribution. Takes first_row, nrow_local and nrow as arguments. If nrow is not provided or ...
bool operator!=(const LinearAlgebraDistribution &other_dist) const
!= operator
OomphCommunicator * Comm_pt
the pointer to the MPI communicator object in this distribution
unsigned Nrow
the number of global rows
LinearAlgebraDistribution(const OomphCommunicator *const comm_pt, const unsigned &n_row, const bool &distributed_=true)
Constructor. Takes the number of global rows and uniformly distributes them over the processors if di...
LinearAlgebraDistribution(const OomphCommunicator &comm, const unsigned &n_row, const bool &distributed_=true)
Constructor. Takes the number of global rows and uniformly distributes them over the processors if di...
friend std::ostream & operator<<(std::ostream &stream, LinearAlgebraDistribution &dist)
<< operator
Vector< unsigned > First_row
the first row on this processor
unsigned global_to_local_row_map(const unsigned &global_i) const
return the local index corresponding to the global index
bool built() const
if the communicator_pt is null then the distribution is not setup then false is returned,...
bool Distributed
flag to indicate whether this distribution describes an object that is distributed over the processor...
Vector< unsigned > first_row_vector() const
return the first_row Vector
unsigned nrow_local(const unsigned &p) const
access function for the num of local rows on this processor. If no MPI the nrow is returned
unsigned nrow() const
access function to the number of global rows.
unsigned nrow_local() const
access function for the num of local rows on this processor. If no MPI then Nrow is returned.
unsigned rank_of_global_row(const unsigned i) const
return the processor rank of the global row number i
LinearAlgebraDistribution(const LinearAlgebraDistribution &old_dist)
Copy Constructor.
LinearAlgebraDistribution()
Default Constructor - creates a Distribution that has not been setup.
LinearAlgebraDistribution(const LinearAlgebraDistribution *old_dist_pt)
pointer based copy constructor
Vector< unsigned > nrow_local_vector() const
return the nrow_local Vector
void build(const LinearAlgebraDistribution *new_dist_pt)
Copy the argument distribution. Also a helper method for the =assignment operator and copy constructo...
LinearAlgebraDistribution(const OomphCommunicator &comm, const unsigned &first_row_, const unsigned &n_row_local, const unsigned &n_row=0)
Constructor. Takes the first_row, nrow_local (both for this processor) and nrow as arguments....
An oomph-lib wrapper to the MPI_Comm communicator object. Just contains an MPI_Comm object (which is ...
Definition: communicator.h:54
An OomphLibError object which should be thrown when an run-time error is encountered....
A slight extension to the standard template vector class so that we can include "graceful" array rang...
Definition: Vector.h:58
void concatenate(const Vector< LinearAlgebraDistribution * > &in_distribution_pt, LinearAlgebraDistribution &out_distribution)
Takes a vector of LinearAlgebraDistribution objects and concatenates them such that the nrow_local of...
//////////////////////////////////////////////////////////////////// ////////////////////////////////...