communicator.h

Go to the documentation of this file.

// LIC// ====================================================================
// LIC// This file forms part of oomph-lib, the object-oriented,
// LIC// multi-physics finite-element library, available
// LIC// at http://www.oomph-lib.org.
// LIC//
// LIC// Copyright (C) 2006-2023 Matthias Heil and Andrew Hazel
// LIC//
// LIC// This library is free software; you can redistribute it and/or
// LIC// modify it under the terms of the GNU Lesser General Public
// LIC// License as published by the Free Software Foundation; either
// LIC// version 2.1 of the License, or (at your option) any later version.
// LIC//
// LIC// This library is distributed in the hope that it will be useful,
// LIC// but WITHOUT ANY WARRANTY; without even the implied warranty of
// LIC// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// LIC// Lesser General Public License for more details.
// LIC//
// LIC// You should have received a copy of the GNU Lesser General Public
// LIC// License along with this library; if not, write to the Free Software
// LIC// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
// LIC// 02110-1301  USA.
// LIC//
// LIC// The authors may be contacted at oomph-lib@maths.man.ac.uk.
// LIC//
// LIC//====================================================================
#ifndef OOMPH_COMMUNICATOR_CLASS_HEADER
#define OOMPH_COMMUNICATOR_CLASS_HEADER
 
// Config header generated by autoconfig
#ifdef HAVE_CONFIG_H
#include <oomph-lib-config.h>
#endif
 
// MPI headers
#ifdef OOMPH_HAS_MPI
#include "mpi.h"
#endif
 
// Oomph-lib error handler
#include "oomph_definitions.h"
#include "oomph_utilities.h"
 
namespace oomph
{
  template<class T>
  class DenseMatrix;
 
  //=============================================================================
  /// An oomph-lib wrapper to the MPI_Comm communicator object. Just
  /// contains an MPI_Comm object (which is a pointer) and wrappers to
  /// the MPI_... methods.
  //=============================================================================
  class OomphCommunicator
  {
  public:
#ifdef OOMPH_HAS_MPI
    /// Construct a communicator from a MPI_Comm object.
    /// if the bool owns_mpi_comm is true then this communicator is responsible
    /// for the destruction of the mpi_communicator. The mpi comm will be freed
    /// when the destructor is called.
    OomphCommunicator(const MPI_Comm mpi_communicator,
                      const bool& owns_mpi_comm = false)
      : Serial_communicator(false)
    {
      // store a pointer to the communicator
      Comm = mpi_communicator;
 
      // hold owns_mpi_comm
      Owns_mpi_comm = owns_mpi_comm;
    }
#endif
 
    /// Serial constructor
    OomphCommunicator()
#ifdef OOMPH_HAS_MPI
      : Owns_mpi_comm(false), Serial_communicator(true)
#endif
    {
    }
 
    /// Copy constructor.
    OomphCommunicator(const OomphCommunicator& communicator)
#ifdef OOMPH_HAS_MPI
      : Owns_mpi_comm(false)
    {
      if (communicator.serial_communicator())
      {
        Serial_communicator = true;
      }
      else
      {
        Comm = communicator.mpi_comm();
        Serial_communicator = false;
      }
    }
#else
    {
    }
#endif
 
    /// Pointer (copy) constructor.
    OomphCommunicator(const OomphCommunicator* communicator_pt)
#ifdef OOMPH_HAS_MPI
      : Owns_mpi_comm(false)
    {
      if (communicator_pt->serial_communicator())
      {
        Serial_communicator = true;
      }
      else
      {
        Comm = communicator_pt->mpi_comm();
        Serial_communicator = false;
      }
    }
#else
    {
    }
#endif
 
    /// Destructor. If MPI and this preconditioner owns the MPI_Comm
    /// object then MPI_Comm_free is called, otherwise nothing happens as
    /// the destruction of the underlying MPI_Comm object is the responsibility
    /// of another communicator.
    ~OomphCommunicator()
    {
#ifdef OOMPH_HAS_MPI
      if (Owns_mpi_comm)
      {
        MPI_Comm_free(&Comm);
      }
#endif
    }
 
    /// assignment operator
    void operator=(const OomphCommunicator& communicator)
    {
#ifdef OOMPH_HAS_MPI
      if (Owns_mpi_comm)
      {
        MPI_Comm_free(&Comm);
      }
      Owns_mpi_comm = false;
      if (communicator.serial_communicator())
      {
        Serial_communicator = true;
      }
      else
      {
        Serial_communicator = false;
        Comm = communicator.mpi_comm();
      }
#endif
    }
 
    /// number of processors
    int nproc() const
    {
#ifdef OOMPH_HAS_MPI
      if (Serial_communicator)
      {
        return 1;
      }
      else
      {
        int n_proc = 1;
        MPI_Comm_size(Comm, &n_proc);
        return n_proc;
      }
#else
      return 1;
#endif
    }
 
    /// my rank
    int my_rank() const
    {
#ifdef OOMPH_HAS_MPI
      if (Serial_communicator)
      {
        return 0;
      }
      else
      {
        int My_rank = 0;
        MPI_Comm_rank(Comm, &My_rank);
        return My_rank;
      }
#else
      return 0;
#endif
    }
 
    /// == operator - only returns true if communicators are MPI_IDENT,
    /// i.e. if both group and context are the same
    bool operator==(const OomphCommunicator& other_comm) const
    {
#ifdef OOMPH_HAS_MPI
      if (Serial_communicator != other_comm.serial_communicator())
      {
        return false;
      }
      else if (Serial_communicator)
      {
        return true;
      }
      else
      {
        int flag;
        MPI_Comm_compare(Comm, other_comm.mpi_comm(), &flag);
        if (flag == MPI_IDENT)
        {
          return true;
        }
        return false;
      }
#else
      return true;
#endif
    }
 
    /// != operator returns !(==operator) (see ==operator for more
    /// details)
    bool operator!=(const OomphCommunicator& other_comm) const
    {
      return !(*this == other_comm);
    }
 
#ifdef OOMPH_HAS_MPI
    /// split the communicator: color is an integer label for the sub
    /// group of processors. key is the rank in this sub group
    OomphCommunicator* split(const int& color, const int& key)
    {
#ifdef PARANOID
      if (Serial_communicator)
      {
        std::ostringstream error_message_stream;
        error_message_stream << "Attempted to split a serial communicator.";
        throw OomphLibError(error_message_stream.str(),
                            OOMPH_CURRENT_FUNCTION,
                            OOMPH_EXCEPTION_LOCATION);
      }
#endif
 
      // pointer for the split oomph-lib communicator
      OomphCommunicator* split_comm_pt;
 
      // the pointer for the new mpi communicator
      MPI_Comm* mpi_comm_pt = new MPI_Comm;
 
      // get the split communicator
      MPI_Comm_split(Comm, color, key, mpi_comm_pt);
 
      // assemble the new oomph-lib ocmmunicator
      // the new oomph-lib communicator owns the MPI_Comm at mpi_comm_pt
      // and is responsible for its destruction
      split_comm_pt = new OomphCommunicator(*mpi_comm_pt, true);
 
      // and return
      return split_comm_pt;
    }
#endif
 
 
#ifdef OOMPH_HAS_MPI
    /// access function to the underlying MPI_Comm object
    MPI_Comm mpi_comm() const
    {
#ifdef PARANOID
      if (Serial_communicator)
      {
        std::ostringstream error_message_stream;
        error_message_stream
          << "Requested the MPI_Comm object for a serial communicator.";
        throw OomphLibError(error_message_stream.str(),
                            OOMPH_CURRENT_FUNCTION,
                            OOMPH_EXCEPTION_LOCATION);
      }
#endif
      return Comm;
    }
 
 
    /// Access function (read only) to the serial communicator flag
    bool serial_communicator() const
    {
      return Serial_communicator;
    }
 
    /// broadcast function for Vector<int>
    void broadcast(const int& source, Vector<int>& x)
    {
      int n;
 
      // Get number of entries on processor source (where the vector exists)
      unsigned long n_long;
      if (this->my_rank() == source)
      {
        n_long = x.size();
 
        // Convert to int
        n = static_cast<int>(n_long);
      }
 
      // Broadcast to everybody how many entries to expect
      MPI_Bcast(&n, 1, MPI_INT, source, this->mpi_comm());
 
      // Resize Vector everywhere else in preparation
      if (this->my_rank() != source)
      {
        x.resize(n);
      }
 
      // Broadcast the Vector directly
      MPI_Bcast(&x[0], n, MPI_INT, source, this->mpi_comm());
    }
 
    /// broadcast function for Vector<double>
    void broadcast(const int& source, Vector<double>& x)
    {
      int n;
 
      // Get number of entries on processor source (where the vector exists)
      unsigned long n_long;
      if (this->my_rank() == source)
      {
        n_long = x.size();
 
        // Convert to int
        n = static_cast<int>(n_long);
      }
 
      // Broadcast to everybody how many entries to expect
      MPI_Bcast(&n, 1, MPI_INT, source, this->mpi_comm());
 
      // Resize Vector everywhere else in preparation
      if (this->my_rank() != source)
      {
        x.resize(n);
      }
 
      // Broadcast the Vector directly
      MPI_Bcast(&x[0], n, MPI_DOUBLE, source, this->mpi_comm());
    }
 
    /// broadcast function for DenseMatrix<double>
    void broadcast(const int& source, DenseMatrix<double>& x);
 
#endif
 
  private:
#ifdef OOMPH_HAS_MPI
    /// the MPI_Comm communicator
    MPI_Comm Comm;
 
    /// boolean indiacting whether this communicator owns the underlying
    /// MPI_Comm - if so the destructor will free
    /// the communicator
    bool Owns_mpi_comm;
 
    /// boolean to indicate if this communicator is for serial problems.
    /// This is true when serial codes are compiled under MPI
    bool Serial_communicator;
 
#endif
  }; // end of class Communicator
} // namespace oomph
#endif