decomposition.hpp

Go to the documentation of this file.

//
//       ,---.     ,--,    .---.     ,--,    ,---.    .-. .-.
//       | .-'   .' .')   / .-. )  .' .'     | .-'    |  \| |
//       | `-.   |  |(_)  | | |(_) |  |  __  | `-.    |   | |
//       | .-'   \  \     | | | |  \  \ ( _) | .-'    | |\  |
//       |  `--.  \  `-.  \ `-' /   \  `-) ) |  `--.  | | |)|
//       /( __.'   \____\  )---'    )\____/  /( __.'  /(  (_)
//      (__)              (_)      (__)     (__)     (__)
//
//  This file is part of ECOGEN.
//
//  ECOGEN is the legal property of its developers, whose names
//  are listed in the copyright file included with this source
//  distribution.
//
//  ECOGEN is free software: you can redistribute it and/or modify
//  it under the terms of the GNU General Public License as published
//  by the Free Software Foundation, either version 3 of the License,
//  or (at your option) any later version.
//
//  ECOGEN is distributed in the hope that it will be useful,
//  but WITHOUT ANY WARRANTY; without even the implied warranty of
//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
//  GNU General Public License for more details.
//
//  You should have received a copy of the GNU General Public License
//  along with ECOGEN (file LICENSE).
//  If not, see <http://www.gnu.org/licenses/>.
 
#ifndef INCLUDED_DECOMP_HPP
#define INCLUDED_DECOMP_HPP
 
#include <algorithm>
#include <numeric>
#include <functional>
#include "key.hpp"
#include <map>
 
namespace decomposition
{
 
class Decomposition
{
 
  public:
    static constexpr int Dim = 3;
    using key_type           = Key<Dim>;
 
  public: //Ctors
    Decomposition()                                            = default;
    Decomposition(const Decomposition& /*other*/)              = default;
    Decomposition(Decomposition&& /*other*/)                   = default;
    Decomposition& operator=(const Decomposition& /*other*/) & = default;
    Decomposition& operator=(Decomposition&& /*other*/) &      = default;
    ~Decomposition()                                           = default;
 
    Decomposition(std::array<int, Dim> _nCells) : nCells_global_(_nCells) {}
 
    void printDomainDecomposition(std::ofstream& fileStream) const noexcept
    {
      fileStream << key_rank_map_.size() << " ";
      auto it = key_rank_map_.begin();
      while (it != key_rank_map_.end()) {
        fileStream << it->first.getIndex() << " " << it->second << " ";
        ++it;
      }
    }
 
    void readDomainDecomposition(std::ifstream& fileStream) noexcept
    {
      int sizeKeyRankMap(0), rank(0);
      fileStream >> sizeKeyRankMap;
      typename key_type::value_type key(0);
      for (int i = 0; i < sizeKeyRankMap; ++i) {
        fileStream >> key;
        fileStream >> rank;
        key_rank_map_.emplace(key, rank);
      }
    }
 
    void updatePhysicalDomainSizes(std::array<int, Dim> _nCells) noexcept { nCells_global_ = _nCells; }
 
    std::vector<key_type> initialize(int nProcs, int _rank, int resumeSimulation) noexcept
    {
      std::vector<key_type> keys;
 
      if (resumeSimulation == 0) {
        auto nCells_t = std::accumulate(nCells_global_.begin(), nCells_global_.end(), 1, std::multiplies<int>());
 
        float chunks = static_cast<float>(nCells_t + 0.5) / nProcs;
        key_type key(0, 0, 0);
        std::vector<int> nCells_per_rank;
        for (int i = 0; i < nProcs; ++i) {
          size_t start     = (size_t)(i * chunks);
          size_t end       = std::min(static_cast<int>((i + 1) * chunks), nCells_t);
          const int nlocal = end - start;
          int count        = 0;
          key_rank_map_.emplace(key, i);
          nCells_per_rank.emplace_back(nlocal);
 
          while (count < nlocal) {
            if (is_valid(key)) ++count;
            ++key;
          }
 
          //Store also end, to check validity:
          if (i == nProcs - 1) {
            nCells_per_rank.emplace_back(0);
            key_rank_map_.emplace(--key, i);
          }
        }
 
        auto key_it = key_rank_map_.begin();
        std::advance(key_it, _rank);
        key         = key_it->first;
        auto nCells = nCells_per_rank[_rank];
        keys.resize(nCells);
 
        for (int i = 0; i < nCells; ++i) {
          bool valid = false;
          while (!valid) {
            if (is_valid(key)) {
              valid   = true;
              keys[i] = key;
            }
            ++key;
          }
        }
      }
      else {
        auto map_key      = key_rank_map_.begin();
        auto map_key_next = std::next(map_key);
        while (map_key->first != key_rank_map_.rbegin()->first) {
          if (map_key->second == _rank) {
            auto key = map_key->first;
            while (key != map_key_next->first) {
              if (is_valid(key) || (key == key_rank_map_.rbegin()->first)) {
                keys.push_back(key);
              }
              ++key;
            }
          }
          ++map_key;
          ++map_key_next;
        }
      }
 
      return keys;
    }
 
    bool areConsecutive(key_type _key0, key_type _key1)
    {
      while (true) {
        ++_key0;
        if (is_valid(_key0)) {
          return _key0 == _key1;
        }
      }
    }
 
    int get_rank(const key_type& _key)
    {
      auto range = key_rank_map_.equal_range(_key);
      if (range.first->first == range.second->first) {
        auto it = range.first;
        return (--it)->second;
      }
      else {
        return range.first->second;
      }
    }
 
    bool is_valid(const key_type& _key) const noexcept
    {
      return (_key.coordinate()[0] < nCells_global_[0] && _key.coordinate()[1] < nCells_global_[1] && _key.coordinate()[2] < nCells_global_[2]);
    }
 
    template <class Coord> bool is_inside(const Coord& _coord)
    {
      for (int d = 0; d < Dim; ++d) {
        if (_coord[d] < 0 || _coord[d] >= nCells_global_[d]) return false;
      }
      return true;
    }
 
    void recombineStarts()
    {
      auto it = key_rank_map_.begin(), itPrev = it++;
      while (it != --(key_rank_map_.end())) {
        if (it->second == itPrev->second) {
          it = key_rank_map_.erase(it);
        }
        else {
          itPrev = it;
          ++it;
        }
      }
    }
 
    void communicateMaps(int _nCpu, std::vector<typename key_type::value_type>& localKeys, std::vector<int>& localRanks)
    {
      //Decompose local map into keys and values (ranks)
      int localMapSize = localKeys.size();
 
      //All gather sizes of maps
      std::vector<int> mapSizes(_nCpu);
      MPI_Allgather(&localMapSize, 1, MPI_INT, &mapSizes[0], 1, MPI_INT, MPI_COMM_WORLD);
 
      //All gather maps (keys and ranks)
      std::vector<int> displacements(_nCpu);
      int sumMapSizes = 0;
      for (int i = 0; i < _nCpu; ++i) {
        displacements[i]  = sumMapSizes;
        sumMapSizes      += mapSizes[i];
      }
      std::vector<typename key_type::value_type> globalKeys(sumMapSizes);
      std::vector<int> globalRanks(sumMapSizes);
      MPI_Allgatherv(
        &localKeys[0], localMapSize, MPI_UNSIGNED_LONG_LONG, &globalKeys[0], &mapSizes[0], &displacements[0], MPI_UNSIGNED_LONG_LONG, MPI_COMM_WORLD);
      MPI_Allgatherv(&localRanks[0], localMapSize, MPI_INT, &globalRanks[0], &mapSizes[0], &displacements[0], MPI_INT, MPI_COMM_WORLD);
 
      //Compose global map from keys and ranks
      auto keyRankEnd = *(key_rank_map_.rbegin());
      key_rank_map_.clear();
      key_rank_map_.emplace(keyRankEnd);
      for (std::size_t i = 0; i < globalKeys.size(); ++i) {
        key_rank_map_.emplace(globalKeys[i], globalRanks[i]);
      }
    }
 
  private:
    std::array<int, Dim> nCells_global_;
    std::map<key_type, int> key_rank_map_;
};
} //namespace decomposition
 
#endif