src/expde/grid/labbo.cc

Go to the documentation of this file.

//    expde: expression templates for partial differential equations.
//    Copyright (C) 2001  Christoph Pflaum
//    This program is free software; you can redistribute it and/or modify
//    it under the terms of the GNU General Public License as published by
//    the Free Software Foundation; either version 2 of the License, or
//    (at your option) any later version.
//
//    This program is distributed in the hope that it will be useful,
//    but WITHOUT ANY WARRANTY; without even the implied warranty of
//    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
//    GNU General Public License for more details.
//
//                 SEE  Notice1.doc made by 
//                 LAWRENCE LIVERMORE NATIONAL LABORATORY
//

// ------------------------------------------------------------
//
// labbo.cc
//
// ------------------------------------------------------------


// Include level 0:
#ifdef COMP_GNUOLD
 #include <iostream.h>
 #include <fstream.h>
 #include <time.h>
 #include <math.h>
#else
 #include <iostream>
 #include <fstream> 
 #include <ctime>
 #include <cmath>
#endif 


// Include level 0:
#include "../parser.h"

// Include level 1:
#include "../paramete.h"
#include "../abbrevi.h"
#include "../math_lib/math_lib.h"

// Include level 2:
#include "../basic/basic.h"

// Include level 3:
#include "../domain/domain.h"

// Include level 4:
#include "../formulas/boundy.h"
#include "../formulas/loc_sten.h"

// Include level 5:
#include "gpar.h"
#include "parallel.h"
#include "mgcoeff.h"
#include "sto_man.h"
#include "gridbase.h"
#include "grid.h"

// Memberfunctions for boundary label
// 1. at boundary points
// 2. for multigrid points
// 3. Parallelization

bool jetzt;


// 1. at boundary points


int Storage_manager::Give_number_label_bo() {
  int i;
  for(i=0;i<Max_label_number;++i) {
    if(occupied[i]==false) {
      occupied[i]=true;
      if(i>=max_num_label) {
        // erhoehe max_num_label
        increase_max_num_label = true;
        ++max_num_label;
      }
      else 
        // erhoehe nicht max_num_label
        increase_max_num_label = false;
      return i+1;
    }
  }
  cout << " Error: Maximal number of labels for the boundary reached! "
       << endl;
  return 0;
}

void Storage_manager::Remove_number_label_bo(int num) {
  if(developer_version) {
    if(num>Max_label_number || num<=0) {
      cout << " Label number wrong in Grid_base::Remove_number_label_bo! "
           << endl;
      return;
    }
  }
  occupied[num-1]=false;
}

bool Grid_base::Give_label_bo(Index3D Ind, dir_3D dir, int num) 
     const {
  static Point_hashtable3* point;
  point = hashtable3_start
    [hashtable3_function(Ind.ind_x.index,Ind.ind_y.index,
                         Ind.ind_z.index, dir, hashtable3_leng)];
  while(point!=NULL) {
    if(point->ind==Ind && point->direction==dir) return point->Give_label(num);
    point = point->next;
  }
  return point->Give_label(num);
}


/*
void Grid_base::Put_label_bo(bool lab, 
                             Index3D Ind, dir_3D dir, int num) 
     const {
  static Point_hashtable3* point;
  point = hashtable3_start
    [hashtable3_function(Ind.ind_x.index,Ind.ind_y.index,
                         Ind.ind_z.index, dir, hashtable3_leng)];
  while(point!=NULL) {
    if(point->ind==Ind && point->direction==dir) point->Put_label(lab,num);
    point = point->next;
  }
  point->Put_label(lab,num);
}
*/

void Grid_base::Put_label_bo(bool (*Formula)(double x,double y,double z),
                            int num) {
  D3vector Vl, Vm, Vr;
  iterate_hash3 {
    Vl=transform_coord_org_shift(bo2point->ind,bo2point->direction,-shift_eps);
    Vm=transform_coord_org_shift(bo2point->ind,bo2point->direction,0);
    Vr=transform_coord_org_shift(bo2point->ind,bo2point->direction,shift_eps);
    bo2point->Put_label(Formula(Vl.x,Vl.y,Vl.z)
                        || Formula(Vm.x,Vm.y,Vm.z)
                        || Formula(Vr.x,Vr.y,Vr.z),num);
  }
  Restrict_label_bo(num);
}

void Grid_base::Put_label_bo(int num_var, int num) {
  double epsi;
  epsi = finest_mesh_size() * eps_boundary_var;
  iterate_hash3 {
    bo2point->Put_label( bo2point->var[num_var] > epsi, num);
  }
  Restrict_label_bo(num);
}

void Grid_base::Put_label_bo_complementary(int num_org, int num) {
  iterate_hash3 {
    if(bo2point->Give_label(num_org)==true) 
      bo2point->Put_label(false,num);
    else 
      bo2point->Put_label(true,num);
  }
  Restrict_label_bo(num);
}

void Grid_base::Put_union_label(int num, int numa, int numb) {
  iterate_hash3 {
    bo2point->Put_label(bo2point->Give_label(numa) ||
                        bo2point->Give_label(numb),num);
  }
  Restrict_label_bo(num);
}


// 2. for multigrid points

/*
  How to construct and restrict label_bo:
  1. label_bo must be set at all corners of boundary cells
     on all levels, since res_op.h needs these informations
  2. on finest grid:
          - boundary points -> true or false
          - set all fine_bocells -> false
          - Zellfreiheitgrade, if one corner Dirichlet, then all of them
  3. recursion: all corners of edges, faces and volumnes
                must be set.
                -> true if Dirichlet and used for interpolation
                -> nothing else
     by this all coarse bocells have a label_bo
 */

void Grid_base::Restrict_label_bo(int num) {
  Index3D I, Inext;
  int i;
  bool Dirichlet;
  Pointtype type;

  if(I_am_active()) {
    // 1. Schritt: increase labels of multigrid points
    //             if max_num_label>1 and increase_max_num_label
    int *new_l, *old_l;
    // for multigrid points
    if(Give_max_num_label()>1 && Give_increase_max_num_label()) {
      iterate_hash1 {
        I = point1->Give_Index();
        type = Give_type(I);
        if(type==multigrid || type==parallel_p) {
          old_l = point1->label_level;
          if(old_l!=NULL) {
            new_l = new int[Give_max_num_label()];
            for(i=0;i<Give_max_num_label()-1;++i)
              new_l[i]=old_l[i];
            new_l[Give_max_num_label()-1] = 0;
            delete old_l;
            point1->label_level = new_l;
          }
        }
      }
    }
    
    if(Give_increase_max_num_label()) {
      // for Zellfreiheitgrade
      iterate_hash2 {
        if(bocell->Exists_bocellpoint()) {
          old_l = bocell->Give_label_level();
          new_l = new int[Give_max_num_label()];
          for(i=0;i<Give_max_num_label()-1;++i)
            new_l[i]=old_l[i];
          new_l[Give_max_num_label()-1] = 0;
          bocell->Set_label_level(new_l);
        }
      }
    }
    
    // 2. Schritt: study points on finest grid
    
    
    // for Rand first: dont_send
    iterate_hash1 {
      point1->send_label = dont_send;
    }
    
    
    // boundary points
    iterate_hash3 {
      I = bo2point->ind.next(bo2point->direction,max_level);
      // means: since one cannot run that boundary point -> Dirichlet
      Dirichlet = bo2point->Give_label(num)==false;
      type = Give_type(I);
      if(type==multigrid || type==parallel_p) {
        Set_label_bo_mg(I,num,Dirichlet,max_level);
      }
      Interpolation_label_bo(I,max_level,num,Dirichlet);
      Interpolation_label_bo(bo2point->ind,max_level,num,Dirichlet);
    }
    
    /*
      Send_label_parallel(max_level,num);
      Send_label_parallel(max_level-1,num);
      // for Rand first: dont_send
      iterate_hash1 {
      point1->send_label = dont_send;
      }
    */
    
    // Zellfreiheitgrade and all fine_bo_cells
    iterate_hash2 {
      I = bocell->Give_Index();
      // for all fine_bo_cells all corners must have a label_bo
      for(i=0;i<8;++i) {
        Inext = I.neighbour((dir_sons)i);
        type = Give_type(Inext);
        if(type==multigrid || type==parallel_p) {
        Set_label_bo_mg(Inext,num,false,max_level);
        }
      }
    }
    
    // Send_label_parallel(max_level,num);
    if(parallel_version)
      Send_label_parallel(max_level-1,num);
    
    /*
    // for Rand first: dont_send
    iterate_hash1 {
      point1->send_label = dont_send;
    }
    */

    iterate_hash2 {
      // Zellfreiheitgrad
      if(bocell->Exists_bocellpoint()) {
        Dirichlet = false;
        // calc Dirichlet cell or not
        for(i=0;i<8;++i) {
          Inext = I.neighbour((dir_sons)i);
          type = Give_type(Inext);
          if(type==multigrid || type==parallel_p)
            if(Give_label_bo_D(Inext,num,max_level)==true)
              Dirichlet = true;
        }
        // set type D-N at neighbour points
        for(i=0;i<8;++i) {
          Inext = I.neighbour((dir_sons)i);
          type = Give_type(Inext);
          if(type==multigrid || type==parallel_p) {
            Set_label_bo_mg(Inext,num,Dirichlet,max_level);
            Interpolation_label_bo(Inext, max_level, num, Dirichlet);
          }
        }
        if(Dirichlet)
          bocell->Set_Dirichlet(num);
      }
    }
    
    if(parallel_version)
      Send_label_parallel(max_level-1,num);
    
    // 3. Schritt: rekusive from fine to coarse
    int level;
    Index3D my_lev_index;
    
    for(level = max_level-1;
        level >= Give_my_coarsest_level() && level >1;
        --level) {

      /*
      // for Rand first: dont_send
      iterate_hash1 {
        point1->send_label = dont_send;
      }
      */

      my_lev_index = Give_my_level_index(level+1);
      iterate_hash1 {
        I = point1->Give_Index();
        if(I << my_lev_index)
          //if(point1->Give_typ()==multigrid && I.Tiefe()==level) {
          if(point1->global_typ==multigrid && I.Tiefe()==level) {
            if(point1->finest_level>=level) {
              if(point1->label_level!=NULL) {
                Dirichlet = point1->Give_label(num)>=level;
                Interpolation_label_bo(I, level, num, Dirichlet);
              }
              else {
                cout << " error 1 in labbo.cc " << endl;
              }
            }
          }
      }
      
     
      if(parallel_version) {
        //      Send_label_parallel(level,num);
        Send_label_parallel(level-1,num);
      }
    }
    
    // 5. Schritt: 
    No_increase_max_num_label();

    // 6.Schritt:  Test ob alles ok.
    iterate_hash1 {
      if(point1->global_typ==multigrid &&
         point1->ind.Tiefe()<max_level && 
         point1->label_level==NULL) {
        cout << " error 2 in labbo.cc "
                 << " x: " << point1->ind.I_x().get()
                 << " y: " << point1->ind.I_y().get()
                 << " z: " << point1->ind.I_z().get()
                 << " my_rank: " << my_rank
                 << endl;
        point1->global_typ = exterior;
      }
    }
  }
}


void Grid_base::Interpolation_label_bo(Index3D I, int level, int num, 
                                       bool Dirichlet) {



/*
if((my_rank==9) && (I == Index3D(0,4,4))) {
  cout <<  " \n \n jezt \n \n " << endl;
}

if((my_rank==15) && (I == Index3D(0,4,8))) {
  cout <<  " \n \n jezt2 \n \n " << endl;
}
if((my_rank==15) && (I == Index3D(64,4,67))) {
  cout <<  " \n \n jezt3 \n \n " << endl;
}
*/

  if(I.I_x().Tiefe()==level) {
    if(I.I_y().Tiefe()==level) {
      if(I.I_z().Tiefe()==level) {
        // middle point
        // edge between EST-WND
        Set_label_bo_mg(I.next_EST(level),num,Dirichlet,level-1);
        Set_label_bo_mg(I.next_WND(level),num,Dirichlet,level-1);
      }
      else {
        // TD-surface
        // edge between ES-WN
        Set_label_bo_mg(I.next_ES(level),num,Dirichlet,level-1);
        Set_label_bo_mg(I.next_WN(level),num,Dirichlet,level-1);
      }
    }
    else {
      if(I.I_z().Tiefe()==level) {
        // NS-surface
        // edge between WT-ED
        Set_label_bo_mg(I.next_WT(level),num,Dirichlet,level-1);
        Set_label_bo_mg(I.next_ED(level),num,Dirichlet,level-1);
      }
      else {
        // EW-edge
        Set_label_bo_mg(I.next_E(level),num,Dirichlet,level-1);
        Set_label_bo_mg(I.next_W(level),num,Dirichlet,level-1);
      }
    }
  }
  else {
    if(I.I_y().Tiefe()==level) {
      if(I.I_z().Tiefe()==level) {
        // EW-surface
        // edge between ST-ND
        Set_label_bo_mg(I.next_ST(level),num,Dirichlet,level-1);
        Set_label_bo_mg(I.next_ND(level),num,Dirichlet,level-1);
      }
      else {
        // NS-edge
        Set_label_bo_mg(I.next_N(level),num,Dirichlet,level-1);
        Set_label_bo_mg(I.next_S(level),num,Dirichlet,level-1);
      }
    }
    else {
      if(I.I_z().Tiefe()==level) {
        // TD-edge
        Set_label_bo_mg(I.next_T(level),num,Dirichlet,level-1);
        Set_label_bo_mg(I.next_D(level),num,Dirichlet,level-1);
      }
    }
  }
}



void Grid_base::Set_label_bo_mg(Index3D I, int num, bool label, int level) {
  Point_hashtable1* point;
  Pointtype type;


  point=hashtable1_point(I);

  if(developer_version) {
    if(point==NULL) {
      cout << " Error 1 in Set_label_bo_mg(...)"
           << " my_rank: " << my_rank 
           << " x: " << I.I_x().get()
           << " y: " << I.I_y().get()
           << " z: " << I.I_z().get()
           << endl;
    }
  }

  /*
if(I==Index3D(9,8,16)) {
  //if(I==Index3D(4,17,16) && my_rank==17) {
  cout << " \n \n set: level: " 
       << level 
       << " my rank: " << my_rank << endl;

  // point=NULL;  
  //  point->label_level=NULL;
  
}
  */

  type = Give_type(I);

  if(type==multigrid || type==parallel_p) {
    if(point->label_level==NULL) {
      // initialize label_level since max_num_label==1      
      if(developer_version) {
        if(Give_max_num_label()!=1) 
          cout << " Error 2 in Set_label_bo_mg! ";
      }
      point->label_level = new int[1];
      point->label_level[0] = 0;
    }

    if(label==true) {
      // Dirichlet auf level
      if(point->Give_label(num) < level)
        point->Put_label_level(num,level);
    }
    // in case of label==false (Neumann) nothing has to be done
    //    }

    //  if(type==parallel_p) {
    if(label==true)
      point->send_label = Dirichlet_send;
    else 
      point->send_label = Neumann_send;
  }
}

bool Grid_base::Give_label_bo_D(Index3D I, int num, int level) const {
  Point_hashtable1* point;
  
  point=hashtable1_point(I);
  if(developer_version) {
    if(point==NULL) {
      cout << " Error in Give_label_bo(Index3D Ind, int num)" << endl;
    }
  }
  return (point->Give_label(num)>=level);
}


// 3. Parallelization




bool Grid_base::Send_label_in_direction_A(int d, 
                                          Point_hashtable1* poi,
                                          int t, 
                                          Index3D my_lev_index,
                                          Index3D next_index) {
  Index3D I;
  Index3D nextS;

  /*
  I = poi->ind;
  if(poi->send_label!=dont_send && t>=I.Tiefe()) {
    if(I << next_index)
      return true;
  }
  */
 I = poi->ind;
  if(poi->send_label!=dont_send && t>=I.Tiefe()) {
    if(I << next_index)
      return true;
    
    if(d<6) {
      nextS = I.next((dir_3D)d,t);
    }
    else if(d<18) {
      nextS = I.next((Edges_cell)(d-6),t);
    }
    else
      nextS = I.next((dir_sons)(d-18),t);
    
    if(nextS << next_index)
      return true;
  }

  return false;
}

bool Grid_base::Send_label_in_direction_B(int d, 
                                          Point_hashtable1* poi,
                                          int t, 
                                          Index3D my_lev_index,
                                          Index3D next_index) {
  Index3D I;
  Index3D nextS;

  I = poi->ind;
  if(poi->send_label!=dont_send && t>=I.Tiefe()) {
    if(I << next_index)
      return true;
    
    if(d<6) {
      nextS = I.next((dir_3D)d,t);
    }
    else if(d<18) {
      nextS = I.next((Edges_cell)(d-6),t);
    }
    else
      nextS = I.next((dir_sons)(d-18),t);
    
    if(nextS << next_index)
      return true;
  }

  return false;
}



// level ist hier Punkte level
void Grid_base::Send_label_parallel(int level, int num_label) {
  int d, level_label;

  Index3D I, next_index;
  int number_bo_send, number_bo_rec, num_message, num;
  Index3D my_lev_index;

  int* send_info;
  int* receive_info;

  MPI_Request req[2];
  MPI_Status status[2];

  int rank_source, rank_destination;

  // Part A: from fine to coarse
  //-----------------------------

  /*
  cout << " Send_label_parallel! level: " 
       << level 
       << " maxlevel: " << max_level 
       << " my_rank: " << my_rank
       << endl;
  */

  // -1. set my_lev_index 
  my_lev_index = Give_my_level_index(level+2);
  //my_lev_index = Give_my_level_index(level+1);

  for(d=0;d<26;++d) { 
    
    // 0. part: basic infos about send and recieve
    if(d<6) {
      rank_source      = give_next_rank_source((dir_3D)d,level+1);
      rank_destination = give_next_rank_destination((dir_3D)d,level+1);  
    }
    else if(d<18) {
      rank_source      = give_next_rank_source((Edges_cell)(d-6),level+1);
      rank_destination = give_next_rank_destination((Edges_cell)(d-6),level+1);  
    }
    else {
      rank_source      = give_next_rank_source((dir_sons)(d-18),level+1);
      rank_destination = give_next_rank_destination((dir_sons)(d-18),level+1);
    }
    

    /*    
    if(d<6) {
      rank_source      = give_next_rank_source((dir_3D)d,level);
      rank_destination = give_next_rank_destination((dir_3D)d,level);  
    }
    else if(d<18) {
      rank_source      = give_next_rank_source((Edges_cell)(d-6),level);
      rank_destination = give_next_rank_destination((Edges_cell)(d-6),level);  
    }
    else {
      rank_source      = give_next_rank_source((dir_sons)(d-18),level);
      rank_destination = give_next_rank_destination((dir_sons)(d-18),level);
    }
    */

    if(developer_version) {
      if(rank_source>p || rank_source<-1) {
        cout << " Problem in Send_label_parallel! " 
             << " level: " << level 
             << " d: " << d << endl;
      }
    }
    

    // 1. index of receiving next processor and correct rank's
    //next_index = Give_next_on_level(d,my_lev_index);
    next_index = Give_level_index_of(rank_destination,level+1);

    // periodisch neu
    if(next_index == Index3D(3,3,3))
      rank_destination = -1;
 
    if(Give_my_coarsest_level()>=level+1)
      rank_source      = -1;

    // 2. part: count number of points to be send
    number_bo_send = 0;
    if(rank_destination != -1) {
      iterate_hash1 {

        /*
        //      if(point1->ind==Index3D(0,2,0)) {
        if(point1->ind==Index3D(37,17,33)) {
          //      if(my_rank==17)
            cout << " destination A: " << rank_destination 
                 << " level: " << level 
                 << " wie: " <<
              Send_label_in_direction_A(d, point1,level,
                                      my_lev_index,next_index)
                 << " label: " <<  (point1->send_label!=dont_send)
                 << " incl: " << (point1->ind << next_index)
                 << " x: " << next_index.I_x().get()
                 << " y: " << next_index.I_y().get()
                 << " z: " << next_index.I_z().get()
                 << " my_rank: " << my_rank
                 << endl;
        }
        */
        

        if(Send_label_in_direction_A(d, point1,level,
                                     my_lev_index,next_index)) {
          ++number_bo_send;
        }
      }
    }
    
    // 3. part: send informations about number_bo_send
    number_bo_rec = 0;
    num_message = 0;
    if(rank_source != -1) {
      MPI_Irecv(&number_bo_rec,1,MPI_INT,rank_source,100+d,comm,
                &req[num_message]);
      ++num_message;
    }
    if(rank_destination != -1) {
      MPI_Isend(&number_bo_send,1,MPI_INT,rank_destination,100+d,comm,
                &req[num_message]);
      ++num_message;
    }
    MPI_Waitall(num_message,req,status);
    
    // 4. part: build informations about points to be send
    // How informations are stored:
    // Index of cell       
    // ind_x, ind_y, ind_z, level(Dirichlet)     >   4 int
    send_info = Give_buffer_send_info(4*number_bo_send);
    receive_info = Give_buffer_receive_info(4*number_bo_rec);
    
    num = 0; 
    if(rank_destination != -1) {
      iterate_hash1 {
        if(Send_label_in_direction_A(d, point1,level,
                                     my_lev_index,next_index)) {
          I = point1->ind;
          
          send_info[num*4]   = I.I_x().get();
          send_info[num*4+1] = I.I_y().get();
          send_info[num*4+2] = I.I_z().get();
          
          if(point1->send_label==Dirichlet_send)
            send_info[num*4+3] = level;
          else 
            send_info[num*4+3] = 0;
          num++;
        }
      }
    }
    
    // 4. part: send points
    num_message = 0;
    if(rank_source != -1 && number_bo_rec>0) {
      MPI_Irecv(receive_info,4*number_bo_rec,MPI_INT,rank_source,120+d,comm,
                &req[num_message]);
      ++num_message;
    }
    if(rank_destination != -1 && number_bo_send>0) {
      MPI_Isend(send_info,4*number_bo_send,MPI_INT,rank_destination,120+d,comm,
                &req[num_message]);
      ++num_message;
    }
    // MPI_Barrier(comm);
    MPI_Waitall(num_message,req,status);
    
    // 5. part: set points
    for(num=0;num<number_bo_rec;++num) {
      I = Index3D(receive_info[num*4],
                  receive_info[num*4+1],
                  receive_info[num*4+2]);
      level_label = receive_info[num*4+3];
      if(level_label!=0)
        Set_label_bo_mg(I, num_label, true, level_label);
      else 
        Set_label_bo_mg(I, num_label, false, level_label);




      /*
        if(I==Index3D(37,17,33)) {
          cout << " get A: " << rank_destination 
               << " level: " << level 
               << " incl: " << level_label
               << " my_rank: " << my_rank
                 << endl;
        }
      */





    }
  }


  /*
  // Part B: ditribute on coarse
  //-----------------------------

  // -1. set my_lev_index 
  my_lev_index = Give_my_level_index(level+1);

  for(d=0;d<26;++d) { 
    
    // 0. part: basic infos about send and recieve
    if(d<6) {
      rank_source      = give_next_rank_source((dir_3D)d,level);
      rank_destination = give_next_rank_destination((dir_3D)d,level);  
    }
    else if(d<18) {
      rank_source      = give_next_rank_source((Edges_cell)(d-6),level);
      rank_destination = give_next_rank_destination((Edges_cell)(d-6),level);  
    }
    else {
      rank_source      = give_next_rank_source((dir_sons)(d-18),level);
      rank_destination = give_next_rank_destination((dir_sons)(d-18),level);
    }

    // 1. index of receiving next processor and correct rank's
    next_index = Give_next_on_level(d,my_lev_index);

    // 2. part: count number of points to be send
    number_bo_send = 0;
    if(rank_destination != -1) {
      iterate_hash1 {
        if(Send_label_in_direction_B(d, point1,level,
                                     my_lev_index,next_index)) {
          ++number_bo_send;
        }
      }
    }
    
    // 3. part: send informations about number_bo_send
    number_bo_rec = 0;
    num_message = 0;
    if(rank_source != -1) {
      MPI_Irecv(&number_bo_rec,1,MPI_INT,rank_source,100+d,comm,
                &req[num_message]);
      ++num_message;
    }
    if(rank_destination != -1) {
      MPI_Isend(&number_bo_send,1,MPI_INT,rank_destination,100+d,comm,
                &req[num_message]);
      ++num_message;
    }
    MPI_Waitall(num_message,req,status);
    
    // 4. part: build informations about points to be send
    // How informations are stored:
    // Index of cell       
    // ind_x, ind_y, ind_z, level(Dirichlet)     >   4 int
    send_info = Give_buffer_send_info(4*number_bo_send);
    receive_info = Give_buffer_receive_info(4*number_bo_rec);
    
    num = 0; 
    if(rank_destination != -1) {
      iterate_hash1 {
        if(Send_label_in_direction_B(d, point1,level,
                                     my_lev_index,next_index)) {
          I = point1->ind;
          
          send_info[num*4]   = I.I_x().get();
          send_info[num*4+1] = I.I_y().get();
          send_info[num*4+2] = I.I_z().get();
          
          if(point1->send_label==Dirichlet_send)
            send_info[num*4+3] = level;
          else 
            send_info[num*4+3] = 0;
          num++;
        }
      }
    }
    
    // 4. part: send points
    num_message = 0;
    if(rank_source != -1 && number_bo_rec>0) {
      MPI_Irecv(receive_info,4*number_bo_rec,MPI_INT,rank_source,120+d,comm,
                &req[num_message]);
      ++num_message;
    }
    if(rank_destination != -1 && number_bo_send>0) {
      MPI_Isend(send_info,4*number_bo_send,MPI_INT,rank_destination,120+d,comm,
                &req[num_message]);
      ++num_message;
    }
    // MPI_Barrier(comm);
    MPI_Waitall(num_message,req,status);
    
    // 5. part: set points
    for(num=0;num<number_bo_rec;++num) {
      I = Index3D(receive_info[num*4],
                  receive_info[num*4+1],
                  receive_info[num*4+2]);
      level_label = receive_info[num*4+3];
      if(level_label!=0)
        Set_label_bo_mg(I, num_label, true, level_label);
      else 
        Set_label_bo_mg(I, num_label, false, level_label);
    }
  }
  */
}

---