src/expde/evpar/communi.cc

Go to the documentation of this file.

//    expde: expression templates for partial differential equations.
//    Copyright (C) 2001  Christoph Pflaum
//    This program is free software; you can redistribute it and/or modify
//    it under the terms of the GNU General Public License as published by
//    the Free Software Foundation; either version 2 of the License, or
//    (at your option) any later version.
//
//    This program is distributed in the hope that it will be useful,
//    but WITHOUT ANY WARRANTY; without even the implied warranty of
//    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
//    GNU General Public License for more details.
//
//                 SEE  Notice1.doc made by 
//                 LAWRENCE LIVERMORE NATIONAL LABORATORY
//

// ------------------------------------------------------------
//
// communi.cc
//
// ------------------------------------------------------------


// Include level 0:

#ifdef COMP_GNUOLD
 #include <iostream.h>
 #include <fstream.h>
 #include <time.h>
 #include <math.h>
#else
 #include <iostream>
 #include <fstream>
 #include <ctime>
 #include <cmath>
#endif 



// Include level 0:
#include "../parser.h"

// Include level 1:
#include "../paramete.h"
#include "../abbrevi.h"
#include "../math_lib/math_lib.h"

// Include level 2:
#include "../basic/basic.h"

// Include level 3:
#include "../domain/domain.h"

// Include level 4:
#include "../formulas/boundy.h"
#include "../formulas/loc_sten.h"

// Include level 5
#include "../grid/gpar.h"
#include "../grid/parallel.h"
#include "../grid/mgcoeff.h"
#include "../grid/sto_man.h"
#include "../grid/gridbase.h"
#include "../grid/grid.h"

// communication of points


/* old
bool Grid_base::Send_I_point_in_direction(int d, 
                                          Point_hashtable1* poi, int t,
                                          bool also_for_prolongation,
                                          Index3D my_lev_index,
                                          Index3D next_index) {
  Index3D I;
  Index3D nextS;

  // new
  if(poi->typ >= interior   ||
     (poi->typ == multigrid && t<=poi->finest_level)) {
    I = poi->ind;
    if(t>=poi->Give_Tiefe()) {
      if(I << next_index) {
        return true;
      }

      if(d<6) {
        nextS = I.next((dir_3D)d,t);
      }
      else if(d<18) {
        nextS = I.next((Edges_cell)(d-6),t);
      }
      else
        nextS = I.next((dir_sons)(d-18),t);
      
      
      if(nextS << next_index) { // also on finest level for variable coefficients
        return true;
      }
      
      //      if(also_for_prolongation && t+1<=poi->my_finest_parallel_level) {
      if(also_for_prolongation) {
        if(d<6) {
          nextS = I.next((dir_3D)d,t+1);
        }
        else if(d<18) {
          nextS = I.next((Edges_cell)(d-6),t+1);
        }
        else
          nextS = I.next((dir_sons)(d-18),t+1);
        
        if(nextS < next_index) {
          return true;
        }
      }
    }
  }
  return false;
}
*/


bool Is_cell_in_direction(dir_sons c, dir_3D d) {
  if(d == Wdir || d == Edir) {
    if(xCoord(c) == d) 
      return true;
    else
      return false;
  }
  if(d == Ndir || d == Sdir) {
    if(yCoord(c) == d) 
      return true;
    else
      return false;
  }
  if(d == Tdir || d == Ddir) {
    if(zCoord(c) == d) 
      return true;
    else
      return false;
  }
  return false;
}


bool Grid_base::Send_B_point_in_direction(int d,
                                          Bo2Point* poi,
                                          Index3D next_index) {
  Index3D Icell;
  Index3D I, I_edge;
  int t;
  int ce, co;

  //  send one layer in left direction
  // this is for the following situation:
  //
  //     * 
  //     ^
  //  W  |  E
  //     +
  //
  // + is a boundary point from E which is needed in W

  I     = poi->ind;
  I_edge = I.next(poi->direction,max_level+1);
  t = max_level;

  // return false;

  // send two layers in right direction and one in left 
  if(I_edge < my_index) {
    // first part: for send on levels
    for(ce=0;ce<8;++ce) {
      if(Is_cell_in_direction((dir_sons)ce,poi->direction)) {
        // study relevant cell
        Icell = I.next((dir_sons)ce,t+1);
        for(co=0;co<8;++co) {
          // study corners of cell
          if(Icell.neighbour((dir_sons)co) << next_index)
            return true;
        }
      }
    }
    // second part: for restriction
    for(ce=0;ce<8;++ce) {
      if(Is_cell_in_direction((dir_sons)ce,poi->direction)) {
        // study relevant cell
        Icell = I.next((dir_sons)ce,t+1).father();
        for(co=0;co<8;++co) {
          // study corners of cell
          if(Icell.neighbour((dir_sons)co) << next_index)
            return true;
        }
      }
    }
  }
  return false;
}


bool Grid_base::Send_Z_point_in_direction(int d, BoCell* poi,
                                          Index3D next_index) {
  Index3D I, Icell;
  int co;

  if(poi->Exists_bocellpoint()) {
    I = poi->ind;
    if(I < my_index) {
      // first part: for send on levels
      Icell = I;
      for(co=0;co<8;++co) {
        // study corners of cell
        if(Icell.neighbour((dir_sons)co) << next_index)
          return true;
      }
      // second part: for restriction
      Icell = I.father();
      for(co=0;co<8;++co) {
        // study corners of cell
        if(Icell.neighbour((dir_sons)co) << next_index) {
          return true;
        }
      }
    }
  }
  return false;
}


void Grid_base::Prepare_communication() {
  // 1. initialize storage
  number_receive_face = new int*[max_level+1];
  number_send_face = new int*[max_level+1];
  number_receive_face_prolongation = new int*[max_level+1];
  number_send_face_prolongation = new int*[max_level+1];

  var_receive = new double***[max_level+1];
  var_send    = new double***[max_level+1];
  var_receive_prolongation = new double***[max_level+1];
  var_send_prolongation    = new double***[max_level+1];

  for(int i=0;i<max_level+1;++i) {
    number_receive_face[i] = new int[26];
    number_send_face[i] = new int[26];
    number_receive_face_prolongation[i] = new int[26];
    number_send_face_prolongation[i]    = new int[26];

    var_receive[i]              = new double**[26];
    var_send[i]                 = new double**[26];
    var_receive_prolongation[i] = new double**[26];
    var_send_prolongation[i]    = new double**[26];

    for(int k=0;k<26;++k) {
      number_receive_face[i][k] = 0;
      number_send_face[i][k] = 0;
      number_receive_face_prolongation[i][k] = 0;
      number_send_face_prolongation[i][k] = 0;
    }
  }

  // 2. prepare
  Prepare_communication_all_grids();
  Prepare_communication_coarser_grids();
}


bool Grid_base::Send_prol_point_in_direction(int d, 
                                             Point_hashtable1* poi, int t,
                                             Index3D my_lev_index,
                                             Index3D next_index) {
  Index3D I, nextS;

  // new
  // t point level of coarse grid points
  if( poi->typ >= interior   ||
     (poi->typ == multigrid && t<=poi->finest_level)) {
    I = poi->ind;
    if(t>=poi->Give_Tiefe()) {
      if(I << next_index) {
        return true;
      }
      if(d<6) {
        nextS = I.next((dir_3D)d,t+1);
      }
      else if(d<18) {
        nextS = I.next((Edges_cell)(d-6),t+1);
        }
      else
        nextS = I.next((dir_sons)(d-18),t+1);
      if(nextS << next_index) {
        return true;
      }
    }
  }

  return false;
}

// for prolongation operators on coarser grids
void Grid_base::Prepare_communication_coarser_grids() {
  int d, num, l;
  Pointtype ptyp;
  MPI_Request req[2];
  MPI_Status status[2];
  int num_message;
  Index3D I;
  Index3D my_lev_index;
  Index3D next_index;
  
  int* send_info[26];
  int* receive_info[26];

  int num_rec, num_send;

  int rank_source, rank_destination;

  // l is level with respect to coarse points
  // send point level is 1 larger
  for(l=n_parallel-2;l>=min_level;--l) {
    for(d=0;d<26;++d) {
      // 0. Step:
      if(d<6) {
        rank_source      = give_next_rank_source((dir_3D)d,l+1);
        rank_destination = give_next_rank_destination((dir_3D)d,l+1);  
      }
      else if(d<18) {
        rank_source      = give_next_rank_source((Edges_cell)(d-6),l+1);
        rank_destination = give_next_rank_destination((Edges_cell)(d-6),l+1);  
      }
      else {
        rank_source      = give_next_rank_source((dir_sons)(d-18),l+1);
        rank_destination = give_next_rank_destination((dir_sons)(d-18),l+1);
      }

      // 1. Step: 
      my_lev_index = Give_my_level_index(l+2);
      //old
      //      next_index = Give_next_on_level(d,my_lev_index);
      //new
      next_index = Give_level_index_of(rank_destination,l+1);
      if(next_index == Index3D(3,3,3))
        rank_destination = -1;
      if(Give_my_coarsest_level()>=l+1)
        rank_source      = -1;

      // 2. Step: Calc number send for faces
      number_send_face_prolongation[l][d] = 0;
    
      // 2.1. Step: interior points



      if(l>=my_coarsest_level) {
        // allowed to send
        iterate_hash1 {
          if(Send_prol_point_in_direction(d,point1,l,
                                          my_lev_index,next_index)) {
            ++number_send_face_prolongation[l][d];
          }
        }
      }


//number_send_face_prolongation[l][d] = 0;


/*
  if(l==2 && d==22)
  cout << " my rank: " << my_rank
  << " rank_source: " << rank_source
  << " rank_destination: " << rank_destination
  << endl;
*/

      // 3. Step: Send number_faces
      num_rec  = 0;
      num_send = number_send_face_prolongation[l][d];
      num_message = 0;
      if(rank_source != -1) {
        MPI_Irecv(&num_rec,1,
                  MPI_INT,rank_source,200+d,comm,
                  &req[num_message]);
        ++num_message;
      }
      if(rank_destination != -1) {
        MPI_Isend(&num_send,1,
                  MPI_INT,rank_destination,200+d,comm,
                  &req[num_message]);
        ++num_message;
      }
      MPI_Waitall(num_message,req,status);
      number_receive_face_prolongation[l][d] = num_rec;
      
      // construct double array 
      if(number_send_face_prolongation[l][d]!=0) {
        var_send_prolongation[l][d] =
          new double*[number_send_face_prolongation[l][d]];
      }
      else {
        var_send_prolongation[l][d] = NULL;
      }
      if(number_receive_face_prolongation[l][d]!=0) {
        var_receive_prolongation[l][d] = 
          new double*[number_receive_face_prolongation[l][d]];
      }
      else {
        var_receive_prolongation[l][d] = NULL;
      }
      
    
      // 4. Step: info = new ...
      if(number_send_face_prolongation[l][d]>0) {
        send_info[d] = new int[3*number_send_face_prolongation[l][d]];
      }
      else {
        send_info[d] = NULL;
      }      
      if(number_receive_face_prolongation[l][d]>0) {
        receive_info[d] = new int[3*number_receive_face_prolongation[l][d]];
      }
      else {
        receive_info[d] = NULL;
      }
      

      
      // 5. Step: transmitter interior points
      // 5.1 build informations about points to be send
      // How informations are stored:
      // Index of point       
      // ind_x, ind_y, ind_z     >   3 int
      num = 0;
      if(l>=my_coarsest_level) {
        // allowed to send
        iterate_hash1 {
          if(Send_prol_point_in_direction(d,point1,l,
                                          my_lev_index,next_index)) {
            I = point1->ind;
            
            send_info[d][num*3]   = I.I_x().get();
            send_info[d][num*3+1] = I.I_y().get();
            send_info[d][num*3+2] = I.I_z().get();

            if(developer_version) {
              if(Give_variable_slow(I,l)==NULL) {
                cout << " error in  5.1. Prepare_communication_coarser_grids()"
                     << endl;
              }
            }

            var_send_prolongation[l][d][num] = Give_variable(I,l);
            num++;
          }
        }
      }

        
      // 5.2 part: send points
      num_message = 0;
      if(rank_source != -1 && number_receive_face_prolongation[l][d]>0) {
        MPI_Irecv(receive_info[d] ,
                  3*number_receive_face_prolongation[l][d],
                  MPI_INT,rank_source,210+d,comm,
                  &req[num_message]);
        ++num_message;
      }
      if(rank_destination != -1 && number_send_face_prolongation[l][d]>0) {
        MPI_Isend(send_info[d],
                  3*number_send_face_prolongation[l][d],
                  MPI_INT,rank_destination,210+d,comm,
                  &req[num_message]);
        ++num_message;
      }
      MPI_Waitall(num_message,req,status);
 
         
      // 5.3 part: set points
      for(num=0;num<number_receive_face_prolongation[l][d];++num) {
        I = Index3D(receive_info[d][num*3],
                    receive_info[d][num*3+1],
                    receive_info[d][num*3+2]);
        var_receive_prolongation[l][d][num] = Give_variable_slow(I,l);
        if(var_receive_prolongation[l][d][num]==NULL) {
          if(Exists_Point(I)==false) {
            Add_point(I, parallel_p);
          }
          Add_double(I,l,Give_max_num_var());
          var_receive_prolongation[l][d][num] = Give_variable(I,l);
        }



        if(developer_version) {
          ptyp = Give_type(I);
          if(ptyp!=parallel_p && ptyp!=exterior) {
            cout << "\n error in 5.3 prolongation, my rank: " 
                 << my_rank 
                 << " Index: " 
                 << " x: " << receive_info[d][num*3]
                 << " y: " << receive_info[d][num*3+1]
                 << " z: " << receive_info[d][num*3+2]
                 << "typ: " << ptyp 
                 << endl;
          }
        }
      }

      // 6. part delete 
      if(send_info[d]!=NULL) {
        delete(send_info[d]);
        send_info[d] = NULL;
      }
      if(receive_info[d]!=NULL) {
        delete(receive_info[d]);
        receive_info[d] = NULL;
      }
    }
  }
}




bool Grid_base::Send_I_point_in_direction(int d, 
                                          Point_hashtable1* poi, int t,
                                          bool also_for_prolongation,
                                          Index3D my_lev_index,
                                          Index3D next_index) {
  Index3D I;
  Index3D nextS;

  I = poi->ind;
  if(I << next_index) {
    return true;
  }
  
  if(d<6) {
    nextS = I.next((dir_3D)d,t);
  }
  else if(d<18) {
    nextS = I.next((Edges_cell)(d-6),t);
  }
  else
    nextS = I.next((dir_sons)(d-18),t);
  
  
  if(nextS << next_index) { // also on finest level for variable coefficients
    return true;
  }
  
  //      if(also_for_prolongation && t+1<=poi->my_finest_parallel_level) {
  if(also_for_prolongation) {
    if(d<6) {
      nextS = I.next((dir_3D)d,t+1);
    }
    else if(d<18) {
      nextS = I.next((Edges_cell)(d-6),t+1);
    }
    else
      nextS = I.next((dir_sons)(d-18),t+1);
    
    if(nextS < next_index) {
      return true;
    }
  }
  return false;
}



void Grid_base::Prepare_communication_all_grids() {
  Index1D nextS;
  Pointtype ptyp;
  int d, num, l;
  MPI_Request req[12];
  MPI_Status status[12];
  int num_message;
  Index3D I, next_index;
  Index3D my_lev_index;

  int* send_info[26];
  int* receive_info[26];
  // I interior, B boundary, Z Zellfreiheitsgrad
  int num_rec_face_I[26],  num_rec_face_B[26],  num_rec_face_Z[26];
  int num_send_face_I[26], num_send_face_B[26], num_send_face_Z[26];

  int three_rec[26][3];
  int three_send[26][3];

  int rank_source, rank_destination;
  bool also_for_prolongation;

  for(l=Max_level();l>=my_coarsest_level;--l) {
  //for(l=Max_level();l>=Max_level();--l) {

    // -1. set my_lev_index 
    my_lev_index = Give_my_level_index(l+1);
    

    // construct extra array for invariant points which are needed.
    int lenght_invar_array = 0;
    iterate_hash1 {
      if(point1->typ >= interior   ||
         (point1->typ == multigrid && l<=point1->finest_level)) {
        if(l>=point1->Give_Tiefe()) {
          lenght_invar_array++;
        }
      }
    }
    Point_hashtable1** invariant_array = new Point_hashtable1*[lenght_invar_array];
    int array_i = 0;
    iterate_hash1 {
      if(point1->typ >= interior   ||
         (point1->typ == multigrid && l<=point1->finest_level)) {
        if(l>=point1->Give_Tiefe()) {
          invariant_array[array_i] = point1;
          array_i++;
        }
      }
    }

    for(d=0;d<26;++d) {
      // 0. Step: 
      next_index = Give_next_on_level(d,my_lev_index);

      // 1. Step:
      if(d<6) {
        rank_source      = give_next_rank_source((dir_3D)d,l);
        rank_destination = give_next_rank_destination((dir_3D)d,l);  
      }
      else if(d<18) {
        rank_source      = give_next_rank_source((Edges_cell)(d-6),l);
        rank_destination = give_next_rank_destination((Edges_cell)(d-6),l);  
      }
      else {
        rank_source      = give_next_rank_source((dir_sons)(d-18),l);
        rank_destination = give_next_rank_destination((dir_sons)(d-18),l);
      }

      // 2. Step: Calc number send for faces
      num_send_face_I[d] = 0;
      num_send_face_B[d] = 0;
      num_send_face_Z[d] = 0;
    
      // 2.1. Step: interior points
      // level_prolongation must be finer for 
      // prolongation operators on coarser grids
      if(l==Max_level() || l<n_parallel-1) {
        also_for_prolongation = false;
      }
      else {
        also_for_prolongation = true;
      }

      for(int i = 0;i < lenght_invar_array;i++) {
        if(Send_I_point_in_direction(d,invariant_array[i],l,
                                     also_for_prolongation,
                                     my_lev_index,
                                     next_index)) {
          ++num_send_face_I[d];
        }
      }

     /* old
      iterate_hash1 {
        if(Send_I_point_in_direction(d,point1,l,
                                     also_for_prolongation,
                                     my_lev_index,
                                     next_index)) {
          ++num_send_face_I[d];
        }
      }
     */
      if(l==Max_level()) {
        // 2.2. Step: boundary points
        iterate_hash3 {
          if(Send_B_point_in_direction((dir_3D)d,bo2point,next_index)) {
            ++num_send_face_B[d];
          }
        }
        // 2.3. Step: Zellfreiheitsgrad
        iterate_hash2 {
          if(Send_Z_point_in_direction(d,bocell,next_index)) {
            ++num_send_face_Z[d];
          }
        }
      }
      // 2.4. Step: sum up
      number_send_face[l][d] =
        num_send_face_I[d] +
        num_send_face_B[d] +
        num_send_face_Z[d];
      
      // 3. Step: Send number_faces
      num_message = 0;
      three_send[d][0]=num_send_face_I[d];
      three_send[d][1]=num_send_face_B[d];
      three_send[d][2]=num_send_face_Z[d];
      three_rec[d][0] =0;
      three_rec[d][1] =0;
      three_rec[d][2] =0;
      if(rank_source != -1) {
        MPI_Irecv(three_rec[d],3,
                  MPI_INT,rank_source,20+d,comm,
                  &req[num_message]);
        ++num_message;
      }
      if(rank_destination != -1) {
        MPI_Isend(three_send[d],3,
                  MPI_INT,rank_destination,20+d,comm,
                  &req[num_message]);
        ++num_message;
      }
      MPI_Waitall(num_message,req,status);
      
      num_rec_face_I[d]=three_rec[d][0];
      num_rec_face_B[d]=three_rec[d][1];
      num_rec_face_Z[d]=three_rec[d][2];
      
      // - sum up
      number_receive_face[l][d] =
        num_rec_face_I[d] +
        num_rec_face_B[d] +
        num_rec_face_Z[d];
      
      // construct double array 
      if(number_send_face[l][d]!=0)
        var_send[l][d] = new double*[number_send_face[l][d]];
      else var_send[l][d] = NULL;
      if(number_receive_face[l][d]!=0)
        var_receive[l][d] = 
          new double*[number_receive_face[l][d]];
      else var_receive[l][d] = NULL;
      
    
      // 4. Step: info = new ...
      if(MAX(3*num_send_face_I[d],
             4*num_send_face_B[d],
             3*num_send_face_Z[d])>0)
        send_info[d] = new int[MAX(3*num_send_face_I[d],
                                   4*num_send_face_B[d],
                                   3*num_send_face_Z[d])];
      else send_info[d]  = NULL;
      
      if(MAX(3*num_rec_face_I[d],
             4*num_rec_face_B[d],
             3*num_rec_face_Z[d])>0)
        receive_info[d] = new int[MAX(3*num_rec_face_I[d],
                                      4*num_rec_face_B[d],
                                      3*num_rec_face_Z[d])];
      else receive_info[d] = NULL;
      
      // 5. Step: transmitter interior points
      // 5.1 build informations about points to be send
      // How informations are stored:
      // Index of point       
      // ind_x, ind_y, ind_z     >   3 int

      num = 0;
      for(int i = 0;i < lenght_invar_array;i++) {
        point1 = invariant_array[i];
        if(Send_I_point_in_direction(d,point1,l,
                                     also_for_prolongation, my_lev_index,
                                     next_index)) {
          I = point1->ind;
          
          send_info[d][num*3]   = I.I_x().get();
          send_info[d][num*3+1] = I.I_y().get();
          send_info[d][num*3+2] = I.I_z().get();
          var_send[l][d][num] = Give_variable(I,l);
          num++;
        }
      }

      /* old
      num = 0;
      iterate_hash1 {
        if(Send_I_point_in_direction(d,point1,l,
                                     also_for_prolongation, my_lev_index,
                                     next_index)) {
          I = point1->ind;
          
          send_info[d][num*3]   = I.I_x().get();
          send_info[d][num*3+1] = I.I_y().get();
          send_info[d][num*3+2] = I.I_z().get();
          var_send[l][d][num] = Give_variable(I,l);
          num++;
        }
      }
      */

      // 5.2 part: send points
      num_message = 0;
      if(rank_source != -1 && num_rec_face_I[d]>0) {
        MPI_Irecv(receive_info[d] ,
                  3*num_rec_face_I[d],
                  MPI_INT,rank_source,40+d,comm,
                  &req[num_message]);
        ++num_message;
      }
      if(rank_destination != -1 && num_send_face_I[d]>0) {
        MPI_Isend(send_info[d],
                  3*num_send_face_I[d],
                  MPI_INT,rank_destination,40+d,comm,
                  &req[num_message]);
        ++num_message;
      }
      MPI_Waitall(num_message,req,status);
    
      // 5.3 part: set points
      for(num=0;num<num_rec_face_I[d];++num) {
        I = Index3D(receive_info[d][num*3],
                    receive_info[d][num*3+1],
                    receive_info[d][num*3+2]);
        var_receive[l][d][num] = Give_variable_slow(I,l);
        if(var_receive[l][d][num]==NULL) {
          Add_double(I,l,Give_max_num_var());
          var_receive[l][d][num] = Give_variable(I,l);
        }



        if(developer_version) {
          ptyp = Give_type(I);
          if(ptyp!=parallel_p && ptyp!=exterior) {
            cout << "\n error in Prepare_communication(), 5.3, my rank: " 
                 << my_rank 
                 << " Index: " 
                 << " x: " << receive_info[d][num*3]
                 << " y: " << receive_info[d][num*3+1]
                 << " z: " << receive_info[d][num*3+2]
                 << "typ: " << ptyp 
                 << endl;
          }
        }
        



           

           /* old
        var_receive[l][d][num] = 
          Give_variable(Index3D(receive_info[d][num*3],
                                receive_info[d][num*3+1],
                                receive_info[d][num*3+2]), 
                        l);
        if(developer_version) {
          if(var_receive[l][d][num]==NULL) {
            cout << " Error in 5.3. l: " << l 
                 << " Index: " 
                 << " x: " << receive_info[d][num*3]
                 << " y: " << receive_info[d][num*3+1]
                 << " z: " << receive_info[d][num*3+2]
                 << endl;
          }
        }
           */
      }

      if(l==Max_level()) {
        // 6.  Step: transmitter boundary points
        // 6.1 build informations about points to be send
        // How informations are stored:
        // Index of corner point, dir_3D       
        // ind_x, ind_y, ind_z     d      >   4 int
        num = 0;
        iterate_hash3 {
          if(Send_B_point_in_direction((dir_3D)d,bo2point,next_index)) {
            I = bo2point->ind;
            
            send_info[d][num*4]   = I.I_x().get();
            send_info[d][num*4+1] = I.I_y().get();
            send_info[d][num*4+2] = I.I_z().get();
            send_info[d][num*4+3] =  bo2point->direction;
            
            var_send[Max_level()][d][num_send_face_I[d]+num] = 
              Give_variable(I,(dir_3D)(bo2point->direction));
            num++;
          }
        }
    
        // 6.2 part: send points
        num_message = 0;
        if(rank_source != -1 && num_rec_face_B[d]>0) {
          MPI_Irecv(receive_info[d] ,
                    4*num_rec_face_B[d],
                    MPI_INT,rank_source,60+d,comm,
                    &req[num_message]);
          ++num_message;
        }
        if(rank_destination != -1 && num_send_face_B[d]>0) {
          MPI_Isend(send_info[d],
                    4*num_send_face_B[d],
                    MPI_INT,rank_destination,60+d,comm,
                    &req[num_message]);
          ++num_message;
        }
        MPI_Waitall(num_message,req,status);
        
        // 6.3 part: set points
        for(num=0;num<num_rec_face_B[d];++num) {
          if(developer_version) {
            if(Exists_Bo_Point(Index3D(receive_info[d][num*4],
                                       receive_info[d][num*4+1],
                                       receive_info[d][num*4+2]), 
                               (dir_3D)receive_info[d][num*4+3])==false) {
              cout << "\n error in Prepare_communication(), 6.3, \n my: " 
                   << my_rank << " my_I: "; 
              transform_coord(my_index.coordinate()).Print();
              cout << " d:"; Print((dir_3D)d); cout << endl;
              Index3D II;
              II = Index3D(receive_info[d][num*4],
                           receive_info[d][num*4+1],
                           receive_info[d][num*4+2]);
              II.Print();
              transform_coord(II.coordinate()).Print();
              cout << " direction: "; Print((dir_3D)receive_info[d][num*4+3]);
              cout << endl;
            }
          }

          var_receive[Max_level()][d][num_rec_face_I[d]+num] = 
            Give_variable(Index3D(receive_info[d][num*4],
                                  receive_info[d][num*4+1],
                                  receive_info[d][num*4+2]), 
                          (dir_3D)receive_info[d][num*4+3]);
        }
        
        // 7. Step: transmitter Zellfreiheitsgrade points
        // 7.1 build informations about points to be send
        // How informations are stored:
        // Index of point       
        // ind_x, ind_y, ind_z     >   3 int
        num = 0;

        iterate_hash2 {
          if(Send_Z_point_in_direction((dir_3D)d,bocell,next_index)) {
            I = bocell->ind;
            
            send_info[d][num*3]   = I.I_x().get();
            send_info[d][num*3+1] = I.I_y().get();
            send_info[d][num*3+2] = I.I_z().get();
            //  var_send[d][num] = Give_variable_cellpoi(I);
            var_send[Max_level()][d][num_send_face_I[d]+num_send_face_B[d]
                                    +num] = 
              bocell->var;
            num++;
          }
        }
        
        // 7.2 part: send points
        num_message = 0;
        if(rank_source != -1 && num_rec_face_Z[d]) {
          MPI_Irecv(receive_info[d] ,
                    3*num_rec_face_Z[d],
                    MPI_INT,rank_source,80+d,comm,
                    &req[num_message]);
          ++num_message;
        }
        if(rank_destination != -1 && num_send_face_Z[d]) {
          MPI_Isend(send_info[d],
                    3*num_send_face_Z[d],
                    MPI_INT,rank_destination,80+d,comm,
                    &req[num_message]);
          ++num_message;
        }
        MPI_Waitall(num_message,req,status);
      
        // 7.3 part: set points
        for(num=0;num<num_rec_face_Z[d];++num) {
          var_receive[Max_level()][d][num_rec_face_I[d]+num_rec_face_B[d]
                                     +num] = 
            Give_variable_cellpoi(Index3D(receive_info[d][num*3],
                                          receive_info[d][num*3+1],
                                          receive_info[d][num*3+2]));
          if(developer_version) {
            if(var_receive[Max_level()][d][num_rec_face_I[d]+num_rec_face_B[d]
                                          +num] == NULL) {
              cout << " Error in 7.3, Prepare_communication_all_grids()! "
                   << " my rank: " << my_rank
                   << " from: " << rank_source << endl;
            }
          }
        }
      }
      // 8. part delete 
      if(send_info[d]!=NULL) {
        delete(send_info[d]);
        send_info[d] = NULL;
      }
      if(receive_info[d]!=NULL) {
        delete(receive_info[d]);
        receive_info[d] = NULL;
      }
    }
    delete invariant_array;
  }
}
  
void Grid::Full_update_Variable(int level) {
  int* array_num;
  int max_var;
  
  max_var = Give_max_num_var();
  array_num  = new int[max_var];
  for(int i=0;i<max_var;++i) {
    array_num[i] = i;
  }
  Update_Variable(Wdir,array_num,max_var,level);
  Update_Variable(Edir,array_num,max_var,level);
  Update_Variable(Ndir,array_num,max_var,level);
  Update_Variable(Sdir,array_num,max_var,level);
  Update_Variable(Tdir,array_num,max_var,level);
  Update_Variable(Ddir,array_num,max_var,level);
  
  /*
  Index1D nextS;
  int d,i, num, lr, iv;
  MPI_Request req[12];
  MPI_Status status[12];
  int num_message;
  double*  send_info[6];
  double*  receive_info[6];
  int number_send[6];
  int number_receive[6];

  int number_vars;

  if(I_am_active()) {
    // 1. Step: calc number to be send and receive
    number_vars = Give_max_num_var();
    for(d=0;d<6;++d) {
      number_send[d] = 
        number_send_face[level][d] * number_vars;
      number_receive[d] = 
        number_receive_face[level][d] * number_vars;
    }
    
    // 2. Step: build informations about var(double) to be send  
    for(d=0;d<6;++d) {
      send_info[d]    = Give_buffer_send((dir_3D)d,number_send[d]); 
      receive_info[d] = Give_buffer_receive((dir_3D)d,number_receive[d]); 
    }
    
    for(i=0;i<3;++i) {
      for(lr=0;lr<2;++lr) {
        d =  Add((dir1D)lr,(main_dir_3D)i);
        for(num=0;num<number_send_face[level][d];++num) {
          for(iv=0;iv<Give_max_num_var();++iv) {
            send_info[d][num*number_vars+iv] = 
              var_send[level][d][num][iv];
            //    send_info[d][num*number_vars+iv] = 100.0;
          }
        }
      }
      
      // 4. Step: send doubles
      num_message = 0;
      for(lr=0;lr<2;++lr) {
        d =  Add((dir1D)lr,(main_dir_3D)i);
        if(give_next_rank_source((dir_3D)d) != -1 && number_receive[d]>0) {
          MPI_Irecv(receive_info[d] ,number_receive[d],
                    MPI_DOUBLE,give_next_rank_source((dir_3D)d),10,comm,
                    &req[num_message]);
          ++num_message;
        }
      }
      for(lr=0;lr<2;++lr) {
        d =  Add((dir1D)lr,(main_dir_3D)i);
        if(give_next_rank_destination((dir_3D)d) != -1 && number_send[d]>0) {
          MPI_Isend(send_info[d],number_send[d],
                    MPI_DOUBLE,give_next_rank_destination((dir_3D)d),10,comm,
                    &req[num_message]);
          ++num_message;
        }
      }
      MPI_Waitall(num_message,req,status);
      
      // 5. part: set informations about var(double)
      for(lr=0;lr<2;++lr) {
        d =  Add((dir1D)lr,(main_dir_3D)i);
        for(num=0;num<number_receive_face[level][d];++num) {
          for(iv=0;iv<Give_max_num_var();++iv) {
            var_receive[level][d][num][iv] = 
              receive_info[d][num*number_vars+iv];
            // var_receive[d][num][i] = 100;
          }
        }
      }
    }
  }
  */
}

void Grid::Update_Variable_no_full(int* array_num, int anz_num,
                                   int level) {
  for(int i=0;i<26;++i) {
    Update_Variable(i, array_num, anz_num, level);
  }
}
void Grid::Update_Variable(dir_3D d, int* array_num, int anz_num, int level) {
  if(d==Wdir) {
    Update_Variable((int)Wdir, array_num, anz_num, level);

    Update_Variable((int)(WDed+6), array_num, anz_num, level);
    Update_Variable((int)(WTed+6), array_num, anz_num, level);
    Update_Variable((int)(SWed+6), array_num, anz_num, level);
    Update_Variable((int)(NWed+6), array_num, anz_num, level);

    Update_Variable((int)(WSTd+18), array_num, anz_num, level);
    Update_Variable((int)(WSDd+18), array_num, anz_num, level);
    Update_Variable((int)(WSTd+18), array_num, anz_num, level);
    Update_Variable((int)(WNDd+18), array_num, anz_num, level);
    Update_Variable((int)(WNTd+18), array_num, anz_num, level);
  }
  if(d==Edir) {
    Update_Variable((int)Edir, array_num, anz_num, level);

    Update_Variable((int)(EDed+6), array_num, anz_num, level);
    Update_Variable((int)(ETed+6), array_num, anz_num, level);
    Update_Variable((int)(SEed+6), array_num, anz_num, level);
    Update_Variable((int)(NEed+6), array_num, anz_num, level);

    Update_Variable((int)(ESDd+18), array_num, anz_num, level);
    Update_Variable((int)(ESTd+18), array_num, anz_num, level);
    Update_Variable((int)(ENDd+18), array_num, anz_num, level);
    Update_Variable((int)(ENTd+18), array_num, anz_num, level);
  }
  if(d==Ndir) {
    Update_Variable((int)Ndir, array_num, anz_num, level);

    Update_Variable((int)(NDed+6), array_num, anz_num, level);
    Update_Variable((int)(NTed+6), array_num, anz_num, level);
    Update_Variable((int)(NWed+6), array_num, anz_num, level);
    Update_Variable((int)(NEed+6), array_num, anz_num, level);

    Update_Variable((int)(ENTd+18), array_num, anz_num, level);
    Update_Variable((int)(WNTd+18), array_num, anz_num, level);
    Update_Variable((int)(ENDd+18), array_num, anz_num, level);
    Update_Variable((int)(WNDd+18), array_num, anz_num, level);
  }
  if(d==Sdir) {
    Update_Variable((int)Sdir, array_num, anz_num, level);

    Update_Variable((int)(SDed+6), array_num, anz_num, level);
    Update_Variable((int)(STed+6), array_num, anz_num, level);
    Update_Variable((int)(SWed+6), array_num, anz_num, level);
    Update_Variable((int)(SEed+6), array_num, anz_num, level);

    Update_Variable((int)(ESTd+18), array_num, anz_num, level);
    Update_Variable((int)(WSTd+18), array_num, anz_num, level);
    Update_Variable((int)(ESDd+18), array_num, anz_num, level);
    Update_Variable((int)(WSDd+18), array_num, anz_num, level);
  }
  if(d==Ddir) {
    Update_Variable((int)Ddir, array_num, anz_num, level);

    Update_Variable((int)(SDed+6), array_num, anz_num, level);
    Update_Variable((int)(NDed+6), array_num, anz_num, level);
    Update_Variable((int)(WDed+6), array_num, anz_num, level);
    Update_Variable((int)(EDed+6), array_num, anz_num, level);

    Update_Variable((int)(ESDd+18), array_num, anz_num, level);
    Update_Variable((int)(WSDd+18), array_num, anz_num, level);
    Update_Variable((int)(ENDd+18), array_num, anz_num, level);
    Update_Variable((int)(WNDd+18), array_num, anz_num, level);
  }
  if(d==Tdir) {
    Update_Variable((int)Tdir, array_num, anz_num, level);

    Update_Variable((int)(STed+6), array_num, anz_num, level);
    Update_Variable((int)(NTed+6), array_num, anz_num, level);
    Update_Variable((int)(WTed+6), array_num, anz_num, level);
    Update_Variable((int)(ETed+6), array_num, anz_num, level);

    Update_Variable((int)(ESTd+18), array_num, anz_num, level);
    Update_Variable((int)(WSTd+18), array_num, anz_num, level);
    Update_Variable((int)(ENTd+18), array_num, anz_num, level);
    Update_Variable((int)(WNTd+18), array_num, anz_num, level);
  }
}

void Grid::Update_Variable(int d, int* array_num, int anz_num, int level) {
  Index1D nextS;
  int num, iv;
  MPI_Request req[12];
  MPI_Status status[12];
  int num_message;
  double*  send_info;
  double*  receive_info;
  int number_send;
  int number_receive;

  //  int number_vars;
  double** var_sendd;
  double** var_received;

  int rank_source, rank_destination;

  /*
  MPI_Barrier(MPI_COMM_WORLD);
  if(my_rank==0) {
    cout << " Parallelisierung in Richtung: " << d << endl;
    for(iv=0;iv<anz_num;++iv) {
      cout << " Nummer: " << array_num[iv] << endl;
    }
  }
  MPI_Barrier(MPI_COMM_WORLD);
  */

  //if(I_am_active() && Max_level()==level) {
  if(I_am_active() && anz_num != 0) {
    // 0. part: basic infos about send and recieve
   if(d<6) {
      rank_source      = give_next_rank_source((dir_3D)d,level);
      rank_destination = give_next_rank_destination((dir_3D)d,level);  
    }
    else if(d<18) {
      rank_source      = give_next_rank_source((Edges_cell)(d-6),level);
      rank_destination = give_next_rank_destination((Edges_cell)(d-6),level);  
    }
    else {
      rank_source      = give_next_rank_source((dir_sons)(d-18),level);
      rank_destination = give_next_rank_destination((dir_sons)(d-18),level);
    }

    // 1. Step: calc number to be send and receive
    number_send = 
      number_send_face[level][d] * anz_num;
    number_receive = 
      number_receive_face[level][d] * anz_num;
   
    // 2. Step: build informations about var(double) to be send  
    send_info    = Give_buffer_send(d,number_send); 
    receive_info = Give_buffer_receive(d,number_receive); 

    var_sendd=var_send[level][d];
    for(num=0;num<number_send_face[level][d];++num) {
      for(iv=0;iv<anz_num;++iv) {
        send_info[num*anz_num+iv] = 
          var_sendd[num][array_num[iv]];
      }
    }

    // 4. Step: send doubles
    num_message = 0;
    if(rank_source != -1 && number_receive>0) {
      MPI_Irecv(receive_info ,number_receive,
                MPI_DOUBLE,rank_source,26,comm,
                &req[num_message]);
      ++num_message;
    }      
    if(rank_destination != -1 && number_send>0) {
      MPI_Isend(send_info,number_send,
                MPI_DOUBLE,rank_destination,26,comm,
                &req[num_message]);
      ++num_message;
    }
    MPI_Waitall(num_message,req,status);

    // 5. part: set informations about var(double)
    var_received=var_receive[level][d];
    for(num=0;num<number_receive_face[level][d];++num) {
      for(iv=0;iv<anz_num;++iv) {
        var_received[num][array_num[iv]] = 
          receive_info[num*anz_num+iv];
      }
    }
  }
}

void Grid::Update_Variable_for_prolongation(int* array_num, int anz_num,
                                            int level) {
  for(int i=0;i<26;++i) {
    Update_Variable_for_prolongation(i, array_num, anz_num, level);
  }
}

void Grid::Update_Variable_for_prolongation(dir_3D d, int* array_num, int anz_num, int level) {
  if(d==Wdir) {
    Update_Variable_for_prolongation((int)Wdir, array_num, anz_num, level);

    Update_Variable_for_prolongation((int)(WDed+6), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(WTed+6), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(SWed+6), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(NWed+6), array_num, anz_num, level);

    Update_Variable_for_prolongation((int)(WSTd+18), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(WSDd+18), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(WSTd+18), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(WNDd+18), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(WNTd+18), array_num, anz_num, level);
  }
  if(d==Edir) {
    Update_Variable_for_prolongation((int)Edir, array_num, anz_num, level);

    Update_Variable_for_prolongation((int)(EDed+6), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(ETed+6), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(SEed+6), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(NEed+6), array_num, anz_num, level);

    Update_Variable_for_prolongation((int)(ESDd+18), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(ESTd+18), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(ENDd+18), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(ENTd+18), array_num, anz_num, level);
  }
  if(d==Ndir) {
    Update_Variable_for_prolongation((int)Ndir, array_num, anz_num, level);

    Update_Variable_for_prolongation((int)(NDed+6), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(NTed+6), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(NWed+6), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(NEed+6), array_num, anz_num, level);

    Update_Variable_for_prolongation((int)(ENTd+18), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(WNTd+18), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(ENDd+18), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(WNDd+18), array_num, anz_num, level);
  }
  if(d==Sdir) {
    Update_Variable_for_prolongation((int)Sdir, array_num, anz_num, level);

    Update_Variable_for_prolongation((int)(SDed+6), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(STed+6), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(SWed+6), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(SEed+6), array_num, anz_num, level);

    Update_Variable_for_prolongation((int)(ESTd+18), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(WSTd+18), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(ESDd+18), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(WSDd+18), array_num, anz_num, level);
  }
  if(d==Ddir) {
    Update_Variable_for_prolongation((int)Ddir, array_num, anz_num, level);

    Update_Variable_for_prolongation((int)(SDed+6), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(NDed+6), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(WDed+6), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(EDed+6), array_num, anz_num, level);

    Update_Variable_for_prolongation((int)(ESDd+18), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(WSDd+18), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(ENDd+18), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(WNDd+18), array_num, anz_num, level);
  }
  if(d==Tdir) {
    Update_Variable_for_prolongation((int)Tdir, array_num, anz_num, level);

    Update_Variable_for_prolongation((int)(STed+6), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(NTed+6), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(WTed+6), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(ETed+6), array_num, anz_num, level);

    Update_Variable_for_prolongation((int)(ESTd+18), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(WSTd+18), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(ENTd+18), array_num, anz_num, level);
    Update_Variable_for_prolongation((int)(WNTd+18), array_num, anz_num, level);
  }
}

void Grid::Update_Variable_for_prolongation(int d, 
                                            int* array_num, 
                                            int anz_num, int level) {
  Index1D nextS;
  int num, iv;
  MPI_Request req[12];
  MPI_Status status[12];
  int num_message;
  double*  send_info;
  double*  receive_info;
  int number_send;
  int number_receive;

  //  int number_vars;
  double** var_sendd;
  double** var_received;

  int rank_source, rank_destination;

  // level is level with respect to coarse points
  // send point level is 1 larger
  if(I_am_active()) {
    // 0. part: basic infos about send and recieve
    if(d<6) {
      rank_source      = give_next_rank_source((dir_3D)d,level+1);
      rank_destination = give_next_rank_destination((dir_3D)d,level+1);  
    }
    else if(d<18) {
      rank_source      = give_next_rank_source((Edges_cell)(d-6),level+1);
      rank_destination = give_next_rank_destination((Edges_cell)(d-6),level+1);  
    }
    else {
      rank_source      = give_next_rank_source((dir_sons)(d-18),level+1);
      rank_destination = give_next_rank_destination((dir_sons)(d-18),level+1);
    }

    // 1. Step: calc number to be send and receive
    number_send = 
      number_send_face_prolongation[level][d] * anz_num;
    number_receive = 
      number_receive_face_prolongation[level][d] * anz_num;
   
    // 2. Step: build informations about var(double) to be send  
    send_info    = Give_buffer_send(d,number_send); 
    receive_info = Give_buffer_receive(d,number_receive); 

    var_sendd=var_send_prolongation[level][d];
    for(num=0;num<number_send_face_prolongation[level][d];++num) {
      for(iv=0;iv<anz_num;++iv) {
        send_info[num*anz_num+iv] = 
          var_sendd[num][array_num[iv]];
      }
    }

    // 4. Step: send doubles
    num_message = 0;
    if(rank_source != -1 && number_receive>0) {
      MPI_Irecv(receive_info ,number_receive,
                MPI_DOUBLE,rank_source,130+d,comm,
                &req[num_message]);
      ++num_message;
    }      
    if(rank_destination != -1 && number_send>0) {
      MPI_Isend(send_info,number_send,
                MPI_DOUBLE,rank_destination,130+d,comm,
                &req[num_message]);
      ++num_message;
    }
    MPI_Waitall(num_message,req,status);

    // 5. part: set informations about var(double)
    var_received=var_receive_prolongation[level][d];
    for(num=0;num<number_receive_face_prolongation[level][d];++num) {
      for(iv=0;iv<anz_num;++iv) {
        var_received[num][array_num[iv]] = 
          receive_info[num*anz_num+iv];
      }
    }
  }
}



void Grid::Print_number_of_communication_doubles() {
  int sum, total_sum;
  int rank_destination, d;
  if(my_rank==0)
    cout << "\n Print number of communication doubles: " 
         << endl;
  for(int l=min_level;l<=max_level;++l) {
    sum = 0;
    if(I_am_active()) {
      for(d=0;d<26;++d) {
        if(d<6) {
          rank_destination = give_next_rank_destination((dir_3D)d,l);  
        }
        else if(d<18) {
          rank_destination = give_next_rank_destination((Edges_cell)(d-6),l);
        }
        else {
          rank_destination = give_next_rank_destination((dir_sons)(d-18),l);
        }
        
        if(rank_destination != -1)
          sum = sum + number_send_face[l][d];
      }
    }
    MPI_Allreduce(&sum,&total_sum,1,MPI_INT,MPI_SUM,comm);
    if(my_rank==0)
      cout << " level: " << l << " is: " << total_sum << endl;
  }
}








void Grid::Sum_ghost_nodes_Variable(int* array_num, int anz_num, int level) {
  for(int d=0;d<26;++d)
    Sum_ghost_nodes_Variable(d, array_num, anz_num, level);
}

void Grid::Sum_ghost_nodes_Variable(int d, 
                                    int* array_num, int anz_num, int level) {
  Index1D nextS;
  int num, iv;
  MPI_Request req[12];
  MPI_Status status[12];
  int num_message;
  double*  send_info;
  double*  receive_info;
  int number_send;
  int number_receive;

  //  int number_vars;
  double** var_sendd;
  double** var_received;

  int rank_source, rank_destination;


  //if(I_am_active() && Max_level()==level) {
  if(I_am_active() && anz_num != 0) {
    // 0. part: basic infos about send and recieve
   if(d<6) {
      rank_source      = give_next_rank_source((dir_3D)d,level);
      rank_destination = give_next_rank_destination((dir_3D)d,level);  
    }
    else if(d<18) {
      rank_source      = give_next_rank_source((Edges_cell)(d-6),level);
      rank_destination = give_next_rank_destination((Edges_cell)(d-6),level);  
    }
    else {
      rank_source      = give_next_rank_source((dir_sons)(d-18),level);
      rank_destination = give_next_rank_destination((dir_sons)(d-18),level);
    }

    // 1. Step: calc number to be send and receive
    number_send = 
      number_send_face[level][d] * anz_num;
    number_receive = 
      number_receive_face[level][d] * anz_num;
   
    // 2. Step: build informations about var(double) to be send  
    send_info    = Give_buffer_send(d,number_send); 
    receive_info = Give_buffer_receive(d,number_receive); 

    var_received=var_receive[level][d];
    for(num=0;num<number_receive_face[level][d];++num) {
      for(iv=0;iv<anz_num;++iv) {
        receive_info[num*anz_num+iv] =
        var_received[num][array_num[iv]];
      }
    }

    // 4. Step: send doubles
    num_message = 0;
    if(rank_destination != -1 && number_send>0) {
      MPI_Irecv(send_info,number_send,
                MPI_DOUBLE,rank_destination,26,comm,
                &req[num_message]);
      ++num_message;
    }
    if(rank_source != -1 && number_receive>0) {
      MPI_Isend(receive_info ,number_receive,
                MPI_DOUBLE,rank_source,26,comm,
                &req[num_message]);
      ++num_message;
    }      

    MPI_Waitall(num_message,req,status);

    // 5. part: set informations about var(double)
    if(rank_destination != -1 && number_send>0) {
      var_sendd=var_send[level][d];
      for(num=0;num<number_send_face[level][d];++num) {
        for(iv=0;iv<anz_num;++iv) {
          var_sendd[num][array_num[iv]] +=
            send_info[num*anz_num+iv];
        }
      } 
    }
  }
}

---