src/IplPaws/PawsPtclAttribDataSource.cpp

Go to the documentation of this file.

// -*- C++ -*-
/***************************************************************************
 *
 * The IPPL Framework
 * 
 * This program was prepared by PSI. 
 * All rights in the program are reserved by PSI.
 * Neither PSI nor the author(s)
 * makes any warranty, express or implied, or assumes any liability or
 * responsibility for the use of this software
 *
 * Visit http://www.acl.lanl.gov/POOMS for more details
 *
 ***************************************************************************/

// -*- C++ -*-
/***************************************************************************
 *
 * The IPPL Framework
 * 
 *
 * Visit http://people.web.psi.ch/adelmann/ for more details
 *
 ***************************************************************************/

// IPPL include files
#include "IpplPaws/PawsPtclAttribDataSource.h"
#include "IpplPaws/PawsDataConnect.h"
#include "Particle/ParticleAttrib.h"
#include "Message/Message.h"
#include "Utility/IpplInfo.h"
#include "Utility/Pstring.h"
#include "Profile/Profiler.h"

// PAWS include files
#include "Paws/Paws.h"


// constructor: the name, the connection, the transfer method, the attrib
template<class T>
PawsParticleAttribDataSource<T>::PawsParticleAttribDataSource(const
                                                              char *nm,
      DataConnect *dc, int tm, ParticleAttrib<T>& pa)
  : DataSourceObject(nm, &pa, dc, tm), attrib(pa),
    pawsArray(0), pawsConnect(0), sizeinfo(0)
{
  TAU_TYPE_STRING(taustr, "void (char *, DataConnect *, int, " +
                  CT(pa));
  TAU_PROFILE(
    "PawsParticleAttribDataSource::PawsParticleAttribDataSource()",
    taustr, TAU_VIZ);

  // do general initialization
  string filestring = "paws";
  if (string(dc->ID()) != filestring) {
    ERRORMSG("Illegal DataConnect object for Paws Data Object." << endl);
    Connection = 0;
  } else {
    // get the data connection pointer for a PAWS connection
    pawsConnect = (PawsDataConnect *)dc;

    // figure out the transfer method
    if (TransferMethod == DataSource::DEFAULT)
      TransferMethod = dc->getDefaultTransferMethod();

    int mode = PAWS_OUT;

    if (TransferMethod == DataSource::INPUT)
      mode = PAWS_IN;
    else if (TransferMethod == DataSource::OUTPUT)
      mode = PAWS_OUT;
    else if (TransferMethod == DataSource::BOTH)
      mode = PAWS_INOUT;

    // Create a new PawsArrayData object to refer to this attribute
    // so that it can connect to other apps.
    pawsArray = new PawsArrayData<PawsType_t>(nm, mode,
                                              PAWS_SYNC,
                                              PAWS_COLUMN,
                                              *(pawsConnect->getPawsApp()));

    // we need an array to hold size integers, one for each node
    sizeinfo = new int[Ippl::getNodes()];
  }
}


// destructor
template<class T>
PawsParticleAttribDataSource<T>::~PawsParticleAttribDataSource() {
  TAU_TYPE_STRING(taustr, CT(*this) + " void ()");
  TAU_PROFILE("PawsParticleAttribDataSource::~PawsParticleAttribDataSource()", 
    taustr, TAU_VIZ);

  //Inform dbgmsg("~PawsParticleAttribDataSource", INFORM_ALL_NODES);

  // delete size info array
  if (sizeinfo != 0)
    delete [] sizeinfo;

  // delete (does disconnect) the array object, if we still have it
  //dbgmsg << "Deleting the PawsArrayData object ..." << endl;
  if (pawsArray != 0 && pawsConnect != 0) {
    //dbgmsg << "Waiting at barrier before deleting PawsArrayData ..." << endl;
    //Ippl::Comm->barrier();
    pawsConnect->barrier();
    delete pawsArray;
  }
}


// Update the object, that is, make sure the receiver of the data has a
// current and consistent snapshot of the current state.  Return success.
template<class T>
bool PawsParticleAttribDataSource<T>::update() {
  TAU_TYPE_STRING(taustr, CT(*this) + " bool ()");
  TAU_PROFILE("PawsParticleAttribDataSource::update()", taustr, TAU_VIZ);

  int localblocks = 0, i, first, last, stride, localinfo[3];
  int offset = 0, totalsize = 0;
  PawsType_t *ptrs[1];
  ptrs[0] = 0;

  //Inform dbgmsg("PawsPtclAttrib");
  //dbgmsg << "Updating dynamic array '" << name() << "' ..." << endl;
  pawsConnect->barrier();

  // if we are sending the data, first do an update, then a send
  if (TransferMethod == DataSource::OUTPUT ||
      TransferMethod == DataSource::BOTH) {

    // first do the update ... for particles, there is at most one block
    // on each processor, but of varying size.  Get info about each block.

    // first do a reduction to get size info for each block
    //dbgmsg << "Sending my local size = " << attrib.size() << " ..." <<endl;
    for (i=0; i < Ippl::getNodes(); ++i)
      sizeinfo[i] = (i == Ippl::myNode() ? attrib.size() : 0);

    reduce(sizeinfo, sizeinfo + Ippl::getNodes(), sizeinfo, OpAddAssign());

    // find our relative offset for our block of data
    //dbgmsg << "Received global block size info:";
    offset = totalsize = 0;
    for (i=0; i < Ippl::getNodes(); ++i) {
      //dbgmsg << "  " << sizeinfo[i];
      if (i < Ippl::myNode())
        offset += sizeinfo[i];
      totalsize += sizeinfo[i];
    }
    //dbgmsg << " ... my offset=" << offset << ", total=" << totalsize << endl;

    // set up all the layout information
    first = 0;
    last = Size * totalsize - 1;
    stride = 1;
    localinfo[0] = Size * offset;
    localinfo[1] = Size * (offset + attrib.size()) - 1;
    localinfo[2] = 1;

    if (attrib.size() > 0) {
      localblocks = 1;
      ptrs[0] = reinterpret_cast<PawsType_t *>(&(attrib[0]));
    } else {
      localblocks = 0;
    }

    // now update the paws array info
    //dbgmsg << "Doing resize with " << localblocks << " local blocks, ";
    //dbgmsg << attrib.size() << " local particles, and total domain = [";
    //dbgmsg << first << "," << last << "," << stride << "]" << endl;
    //pawsConnect->barrier();
    pawsArray->resize(ptrs, 1, &first, &last, &stride, localblocks, localinfo);

    // after the update, do a send
    pawsArray->send();

  } else {

    // wait for resize info, if necessary ... if this returns
    // PAWS_ERROR, we're not connected to anything so we don't
    // need to do a receive, etc.

    //dbgmsg << "Waiting for resize information ..." << endl;
    //pawsConnect->barrier();
    if (pawsArray->resizeWait() == PAWS_ERROR)
      return true;

    // first get the total domain
    pawsArray->size(&totalsize);
    totalsize /= Size;
    //dbgmsg << "After waiting for resize info, new size = " << totalsize;
    //dbgmsg << endl;

    // put N/P elements on each node, extending or deleting as necessary
    int onnode = totalsize / Ippl::getNodes();
    int extra = totalsize % Ippl::getNodes();
    offset = onnode * Ippl::myNode();
    if (Ippl::myNode() < extra) {
      onnode++;
      offset += Ippl::myNode();
    } else {
      offset += extra;
    }

    //dbgmsg << "Putting " << onnode << " particles with offset = " << offset;
    //dbgmsg << " on my node ..." << endl;
    if (attrib.size() < onnode)
      attrib.create(onnode - attrib.size());
    else if (attrib.size() > onnode)
      attrib.destroy(attrib.size() - onnode, onnode - 1);

    // set up description of the blocks on our local node
    first = 0;
    last = Size * totalsize - 1;
    stride = 1;
    localinfo[0] = Size * offset;
    localinfo[1] = Size * (offset + attrib.size()) - 1;
    localinfo[2] = 1;
    if (attrib.size() > 0) {
      localblocks = 1;
      ptrs[0] = reinterpret_cast<PawsType_t *>(&(attrib[0]));
    } else {
      localblocks = 0;
    }

    // now update the paws array info
    //dbgmsg << "Doing update with " << localblocks << " local blocks, ";
    //dbgmsg << attrib.size() << " local particles, and total domain = [";
    //dbgmsg << first << "," << last << "," << stride << "]" << endl;
    pawsArray->update(ptrs, 1, &first, &last, &stride, localblocks, localinfo);

    // after the update, do a send
    //dbgmsg << "Calling receive() ..." << endl;
    pawsArray->receive();
  }

  // just return, the user should do an update for the ParticleBase
  return true;
}


// Indicate to the receiver that we're allowing them time to manipulate the
// data (e.g., for a viz program, to rotate it, change representation, etc.)
// This should only return when the manipulation is done.
template<class T>
void PawsParticleAttribDataSource<T>::interact(const char *str) {
  TAU_TYPE_STRING(taustr, CT(*this) + " void ()");
  TAU_PROFILE("PawsParticleAttribDataSource::interact()", taustr, TAU_VIZ);

  // hand off control to Paws API
  pawsConnect->poll();
}

// List the object to a stream.  This is required by Paws.
// Currently, it is just a stub.
template<class T>
ostream& operator<<(ostream& out, const PawsParticleAttribDataSource<T>& attrib)
{
  out << "PawsParticleAttribDataSource" << endl;
  return out;
}

/***************************************************************************
 * $RCSfile: PawsPtclAttribDataSource.cpp,v $   $Author: adelmann $
 * $Revision: 1.1.1.1 $   $Date: 2003/01/23 07:40:32 $
 * IPPL_VERSION_ID: $Id: PawsPtclAttribDataSource.cpp,v 1.1.1.1 2003/01/23 07:40:32 adelmann Exp $ 
 ***************************************************************************/

---