59 template<
unsigned Dim>
73 for (
unsigned int d=0; d<
Dim; d++) {
80 for (
unsigned int d=0; d<
Dim; d++) vfndi[d] =
Index(vpd[d]);
99 int* anyTrueElements =
new int[vnodes];
100 for (
int v=0; v < vnodes; v++) anyTrueElements[v] = 0;
103 typename std::vector<BareField<bool,Dim>* >
::iterator bfi;
104 for (bfi = weights.begin(); bfi != weights.end(); ++bfi) {
109 weightItr != weight.
end_if();
111 if ((*weightItr).second->IsCompressed()) {
112 if ((*weightItr).second->getCompressedData()) {
113 anyTrueElements[(*weightItr).second->getVnode()] += 1;
116 anyTrueElements[(*weightItr).second->getVnode()] += 1;
130 int* anyTrueElementsPartial =
new int[vnodes];
131 int notReceived = npe - 1;
132 while (notReceived > 0) {
134 int tag = partialTag;
136 msg2->
get(anyTrueElementsPartial);
139 for (
int v=0; v < vnodes; v++) {
143 if ((anyTrueElements[v] != 0) && (anyTrueElementsPartial[v] != 0)) {
144 ERRORMSG(
"VnodeMultiRepartition(): anyTrueElements[" << v <<
"] = "
145 << anyTrueElements[v] <<
" and anyTrueElementsPartial["
146 << v <<
"] = " << anyTrueElementsPartial[v]
147 <<
" ; inconsistent!" <<
endl);
149 anyTrueElements[v] += anyTrueElementsPartial[v];
154 msg->
put(anyTrueElements, anyTrueElements + vnodes);
157 delete [] anyTrueElementsPartial;
161 msg->
put(anyTrueElements, anyTrueElements + vnodes);
166 msg->
get(anyTrueElements);
174 int vnode = (*vfItr).second->getVnode();
183 (*vfItr).second->Compress(anyTrueElements[vnode]);
196 int nZeroSizeVnodes = 0;
200 if ((*lLocals).second->getDomain().size() == 0) {nZeroSizeVnodes += 1;}
205 if ((*lRemotes).second->getDomain().size() == 0) {nZeroSizeVnodes += 1;}
213 if (nZeroSizeVnodes != 0) {
214 WARNMSG(
"VnodeMultiRepartition() was not able to get a successful "
215 <<
"rebalance. So, it is leaving the FieldLayout vnode "
216 <<
"PE-partioning the way it was whenyou called it. Sorry about "
217 <<
"that; there just aren't enough noncompressed vnodes to go "
218 <<
"around for this many PEs---at least not enough located in a "
219 <<
"way that the underlying BinaryBalancer algorithm can deal "
222 delete [] anyTrueElements;
242 int* peOwner =
new int[vnodes];
244 for (
int v=0; v < vnodes; v++) peOwner[v] = -1;
254 for (lfi = (*vfItr).second->begin();
255 lfi != (*vfItr).second->end(); ++lfi) {
262 for (
unsigned int d=0; d<
Dim; d++) {
263 lfBase[d] = (*vfItr).second->getOwned()[d].first();
265 for (
unsigned int d=0; d<
Dim; d++) vfIndex[d] = lfi.
GetOffset(d) + lfBase[d];
267 int vnode = vfIndex[0];
269 for (
unsigned int d=1; d<
Dim; d++) {
270 multipplier *= vpd[d-1];
271 vnode += vfIndex[d]*multipplier;
273 if (vnode >= vnodes) {
274 ERRORMSG(
"VnodeMultiRepartition(): vnode = " << vnode
275 <<
" but vnodes is only " << vnodes <<
" ; inconsistent!"
278 "VnodeMultiRepartition: exit because of vnode value error.");
295 int* peOwnerPartial =
new int[vnodes];
296 int notReceived = npe - 1;
297 while (notReceived > 0) {
299 int tag = partialTag;
301 msg2->
getmsg((
void *)peOwnerPartial);
304 for (
int v=0; v < vnodes; v++) {
305 if (peOwnerPartial[v] != -1) {
309 if (peOwner[v] != -1) {
310 ERRORMSG(
"VnodeMultiRepartition(): peOwner[" << v <<
"] = "
311 << peOwner[v] <<
" and peOwnerPartial["
312 << v <<
"] = " << peOwnerPartial[v]
313 <<
" ; inconsistent!" <<
endl);
315 peOwner[v] = peOwnerPartial[v];
321 msg4->
put(peOwner, peOwner + vnodes);
324 delete [] peOwnerPartial;
328 msg4->
put(peOwner, peOwner + vnodes);
343 for (
int v=0; v < vnodes; v++)
if (peOwner[v] == pe) ++nVnodesIOwn;
356 int vnode = (*oldLocals).second->
getVnode();
357 if (peOwner[vnode] == pe) {
359 Vnode<Dim>((*oldLocals).second->getDomain(), pe, vnode);
369 int vnode = (*oldRemotes).second->getVnode();
370 if (peOwner[vnode] == pe) {
372 Vnode<Dim>((*oldRemotes).second->getDomain(), pe, vnode);
379 layout.
Repartition(domains, domains + nVnodesIOwn);
383 delete [] anyTrueElements;
397 if (nZeroSizeVnodes != 0) {
415 for (
int v=0; v < vnodes; v++) anyTrueElements[v] = 0;
418 std::vector<BareField<bool,Dim>* >
::iterator bfi2;
419 for (bfi2 = weights.begin(); bfi2 != weights.end(); ++bfi2) {
423 for (weight2Itr = weight2.
begin_if();
424 weight2Itr != weight2.
end_if();
426 if ((*weight2Itr).second->IsCompressed()) {
427 if ((*weight2Itr).second->getCompressedData()) {
428 anyTrueElements[(*weight2Itr).second->getVnode()] += 1;
431 anyTrueElements[(*weight2Itr).second->getVnode()] += 1;
438 double normfact =
sum(Abs(vf2));
439 normfact = normfact/vfndi.size();
440 double epsilon = 1.0/normfact;
454 int* anyTrueElementsPartial =
new int[vnodes];
455 int notReceived = npe - 1;
456 while (notReceived > 0) {
458 int tag = partialTag;
460 msg2->
get(anyTrueElementsPartial);
463 for (
int v=0; v < vnodes; v++) {
467 if ((anyTrueElements[v] != 0) && (anyTrueElementsPartial[v] != 0)) {
468 ERRORMSG(
"VnodeMultiRepartition(): anyTrueElements[" << v <<
"] = "
469 << anyTrueElements[v] <<
" and anyTrueElementsPartial["
470 << v <<
"] = " << anyTrueElementsPartial[v]
471 <<
" ; inconsistent!" <<
endl);
473 anyTrueElements[v] += anyTrueElementsPartial[v];
478 msg3->
put(anyTrueElements, anyTrueElements + vnodes);
481 delete [] anyTrueElementsPartial;
485 msg3->
put(anyTrueElements, anyTrueElements + vnodes);
490 msg3->
get(anyTrueElements);
499 int vnode = (*vf2Itr).second->getVnode();
508 (*vf2Itr).second->Compress(anyTrueElements[vnode]);
521 int nZeroSizeVnodes2 = 0;
524 for (l2Locals = l2.
begin_iv(); l2Locals != l2.
end_iv(); ++l2Locals) {
525 if ((*l2Locals).second->getDomain().size() == 0) {nZeroSizeVnodes2 += 1;}
529 for (l2Remotes = l2.begin_rdv(); l2Remotes != l2.end_rdv(); ++l2Remotes) {
530 if ((*l2Remotes).second->getDomain().size() == 0) {nZeroSizeVnodes2 += 1;}
535 if (nZeroSizeVnodes2 != 0) {
536 WARNMSG(
"VnodeMultiRepartition(): even on a desperate 2nd attempt by adding in some"
537 <<
"random nonzero vnodes, was not able to get a successful rebalance. So, "
538 <<
"leaving the FieldLayout partioning the way it was when you called"
539 <<
" VnodeMultiRepartition(). Sorry about that." <<
endl);
void VnodeMultiRepartition(FieldLayout< Dim > &layout, std::vector< BareField< bool, Dim > * > &weights)
#define VNMB_COMPLETE_TAG
ac_id_larray::iterator iterator_if
RandomNumberGen IpplRandom
T::PETE_Expr_t::PETE_Return_t sum(const PETE_Expr< T > &expr)
int next_tag(int t, int s=1000)
void Repartition(const NDIndex< Dim > *, const NDIndex< Dim > *)
unsigned getVnodesPerDirection(unsigned dir)
int GetOffset(unsigned d) const
iterator_dv end_rdv(const GuardCellSizes< Dim > &gc=gc0())
void BinaryRepartition(FieldLayout< Dim > &layout, BareField< double, Dim > &weights)
ac_id_vnodes::iterator iterator_iv
virtual int broadcast_others(Message *, int, bool delmsg=true)
Message & get(const T &cval)
Message & put(const T &val)
e_dim_tag getDistribution(unsigned int d) const
iterator_dv begin_rdv(const GuardCellSizes< Dim > &gc=gc0())
std::string::iterator iterator
Message * receive_block(int &node, int &tag)
static Communicate * Comm
bool send(Message *, int node, int tag, bool delmsg=true)
Inform & endl(Inform &inf)