00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011 #ifndef DISC_FIELD_H
00012 #define DISC_FIELD_H
00013
00014
00015 #ifdef IPPL_PRINTDEBUG
00016 #define DFDBG(x) x
00017 #define CDFDBG(x) x
00018 #else
00019 #define DFDBG(x)
00020 #define CDFDBG(x)
00021 #endif
00022
00023
00024 #include "Index/NDIndex.h"
00025 #include "Field/BrickExpression.h"
00026 #include "Field/Field.h"
00027 #include "Utility/DiscBuffer.h"
00028 #include "Utility/DiscConfig.h"
00029 #include "Utility/Pstring.h"
00030 #include "Utility/Inform.h"
00031 #include "Utility/vmap.h"
00032 #include "Utility/IpplTimings.h"
00033 #include <stdio.h>
00034 #include <stdlib.h>
00035 #include <unistd.h>
00036 #include <fcntl.h>
00037 #ifdef __MWERKS__
00038 #include <stat.h>
00039
00040 #else
00041 #include <sys/stat.h>
00042 #include <sys/types.h>
00043 #endif // __MWERKS__
00044
00045 #ifdef IPPL_STDSTL
00046 #include <vector>
00047 using std::vector;
00048 #else
00049 #include <vector.h>
00050 #endif // IPPL_STDSTL
00051
00052 #ifdef IPPL_USE_STANDARD_HEADERS
00053 #include <iostream>
00054 using namespace std;
00055 #else
00056 #include <iostream.h>
00057 #endif
00058
00059
00060 template<unsigned Dim, class T> class UniformCartesian;
00061 template<class T, unsigned Dim, class M, class C> class Field;
00062 template<unsigned Dim> class FieldLayout;
00063
00064
00065
00066
00067
00068 template <unsigned Dim, class T>
00069 struct DFOffsetData {
00070 int vnodedata[6*Dim];
00071 bool isCompressed;
00072 #if defined(IPPL_LONGLONG)
00073 long long offset;
00074 #else
00075 long offset;
00076 #endif
00077 T compressedVal;
00078 };
00079
00080
00081 template <unsigned Dim>
00082 class DiscField {
00083
00084 public:
00085
00086
00087
00088
00089
00090
00091
00092
00093
00094
00095
00096
00097 DiscField(const char* fname, const char* config, unsigned int numFields,
00098 const char* typestr = 0);
00099
00100
00101
00102
00103 DiscField(const char* fname, unsigned int numFields,
00104 const char* typestr = 0);
00105
00106
00107
00108
00109 DiscField(const char* fname, const char* config);
00110
00111
00112
00113
00114 DiscField(const char* fname);
00115
00116
00117 ~DiscField();
00118
00119
00120
00121
00122
00123
00124
00125 void query(int& numRecords, int& numFields, vector<int>& size) const;
00126
00127
00128 unsigned int get_NumRecords() const { return NumRecords; }
00129
00130
00131 unsigned int get_NumFields() const { return NumFields; }
00132
00133
00134 NDIndex<Dim> get_Domain() const { return Size; }
00135
00136
00137
00138
00139
00140
00141 unsigned int get_Dimension() const { return DataDimension; }
00142
00143
00144 const char *get_TypeString() { return TypeString.c_str(); }
00145
00146
00147 const char *get_DiscType() {
00148 if (DiscType.length() > 0)
00149 return DiscType.c_str();
00150 return 0;
00151 }
00152
00153
00154
00155
00156
00157
00158
00159
00160
00161
00162
00163
00164
00165
00166
00167
00168
00169
00170
00171
00172 template <class T, class M, class C>
00173 bool read(Field<T,Dim,M,C>& f, const NDIndex<Dim> &readDomain,
00174 unsigned int varID, unsigned int record) {
00175
00176
00177 bool canread = false;
00178 if (!ConfigOK) {
00179 ERRORMSG("Cannot read in DiscField::read - config file error." << endl);
00180 } else if (DataDimension != Dim) {
00181 ERRORMSG("Bad dimension "<< DataDimension <<" in DiscField::read"<<endl);
00182 ERRORMSG("(" << DataDimension << " != " << Dim << ")" << endl);
00183 } else if (WritingFile) {
00184 ERRORMSG("DiscField::read called for DiscField opened for write."<<endl);
00185 } else if (varID >= NumFields) {
00186 ERRORMSG(varID << " is a bad Field ID in DiscField::read." << endl);
00187 ERRORMSG("(" << varID << " is >= " << NumFields << ")" << endl);
00188 } else if (record >= NumRecords) {
00189 ERRORMSG(record << " is a bad record number in DiscField::read."<<endl);
00190 ERRORMSG("(" << record << " is >= " << NumRecords << ")" << endl);
00191 } else if (!(f.getLayout().getDomain().contains(readDomain))) {
00192 ERRORMSG("DiscField::read - the total field domain ");
00193 ERRORMSG(f.getLayout().getDomain() << " must contain the requested ");
00194 ERRORMSG("read domain " << readDomain << endl);
00195 } else if (!(get_Domain().contains(readDomain))) {
00196 ERRORMSG("DiscField::read - the DiscField domain ");
00197 ERRORMSG(get_Domain() << " must contain the requested ");
00198 ERRORMSG("read domain " << readDomain << endl);
00199 } else {
00200 canread = true;
00201 }
00202
00203
00204 if (!canread) {
00205 Ippl::abort("Exiting due to DiscField error.");
00206 return false;
00207 }
00208
00209
00210 typedef typename LField<T,Dim>::iterator LFI;
00211 typedef BrickExpression<Dim,LFI,LFI,OpAssign> Expr_t;
00212
00213
00214 static IpplTimings::TimerRef readtimer =
00215 IpplTimings::getTimer("DiscField read");
00216 IpplTimings::startTimer(readtimer);
00217
00218 DFDBG(string dbgmsgname("DF:read:"));
00219 DFDBG(dbgmsgname += Config->getConfigFile());
00220 DFDBG(Inform dbgmsg(dbgmsgname.c_str(), INFORM_ALL_NODES));
00221 DFDBG(dbgmsg << "At start of read: Field layout=" << f.getLayout()<<endl);
00222 DFDBG(dbgmsg << "At start of read: Read domain =" << readDomain << endl);
00223
00224
00225
00226 int tag = Ippl::Comm->next_tag(DF_READ_TAG, DF_TAG_CYCLE);
00227
00228
00229
00230 int expected = compute_expected(f.getLayout(), readDomain);
00231
00232
00233
00234
00235
00236
00237
00238
00239
00240
00241
00242
00243
00244 DFDBG(dbgmsg << "Reading data from " << numFiles()<<" filesets:"<<endl);
00245
00246 for (int sf=0; sf < numFiles(); ++sf) {
00247
00248
00249
00250 int outputDatafd = (-1);
00251
00252
00253 vector<DFOffsetData<Dim,T> > offdata;
00254
00255
00256 int vnodes = 0;
00257
00258
00259 int maxsize = 0;
00260
00261
00262 if (Ippl::myNode() == myBox0()) {
00263
00264
00265 vnodes = read_layout(record, sf);
00266
00267
00268 read_offset(varID, record, sf, offdata, vnodes);
00269 }
00270
00271
00272
00273
00274
00275
00276
00277
00278 distribute_offsets(offdata, vnodes, maxsize, readDomain);
00279
00280 DFDBG(dbgmsg << "After reading and distributing offset data: ");
00281 DFDBG(dbgmsg << "Node " << Ippl::myNode() << " will read ");
00282 DFDBG(dbgmsg << vnodes << " vnodes, with maxsize = " << maxsize);
00283 DFDBG(dbgmsg << endl);
00284
00285
00286
00287
00288
00289
00290 for (int vn=0; vn < vnodes; ++vn) {
00291
00292 NDIndex<Dim> vnodeblock;
00293 offset_data_to_domain(offdata[vn], vnodeblock);
00294
00295
00296
00297 if (! vnodeblock.touches(readDomain)) {
00298 DFDBG(dbgmsg << "Skipping vnode " << vn << ", no intersection ");
00299 DFDBG(dbgmsg << "between " << vnodeblock << " and ");
00300 DFDBG(dbgmsg << readDomain << endl);
00301
00302 continue;
00303 }
00304
00305
00306
00307
00308 int msdim = (Dim-1);
00309 int chunkelems = Ippl::chunkSize() / sizeof(T);
00310 NDIndex<Dim> chunkblock = chunk_domain(vnodeblock, chunkelems, msdim,
00311 offdata[vn].isCompressed);
00312
00313 DFDBG(dbgmsg << "Reading in chunks in blocks of size " << chunkblock);
00314 DFDBG(dbgmsg << " and max buffer elems = " << maxsize);
00315 DFDBG(dbgmsg << " in vnode " << vn << " with total domain ");
00316 DFDBG(dbgmsg << vnodeblock << endl);
00317
00318
00319
00320 NDIndex<Dim> currblock = vnodeblock;
00321 currblock[msdim] = Index(vnodeblock[msdim].first() - 1,
00322 vnodeblock[msdim].first() - 1);
00323 for (int md = (msdim+1); md < Dim; ++md)
00324 currblock[md] = Index(vnodeblock[md].first(),vnodeblock[md].first());
00325
00326
00327
00328
00329 Offset_t seekpos = (-1);
00330
00331
00332 int unread = vnodeblock.size();
00333 while (unread > 0) {
00334
00335
00336
00337
00338
00339 bool incrhigher=(currblock[msdim].last()==vnodeblock[msdim].last());
00340 int a = (incrhigher ?
00341 vnodeblock[msdim].first() :
00342 currblock[msdim].last() + 1);
00343 int b = a + chunkblock[msdim].length() - 1;
00344 if (b > vnodeblock[msdim].last())
00345 b = vnodeblock[msdim].last();
00346
00347
00348 currblock[msdim] = Index(a, b);
00349
00350
00351 if (incrhigher) {
00352 for (int cd = (msdim+1); cd < Dim; ++cd) {
00353 if (currblock[cd].last() < vnodeblock[cd].last()) {
00354
00355 currblock[cd] = Index(currblock[cd].first() + 1,
00356 currblock[cd].last() + 1);
00357 break;
00358 } else {
00359
00360 currblock[cd] = Index(vnodeblock[cd].first(),
00361 vnodeblock[cd].first());
00362 }
00363 }
00364 }
00365
00366
00367
00368
00369 int nelems = currblock.size();
00370 unread -= nelems;
00371
00372 DFDBG(dbgmsg << "Starting processing of chunk with domain ");
00373 DFDBG(dbgmsg << currblock << " in vnode " << vn);
00374 DFDBG(dbgmsg << " at offset = " << offdata[vn].offset << endl);
00375 DFDBG(dbgmsg << "After this, still have " << unread << " unread.");
00376 DFDBG(dbgmsg << endl);
00377
00378
00379 if (!offdata[vn].isCompressed && seekpos < 0) {
00380 seekpos = offdata[vn].offset * sizeof(T);
00381 DFDBG(dbgmsg << "Set seek position = " << seekpos << endl);
00382 }
00383
00384
00385
00386 if (! currblock.touches(readDomain)) {
00387 DFDBG(dbgmsg << "Skipping sub-vnode chunk " << currblock);
00388 DFDBG(dbgmsg << ", no intersection with readDomain ");
00389 DFDBG(dbgmsg << readDomain << endl);
00390
00391
00392 Offset_t readbytes = nelems * sizeof(T);
00393 seekpos += readbytes;
00394 DFDBG(dbgmsg << "Updating offset at end of skip operation to ");
00395 DFDBG(dbgmsg << seekpos << endl);
00396
00397
00398 continue;
00399 }
00400
00401
00402 NDIndex<Dim> readDomainSection = currblock.intersect(readDomain);
00403 DFDBG(dbgmsg << "Intersection of chunk " << currblock);
00404 DFDBG(dbgmsg << " and read domain " << readDomain << " = ");
00405 DFDBG(dbgmsg << readDomainSection << endl);
00406
00407
00408
00409 T *buffer = 0;
00410 if (!offdata[vn].isCompressed) {
00411
00412 if (outputDatafd < 0) {
00413 DFDBG(dbgmsg << "Opening input data file ...");
00414 DFDBG(dbgmsg << endl);
00415 outputDatafd = open_df_file_fd(Config->getFilename(sf), ".data",
00416 O_RDONLY);
00417 }
00418
00419
00420
00421
00422
00423
00424
00425
00426 long nbytes = maxsize*sizeof(T);
00427 #ifdef IPPL_DIRECTIO
00428 if (openedDirectIO) {
00429 nbytes += dioinfo.d_miniosz;
00430 size_t ndiff = nbytes % dioinfo.d_miniosz;
00431 if (ndiff > 0)
00432 nbytes += (dioinfo.d_miniosz - ndiff);
00433 }
00434 #endif
00435 buffer = static_cast<T *>(DiscBuffer::resize(nbytes));
00436 DFDBG(dbgmsg << "On box0: resized buf to " << DiscBuffer::size());
00437 DFDBG(dbgmsg << " bytes ... current block will need ");
00438 DFDBG(dbgmsg << nelems * sizeof(T) << " bytes." << endl);
00439
00440
00441
00442 T * readbuffer = buffer;
00443 Offset_t readbytes = nelems * sizeof(T);
00444 Offset_t readoffset = seekpos;
00445
00446
00447
00448 seekpos += readbytes;
00449
00450 #ifdef IPPL_DIRECTIO
00451
00452
00453 if (openedDirectIO) {
00454
00455
00456
00457
00458
00459 PAssert(readoffset >= 0);
00460 Offset_t extra = readoffset % dioinfo.d_miniosz;
00461 readoffset -= extra;
00462 DFDBG(dbgmsg << "DIO: Moving read offset back by " << extra);
00463 DFDBG(dbgmsg << " bytes, to readoffset = " << readoffset<<endl);
00464
00465
00466
00467
00468
00469 readbytes += extra;
00470 size_t ndiff = readbytes % dioinfo.d_miniosz;
00471 if (ndiff > 0)
00472 readbytes += (dioinfo.d_miniosz - ndiff);
00473 PAssert(nbytes >= readbytes);
00474 DFDBG(dbgmsg << "DIO: Adjusted readbytes from ");
00475 DFDBG(dbgmsg << (nelems * sizeof(T)) << " to " << readbytes);
00476 DFDBG(dbgmsg << endl);
00477
00478
00479
00480
00481 PAssert(extra % sizeof(T) == 0);
00482 buffer += (extra / sizeof(T));
00483 DFDBG(dbgmsg << "DIO: Adjusted buffer pointer forward ");
00484 DFDBG(dbgmsg << (extra / sizeof(T)) << " elements." << endl);
00485 }
00486 #endif
00487
00488
00489 DFDBG(dbgmsg << "Calling read_data with readbytes=" << readbytes);
00490 DFDBG(dbgmsg << ", readoffset=" << readoffset << endl);
00491 read_data(outputDatafd, readbuffer, readbytes, readoffset);
00492 }
00493
00494
00495
00496
00497
00498 DFDBG(dbgmsg << "Finding destination nodes for block with ");
00499 DFDBG(dbgmsg << "domain = " << currblock << ", compressed = ");
00500 DFDBG(dbgmsg << offdata[vn].isCompressed << " ..." << endl);
00501 DFDBG(dbgmsg << "We will use the portion " << readDomainSection);
00502 DFDBG(dbgmsg << " from this block." << endl);
00503
00504
00505
00506 typename FieldLayout<Dim>::touch_iterator_dv rv_i;
00507
00508 int remaining = readDomainSection.size();
00509
00510
00511
00512
00513
00514 typename FieldLayout<Dim>::touch_range_dv
00515 range(f.getLayout().touch_range_rdv(readDomainSection));
00516 for (rv_i = range.first; rv_i != range.second; ++rv_i) {
00517
00518
00519 NDIndex<Dim> ri = readDomainSection.intersect((*rv_i).first);
00520 DFDBG(dbgmsg << "Block intersects with remote domain ");
00521 DFDBG(dbgmsg << (*rv_i).first << " = " << ri << endl);
00522
00523
00524 int rnode = (*rv_i).second->getNode();
00525
00526
00527
00528 Message *msg = new Message;
00529 ri.putMessage(*msg);
00530 LFI cbi(buffer, ri, currblock, offdata[vn].compressedVal);
00531 cbi.TryCompress();
00532 cbi.putMessage(*msg, false);
00533 DFDBG(dbgmsg << "Sending subblock " << ri << " from block ");
00534 DFDBG(dbgmsg << currblock << " to node " << rnode);
00535 DFDBG(dbgmsg << " with tag " << tag << endl);
00536 Ippl::Comm->send(msg, rnode, tag);
00537
00538
00539 remaining -= ri.size();
00540 DFDBG(dbgmsg << "After send, remaining = " << remaining << endl);
00541 }
00542
00543
00544
00545 typename BareField<T,Dim>::iterator_if lf_i = f.begin_if();
00546 for (; remaining > 0 && lf_i != f.end_if(); ++lf_i) {
00547
00548
00549 LField<T,Dim> &lf = *(*lf_i).second;
00550 const NDIndex<Dim>& lo = lf.getOwned();
00551
00552 const NDIndex<Dim>& ro = readDomainSection;
00553
00554
00555 if (lo.touches(ro)) {
00556
00557 NDIndex<Dim> ri = lo.intersect(ro);
00558
00559 DFDBG(dbgmsg << "Doing local copy of domain " << ri);
00560 DFDBG(dbgmsg << " into LField with domain " << lo << endl);
00561
00562
00563 if (lf.IsCompressed() &&
00564 offdata[vn].isCompressed &&
00565 ro.contains(lo)) {
00566 DFDBG(dbgmsg << " Doing comp-comp assign." << endl);
00567 PETE_apply(OpAssign(),*lf.begin(),offdata[vn].compressedVal);
00568 } else {
00569
00570
00571 LFI rhs_i(buffer, ri, currblock, offdata[vn].compressedVal);
00572
00573
00574 if (rhs_i.CanCompress(*rhs_i) && f.compressible() &&
00575 ri.contains(lf.getAllocated())) {
00576
00577 DFDBG(dbgmsg << " Doing lfield-comp assign." << endl);
00578 lf.Compress(*rhs_i);
00579 } else {
00580
00581 lf.Uncompress(true);
00582
00583
00584 LFI lhs_i = lf.begin(ri);
00585
00586
00587 DFDBG(dbgmsg << " Doing uncomp-uncomp assign." << endl);
00588 Expr_t(lhs_i,rhs_i).apply();
00589 }
00590 }
00591
00592
00593
00594
00595
00596 int bsize = ri.size();
00597 remaining -= bsize;
00598 expected -= bsize;
00599
00600 DFDBG(dbgmsg << "Finished copying in local data, now ");
00601 DFDBG(dbgmsg << "expecting " << expected << " elems with ");
00602 DFDBG(dbgmsg << remaining << " elems remaining." << endl);
00603 }
00604 }
00605
00606
00607 if (remaining > 0)
00608 Ippl::abort("remaining > 0 at end of box0 vnode read!!!");
00609 }
00610 }
00611
00612
00613
00614 if (outputDatafd >= 0)
00615 close(outputDatafd);
00616 }
00617
00618
00619
00620 while (expected > 0) {
00621
00622 int node = COMM_ANY_TAG;
00623 DFDBG(dbgmsg << "Waiting for DF data, still expecting " << expected);
00624 DFDBG(dbgmsg << " elements ..." << endl);
00625 Message *msg = Ippl::Comm->receive_block(node, tag);
00626
00627
00628 NDIndex<Dim> ro;
00629 ro.getMessage(*msg);
00630 DFDBG(dbgmsg << "Received DF data from node " << node << " with tag ");
00631 DFDBG(dbgmsg << tag << ", with domain = " << ro << endl);
00632
00633
00634 T rhs_compressed_data;
00635 LFI rhs_i(rhs_compressed_data);
00636 rhs_i.getMessage(*msg);
00637
00638
00639 typename BareField<T,Dim>::iterator_if lf_i = f.begin_if();
00640 bool foundlf = false;
00641 for (; lf_i != f.end_if(); ++lf_i) {
00642
00643 LField<T,Dim> &lf = *(*lf_i).second;
00644 const NDIndex<Dim>& lo = lf.getOwned();
00645
00646
00647
00648 if (lo.contains(ro)) {
00649 DFDBG(dbgmsg << "Found local lfield with domain " << lo);
00650 DFDBG(dbgmsg << " that contains received domain " << ro << endl);
00651
00652
00653 if ( !(rhs_i.IsCompressed() && lf.IsCompressed() &&
00654 (*rhs_i == *lf.begin())) ) {
00655
00656
00657
00658
00659
00660 lf.Uncompress(!ro.contains(lo));
00661
00662 DFDBG(dbgmsg << "Assigning value: lhs compressed = ");
00663 DFDBG(dbgmsg << lf.IsCompressed() << ", rhs compressed = ");
00664 DFDBG(dbgmsg << rhs_i.IsCompressed() << endl);
00665
00666
00667
00668 LFI lhs_i = lf.begin(ro);
00669
00670
00671 Expr_t(lhs_i,rhs_i).apply();
00672 } else {
00673 DFDBG(dbgmsg << "Local LField is compressed and has same value ");
00674 DFDBG(dbgmsg << "as received data." << endl);
00675 }
00676
00677
00678 expected -= ro.size();
00679
00680
00681
00682 foundlf = true;
00683 break;
00684 }
00685 }
00686
00687
00688
00689 if (!foundlf) {
00690 ERRORMSG("Did not find destination local vnode for received domain ");
00691 ERRORMSG(ro << " from node " << node << endl);
00692 Ippl::abort("DID NOT FIND DESINATION LOCAL VNODE IN DISCFIELD::READ");
00693 }
00694
00695
00696 delete msg;
00697 }
00698
00699
00700 IpplTimings::stopTimer(readtimer);
00701
00702
00703
00704
00705 DFDBG(dbgmsg << "Finished with read. Updating field GC's." << endl);
00706 f.setDirtyFlag();
00707 f.fillGuardCellsIfNotDirty();
00708 f.Compress();
00709
00710
00711 Ippl::Comm->barrier();
00712
00713
00714
00715
00716
00717
00718
00719
00720
00721
00722
00723
00724
00725
00726
00727 return true;
00728 }
00729
00730
00731 template <class T, class M, class C>
00732 bool read(Field<T,Dim,M,C>& f, unsigned int varID, unsigned int record) {
00733 return read(f, f.getLayout().getDomain(), varID, record);
00734 }
00735
00736 template <class T, class M, class C>
00737 bool read(Field<T,Dim,M,C>& f, const NDIndex<Dim> &readDomain,
00738 unsigned int varID) {
00739 return read(f, readDomain, varID, 0);
00740 }
00741
00742 template <class T, class M, class C>
00743 bool read(Field<T,Dim,M,C>& f, unsigned int varID) {
00744 return read(f, f.getLayout().getDomain(), varID, 0);
00745 }
00746
00747 template <class T, class M, class C>
00748 bool read(Field<T,Dim,M,C>& f, const NDIndex<Dim> &readDomain) {
00749 return read(f, readDomain, 0, 0);
00750 }
00751
00752 template <class T, class M, class C>
00753 bool read(Field<T,Dim,M,C>& f) {
00754 return read(f, f.getLayout().getDomain(), 0, 0);
00755 }
00756
00757
00759
00760
00761
00762
00763
00764
00765
00766
00767
00768
00769
00770
00771
00772
00773
00774 template<class T, class M, class C>
00775 bool write(Field<T,Dim,M,C>& f, unsigned int varID) {
00776 TAU_TYPE_STRING(taustr, "bool (" + CT(f) + ", unsigned int)" );
00777 TAU_PROFILE("DiscField::write()", taustr,
00778 TAU_UTILITY | TAU_FIELD | TAU_IO);
00779
00780
00781 if (!ConfigOK) {
00782 ERRORMSG("Cannot write in DiscField::write - config file error."<<endl);
00783 Ippl::abort("Exiting due to DiscField error.");
00784 return false;
00785 }
00786 else if (!WritingFile) {
00787 ERRORMSG("DiscField::write called for DiscField opened for read."<<endl);
00788 Ippl::abort("Exiting due to DiscField error.");
00789 return false;
00790 }
00791 else if (varID >= NumFields) {
00792 ERRORMSG(varID << " is a bad variable ID in DiscField::write." << endl);
00793 Ippl::abort("Exiting due to DiscField error.");
00794 return false;
00795 }
00796 else if (NeedStartRecord == 0 && ValidField[varID]) {
00797 ERRORMSG("DiscField:write - attempt to overwrite Field " << varID);
00798 ERRORMSG(" at record " << NumRecords - 1 << endl);
00799 Ippl::abort("Exiting due to DiscField error.");
00800 return false;
00801 }
00802
00803 DFDBG(string dbgmsgname("DF:write:"));
00804 DFDBG(dbgmsgname += Config->getConfigFile());
00805 DFDBG(Inform dbgmsg(dbgmsgname.c_str(), INFORM_ALL_NODES));
00806 DFDBG(dbgmsg << "At start of write: Field layout=" << f.getLayout()<<endl);
00807
00808 INCIPPLSTAT(incDiscWrites);
00809
00810
00811 typedef typename LField<T,Dim>::iterator LFI;
00812
00813
00814
00815 int tag = Ippl::Comm->next_tag(FB_WRITE_TAG, FB_TAG_CYCLE);
00816
00817
00818 FieldLayout<Dim>& layout = f.getLayout();
00819 typename Field<T,Dim,M,C>::iterator_if local;
00820
00821
00822
00823 if (NeedStartRecord != 0) {
00824
00825
00826 if (!make_globalID(layout)) {
00827 ERRORMSG("DiscField::write - all Field's must have the same ");
00828 ERRORMSG("global domain in a single DiscField.\n");
00829 ERRORMSG("The original domain is " << get_Domain() << ", ");
00830 ERRORMSG("the attempted new domain is " << layout.getDomain() << endl);
00831 Ippl::abort("Exiting due to DiscField error.");
00832 }
00833
00834
00835 if (numFiles() > 0 && myBox0() == Ippl::myNode()) {
00836 int nvtally = 0;
00837 NumVnodes[0].push_back(globalID.size());
00838 if (NumRecords > 0)
00839 nvtally = VnodeTally[0][NumRecords-1] + NumVnodes[0][NumRecords-1];
00840 VnodeTally[0].push_back(nvtally);
00841 }
00842
00843
00844 for (unsigned int i=0; i < NumFields; ++i)
00845 ValidField[i] = false;
00846
00847
00848 NumRecords++;
00849 NumWritten = 0;
00850
00851
00852 if (Ippl::myNode() == myBox0()) {
00853
00854
00855 DFDBG(dbgmsg << "Writing meta file ..." << endl);
00856 if (!write_meta()) {
00857 ERRORMSG("Could not write .meta file on node " << Ippl::myNode());
00858 ERRORMSG(endl);
00859 Ippl::abort("Exiting due to DiscField error.");
00860 return false;
00861 }
00862
00863
00864
00865 DFDBG(dbgmsg << "Writing layout file ..." << endl);
00866 if (!write_layout()) {
00867 ERRORMSG("Could not update .layout file on node "<<Ippl::myNode());
00868 ERRORMSG(endl);
00869 Ippl::abort("Exiting due to DiscField error.");
00870 return false;
00871 }
00872 }
00873 }
00874
00875
00876
00877 if (Ippl::myNode() == myBox0()) {
00878
00879
00880
00881
00882
00883
00884 FILE *outputOffset = open_df_file(Config->getFilename(0),
00885 ".offset", string("a"));
00886 int wVarID = (int)varID;
00887 DFDBG(dbgmsg << "Writing Field ID = " << wVarID<<" to offset file ...");
00888 DFDBG(dbgmsg << endl);
00889 if (fwrite(&wVarID, sizeof(int), 1, outputOffset) != 1) {
00890 ERRORMSG("DiscField::write - cannot write field number to .offset ");
00891 ERRORMSG("file" << endl);
00892 Ippl::abort("Exiting due to DiscField error.");
00893 fclose(outputOffset);
00894 return false;
00895 }
00896
00897
00898
00899 DFDBG(dbgmsg << "Trying to open output data file ..." << endl);
00900 int outputDatafd = open_df_file_fd(Config->getFilename(0), ".data",
00901 O_RDWR|O_CREAT);
00902 DFDBG(dbgmsg << "Opened out file, fd=" << outputDatafd << endl);
00903
00904
00905
00906
00907 DFDBG(dbgmsg << "This box0 expected to receive " << globalID.size());
00908 DFDBG(dbgmsg << " blocks from other nodes, minus the ");
00909 DFDBG(dbgmsg << layout.size_iv() << " local blocks." << endl);
00910 int unreceived = globalID.size();
00911 int fromothers = unreceived - layout.size_iv();
00912
00913
00914
00915
00916
00917
00918 local = f.begin_if();
00919 while (unreceived > 0) {
00920
00921
00922 bool checkremote = (fromothers > 0);
00923 while (checkremote) {
00924
00925 int any_node = COMM_ANY_NODE;
00926 Message *msg = Ippl::Comm->receive(any_node, tag);
00927
00928
00929 if (msg != 0) {
00930
00931 NDIndex<Dim> ro;
00932 ro.getMessage(*msg);
00933 DFDBG(dbgmsg << "Received an LField msg from node ");
00934 DFDBG(dbgmsg << any_node << " with tag " << tag << ", domain = ");
00935 DFDBG(dbgmsg << ro << endl);
00936
00937
00938 T rhs_compressed_data;
00939 LFI cbi(rhs_compressed_data);
00940 cbi.getMessage(*msg);
00941
00942
00943 write_offset_and_data(outputOffset, outputDatafd, cbi, ro);
00944
00945
00946 delete msg;
00947
00948
00949 unreceived -= 1;
00950 fromothers -= 1;
00951 } else {
00952
00953 checkremote = false;
00954 }
00955 }
00956
00957
00958 if (local != f.end_if()) {
00959
00960 LField<T,Dim>& l = *(*local).second.get();
00961 LFI cbi = l.begin();
00962
00963
00964 write_offset_and_data(outputOffset, outputDatafd, cbi, l.getOwned());
00965
00966
00967 ++local;
00968 unreceived -= 1;
00969 }
00970 }
00971
00972
00973 if (outputDatafd >= 0)
00974 close(outputDatafd);
00975
00976
00977 if (outputOffset != 0)
00978 fclose(outputOffset);
00979
00980 } else {
00981
00982 for (local = f.begin_if(); local != f.end_if(); ++local) {
00983
00984 LField<T,Dim>& l = *(*local).second.get();
00985 const NDIndex<Dim> &ro = l.getOwned();
00986 LFI cbi = l.begin();
00987
00988
00989 Message *msg = new Message;
00990
00991
00992 ro.putMessage(*msg);
00993 cbi.putMessage(*msg, false);
00994
00995
00996 int node = myBox0();
00997 DFDBG(dbgmsg << "Sending local block " << ro << " to node " << node);
00998 DFDBG(dbgmsg << " with tag " << tag << endl);
00999 Ippl::Comm->send(msg, node, tag);
01000 }
01001 }
01002
01003
01004 ValidField[varID] = true;
01005 NeedStartRecord = (++NumWritten == NumFields);
01006
01007
01008 Ippl::Comm->barrier();
01009
01010
01011 return true;
01012 }
01013
01014
01015 template<class T, class M, class C>
01016 bool write(Field<T,Dim,M,C>& f) {
01017 return write(f, 0);
01018 }
01019
01020
01021
01022
01023
01024
01025
01026 void printDebug(ostream&);
01027 void printDebug();
01028
01029 private:
01030
01031 typedef vmap<NDIndex<Dim>, int> GlobalIDList_t;
01032 #if defined(IPPL_LONGLONG)
01033 typedef long long Offset_t;
01034 #else
01035 typedef long Offset_t;
01036 #endif
01037
01038
01039
01040
01041
01042
01043 DiscConfig *Config;
01044 bool ConfigOK;
01045
01046
01047 bool WritingFile;
01048
01049
01050 string BaseFile;
01051 string TypeString;
01052 string DiscType;
01053
01054
01055 unsigned int DataDimension;
01056
01057
01058
01059
01060
01061
01062
01063
01064 int NeedStartRecord;
01065
01066
01067
01068 unsigned int NumFields;
01069 unsigned int NumRecords;
01070 unsigned int NumWritten;
01071
01072
01073
01074 Offset_t CurrentOffset;
01075
01076
01077 NDIndex<Dim> Size;
01078
01079
01080 vector<bool> ValidField;
01081
01082
01083
01084
01085
01086 vector<int> *VnodeTally;
01087
01088
01089
01090
01091
01092 vector<int> *NumVnodes;
01093
01094
01095
01096
01097
01098
01099
01100
01101 GlobalIDList_t globalID;
01102
01103
01104 #ifdef IPPL_DIRECTIO
01105 struct dioattr dioinfo;
01106 bool openedDirectIO;
01107 #endif
01108
01109
01110
01111
01112
01113
01114 void initialize(const char* base, const char* config,
01115 const char* typestr, unsigned int numFields);
01116
01117
01118
01119
01120
01121 FILE *open_df_file(const string& fnm, const string& mode);
01122 FILE *open_df_file(const string& fnm, const string& suffix,
01123 const string& mode);
01124
01125
01126
01127
01128
01129
01130 int open_df_file_fd(const string& fnm, const string& suf, int flags);
01131
01132
01133 bool create_files();
01134
01135
01136 unsigned int numSMPs() const {
01137 return Config->numSMPs();
01138 }
01139
01140
01141 unsigned int fileSMPs() const {
01142 return Config->fileSMPs();
01143 }
01144
01145
01146 unsigned int mySMP() const {
01147 return Config->mySMP();
01148 }
01149
01150
01151 unsigned int myBox0() const {
01152 return Config->getSMPBox0();
01153 }
01154
01155
01156
01157 unsigned int numFiles() const {
01158 return Config->getNumFiles();
01159 }
01160 unsigned int numFiles(unsigned int s) const {
01161 return Config->getNumFiles(s);
01162 }
01163
01164
01165
01166 unsigned int pNodesPerSMP(unsigned int node) const {
01167 return Config->pNodesPerSMP(node);
01168 }
01169
01170
01171
01172
01173 bool parse_config(const char *, bool);
01174
01175
01176
01177
01178
01179
01180 int compute_expected(const FieldLayout<Dim> &, const NDIndex<Dim> &);
01181
01182
01183
01184
01185
01186 bool make_globalID(FieldLayout<Dim> &);
01187
01188
01189
01190 NDIndex<Dim> chunk_domain(const NDIndex<Dim> &currblock,
01191 int chunkelems,
01192 int &msdim,
01193 bool iscompressed);
01194
01195
01196
01197
01198
01199
01200
01201 bool write_meta();
01202 bool read_meta();
01203
01204
01205 bool read_NDIndex(FILE *, NDIndex<Dim> &);
01206 bool write_NDIndex(FILE *, const NDIndex<Dim> &);
01207
01208
01209 bool write_layout();
01210
01211
01212 int read_layout(int record, int sf);
01213
01215
01216
01217
01218
01219
01220
01221
01222
01223
01224
01225
01226
01227
01228
01229
01230
01231 template<class T>
01232 void write_offset_and_data(FILE *outputOffset, int outputDatafd,
01233 CompressedBrickIterator<T,Dim> &cbi,
01234 const NDIndex<Dim> &owned) {
01235 TAU_TYPE_STRING(taustr, "void (FILE *, FILE *, " + CT(cbi) + ", NDIndex");
01236 TAU_PROFILE("DiscField::write_offset_and_data()", taustr,
01237 TAU_UTILITY | TAU_FIELD | TAU_IO);
01238
01239 DFDBG(string dbgmsgname("DF:write_offset_and_data"));
01240 DFDBG(Inform dbgmsg(dbgmsgname.c_str(), INFORM_ALL_NODES));
01241
01242
01243
01244 DFOffsetData<Dim,T> offset;
01245 memset(static_cast<void *>(&offset), 0, sizeof(DFOffsetData<Dim,T>));
01246
01247 domain_to_offset_data(owned, offset);
01248 offset.isCompressed = cbi.IsCompressed();
01249 offset.offset = CurrentOffset;
01250
01251
01252 if (offset.isCompressed) {
01253
01254
01255 offset.compressedVal = *cbi;
01256 DFDBG(dbgmsg << " Writing compressed vnode " << owned <<" w/value = ");
01257 DFDBG(dbgmsg << offset.compressedVal << endl);
01258
01259 } else {
01260
01261
01262
01263
01264
01265
01266
01267
01268
01269 long elems = owned.size();
01270 long chunkbytes = Ippl::chunkSize();
01271 #ifdef IPPL_DIRECTIO
01272 if (openedDirectIO) {
01273
01274
01275 PAssert(dioinfo.d_miniosz % sizeof(T) == 0);
01276 if (chunkbytes == 0 || chunkbytes > dioinfo.d_maxiosz)
01277 chunkbytes = dioinfo.d_maxiosz;
01278 else if (chunkbytes < dioinfo.d_miniosz)
01279 chunkbytes = dioinfo.d_miniosz;
01280 else if (chunkbytes % dioinfo.d_miniosz > 0)
01281 chunkbytes -= (chunkbytes % dioinfo.d_miniosz);
01282 }
01283 #endif
01284 long chunksize = chunkbytes / sizeof(T);
01285 if (chunksize < 1 || chunksize > elems)
01286 chunksize = elems;
01287
01288 DFDBG(dbgmsg << "Total elems = " << elems << endl);
01289 DFDBG(dbgmsg << "Bytes in each chunk = " << chunkbytes);
01290 DFDBG(dbgmsg << " (orig chunkbytes = " << Ippl::chunkSize()<<")"<<endl);
01291 DFDBG(dbgmsg << "Elems in each chunk = " << chunksize << endl);
01292
01293
01294
01295
01296
01297
01298 T *cbiptr = 0;
01299 if (cbi.whole())
01300 cbiptr = &(*cbi);
01301
01302
01303
01304 DFDBG(dbgmsg << " Writing vnode " << owned << " in ");
01305 DFDBG(dbgmsg << "chunks of " << chunksize << " elements for ");
01306 DFDBG(dbgmsg << elems << " total elements ..." << endl);
01307
01308 int needwrite = elems;
01309 while (needwrite > 0) {
01310
01311 int amount = chunksize;
01312 if (amount > needwrite)
01313 amount = needwrite;
01314
01315
01316
01317
01318
01319 size_t nbytes = amount*sizeof(T);
01320 #ifdef IPPL_DIRECTIO
01321 if (openedDirectIO) {
01322 size_t ndiff = nbytes % dioinfo.d_miniosz;
01323 if (ndiff > 0)
01324 nbytes += (dioinfo.d_miniosz - ndiff);
01325 }
01326 #endif
01327 DFDBG(dbgmsg << " Will write total nbytes = " << nbytes);
01328 DFDBG(dbgmsg << ", this has extra " << (nbytes - amount*sizeof(T)));
01329 DFDBG(dbgmsg << " bytes." << endl);
01330
01331
01332
01333 T *buffer = cbiptr;
01334
01335
01336 if (buffer == 0) {
01337 DFDBG(dbgmsg << " Getting copy buffer of total size ");
01338 DFDBG(dbgmsg << nbytes << " bytes ..." << endl);
01339 buffer = static_cast<T *>(DiscBuffer::resize(nbytes));
01340
01341
01342 DFDBG(dbgmsg << " Copying data into buffer ..." << endl);
01343 T *bufptr = buffer;
01344 T *bufend = buffer + amount;
01345 for ( ; bufptr != bufend; ++bufptr, ++cbi)
01346 new (bufptr) T(*cbi);
01347 }
01348
01349
01350 DFDBG(dbgmsg << " About to write " << nbytes << " to fd = ");
01351 DFDBG(dbgmsg << outputDatafd << endl);
01352
01353 off_t seekoffset = CurrentOffset * sizeof(T);
01354 bool seekok = true;
01355 Timer wtimer;
01356 wtimer.clear();
01357 wtimer.start();
01358
01359 #ifdef IPPL_DIRECTIO
01360 long nout = ::pwrite(outputDatafd, buffer, nbytes, seekoffset);
01361 #else
01362 long nout = 0;
01363 if (::lseek(outputDatafd, seekoffset, SEEK_SET) == seekoffset) {
01364 char *wbuf = (char *)buffer;
01365 nout = ::write(outputDatafd, wbuf, nbytes);
01366 } else {
01367 seekok = false;
01368 }
01369 #endif
01370
01371 wtimer.stop();
01372 DiscBuffer::writetime += wtimer.clock_time();
01373 DiscBuffer::writebytes += nbytes;
01374
01375 if (!seekok) {
01376 ERRORMSG("Seek error in DiscField::write_offset_and_data" << endl);
01377 ERRORMSG("Could not seek to position " << seekoffset << endl);
01378 Ippl::abort("Exiting due to DiscField error.");
01379 }
01380
01381 if (nout != nbytes) {
01382 ERRORMSG("Write error in DiscField::write_offset_and_data" << endl);
01383 ERRORMSG("Could not write " << nbytes << " bytes." << endl);
01384 Ippl::abort("Exiting due to DiscField error.");
01385 }
01386
01387 DFDBG(dbgmsg << " Finished writing " << nout << " bytes in ");
01388 DFDBG(dbgmsg << wtimer.clock_time() << " seconds." << endl);
01389
01390
01391 needwrite -= amount;
01392 if (cbiptr != 0)
01393 cbiptr += amount;
01394
01395
01396
01397 CurrentOffset += (nbytes / sizeof(T));
01398 ADDIPPLSTAT(incDiscBytesWritten, nbytes);
01399
01400 DFDBG(dbgmsg << " Finishing writing chunk, still " << needwrite);
01401 DFDBG(dbgmsg << " elements to write out from this block." << endl);
01402 }
01403 }
01404
01405
01406 DFDBG(dbgmsg << "Writing offset data to file, iscompressed = ");
01407 DFDBG(dbgmsg << offset.isCompressed);
01408 DFDBG(dbgmsg << ", offset = " << offset.offset << endl);
01409 if (fwrite(&offset, sizeof(DFOffsetData<Dim,T>), 1, outputOffset) != 1) {
01410 ERRORMSG("Write error in DiscField::write_offset_and_data" << endl);
01411 Ippl::abort("Exiting due to DiscField error.");
01412 }
01413 }
01414
01416
01417
01418
01419
01420 template <class T>
01421 bool read_offset(unsigned int varID,
01422 unsigned int record,
01423 unsigned int sf,
01424 vector<DFOffsetData<Dim,T> > &offdata,
01425 int vnodes) {
01426 TAU_TYPE_STRING(taustr, CT(*this) +
01427 " bool (unsigned int, unsigned int, unsigned int)");
01428 TAU_PROFILE("DiscField::read_offset()", taustr,
01429 TAU_UTILITY | TAU_FIELD | TAU_IO);
01430
01431
01432 FILE *outputOffset = open_df_file(Config->getFilename(sf),
01433 ".offset", string("r"));
01434
01435
01436 Offset_t seekpos = NumFields * (record * sizeof(int) +
01437 VnodeTally[sf][record] *
01438 sizeof(DFOffsetData<Dim,T>));
01439 if (fseek(outputOffset, seekpos, SEEK_SET) != 0) {
01440 ERRORMSG("Error seeking to position " << static_cast<long>(seekpos));
01441 ERRORMSG(" in .offset file " << endl);
01442 Ippl::abort("Exiting due to DiscField error.");
01443 fclose(outputOffset);
01444 return false;
01445 }
01446
01447
01448
01449 unsigned int checked = 0;
01450 while (checked < NumFields) {
01451
01452 int rVarID;
01453 if (fread(&rVarID, sizeof(int), 1, outputOffset) != 1) {
01454 ERRORMSG("Error reading field ID from .offset file" << endl);
01455 Ippl::abort("Exiting due to DiscField error.");
01456 fclose(outputOffset);
01457 return false;
01458 }
01459
01460
01461 if (rVarID == varID) {
01462
01463
01464 offdata.resize(vnodes);
01465 int result = fread(&(offdata[0]), sizeof(DFOffsetData<Dim,T>),
01466 offdata.size(), outputOffset);
01467 if (result != offdata.size()) {
01468 ERRORMSG("Read error in DiscField::find_file_in_offset" << endl);
01469 ERRORMSG("Results is " << result << ", should be ");
01470 ERRORMSG(offdata.size() << endl);
01471 ERRORMSG("outputOffset is " << (void *)outputOffset << endl);
01472 Ippl::abort("Exiting due to DiscField error.");
01473 fclose(outputOffset);
01474 return false;
01475 }
01476
01477
01478 fclose(outputOffset);
01479 return true;
01480 }
01481
01482
01483 checked++;
01484 seekpos += (NumVnodes[sf][record] * sizeof(DFOffsetData<Dim,T>) +
01485 sizeof(int));
01486 if (fseek(outputOffset, seekpos, SEEK_SET) != 0) {
01487 ERRORMSG("Error seeking to position " << static_cast<long>(seekpos));
01488 ERRORMSG(" in .offset file " << endl);
01489 Ippl::abort("Exiting due to DiscField error.");
01490 fclose(outputOffset);
01491 return false;
01492 }
01493 }
01494
01495
01496 ERRORMSG("Could not find data for field " << varID << " of record ");
01497 ERRORMSG(record << " in .offset file." << endl);
01498 Ippl::abort("Exiting due to DiscField error.");
01499 fclose(outputOffset);
01500 return false;
01501 }
01502
01504
01505
01506
01507
01508
01509
01510
01511 template <class T>
01512 void distribute_offsets(vector<DFOffsetData<Dim,T> > &offdata,
01513 int &vnodes, int &maxsize,
01514 const NDIndex<Dim> &readDomain) {
01515
01516 DFDBG(Inform dbgmsg("DiscField::distribute_offsets", INFORM_ALL_NODES));
01517
01518
01519 vnodes = 0;
01520 maxsize = 0;
01521
01522
01523 if (!Ippl::perSMPParallelIO()) {
01524 DFDBG(dbgmsg << "Per-SMP parallel IO is disabled, so only box0 nodes ");
01525 DFDBG(dbgmsg << "will read data." << endl);
01526
01527 if (Ippl::myNode() == myBox0())
01528 vnodes = offdata.size();
01529
01530 } else {
01531
01532
01533 int tag = Ippl::Comm->next_tag(DF_OFFSET_TAG, DF_TAG_CYCLE);
01534
01535
01536
01537 if (Config->getNodeSMPIndex(myBox0()) != mySMP()) {
01538 DFDBG(dbgmsg << "Node " << Ippl::myNode() << " has box0 = ");
01539 DFDBG(dbgmsg << myBox0() << " on a different SMP. Return from ");
01540 DFDBG(dbgmsg << "distribute_offsets with zero vnodes." << endl);
01541 return;
01542 }
01543
01544
01545
01546 if (Ippl::myNode() == myBox0()) {
01547
01548 int pernode = offdata.size() / pNodesPerSMP(myBox0());
01549
01550
01551 int extra = offdata.size() % pNodesPerSMP(myBox0());
01552
01553 DFDBG(dbgmsg << "Assigning " << pernode << " vnodes to each node, ");
01554 DFDBG(dbgmsg << "with " << extra << " extra." << endl);
01555
01556
01557
01558 int nextvnode = pernode;
01559 if (extra > 0) {
01560 nextvnode += 1;
01561 extra -= 1;
01562 }
01563 DFDBG(dbgmsg << "This box0 node will get the first " << nextvnode);
01564 DFDBG(dbgmsg << " vnodes." << endl);
01565
01566
01567 vnodes = nextvnode;
01568
01569
01570
01571 for (int n=0; n < Config->getNumSMPNodes(); ++n) {
01572 int node = Config->getSMPNode(mySMP(), n);
01573 if (node != Ippl::myNode()) {
01574
01575 int numvnodes = pernode;
01576 if (extra > 0) {
01577 numvnodes += 1;
01578 extra -= 1;
01579 }
01580
01581
01582
01583
01584 Message *msg = new Message;
01585 msg->put(numvnodes);
01586 if (numvnodes > 0) {
01587 msg->setCopy(false);
01588 msg->setDelete(false);
01589 msg->putmsg(static_cast<void *>(&(offdata[nextvnode])),
01590 sizeof(DFOffsetData<Dim,T>),
01591 numvnodes);
01592 }
01593
01594
01595 DFDBG(dbgmsg << "Sending offset info for " << numvnodes);
01596 DFDBG(dbgmsg << " vnodes to node " << node << " with tag " << tag);
01597 DFDBG(dbgmsg << ", starting from box0 " << nextvnode << endl);
01598 Ippl::Comm->send(msg, node, tag);
01599
01600
01601 nextvnode += numvnodes;
01602 }
01603 }
01604
01605
01606 if (nextvnode != offdata.size())
01607 Ippl::abort("ERROR: Could not give away all my vnodes!");
01608
01609 } else {
01610
01611 int node = myBox0();
01612 DFDBG(dbgmsg << "Waiting for offset data from node " << node << endl);
01613 Message *msg = Ippl::Comm->receive_block(node, tag);
01614
01615
01616 msg->get(vnodes);
01617 DFDBG(dbgmsg << "Received offset info for " << vnodes << " vnodes ");
01618 DFDBG(dbgmsg << "from node " << node << endl);
01619
01620
01621 if (vnodes > 0) {
01622
01623 offdata.resize(vnodes);
01624
01625
01626 ::getMessage_iter(*msg, &(offdata[0]));
01627 }
01628
01629
01630 delete msg;
01631 }
01632 }
01633
01634
01635
01636 DFDBG(dbgmsg << "Scanning offset data for maxsize value ..." << endl);
01637 for (int v=0; v < vnodes; ++v) {
01638
01639 NDIndex<Dim> dom;
01640 offset_data_to_domain(offdata[v], dom);
01641 if (dom.touches(readDomain)) {
01642
01643 int msdim = (Dim-1);
01644 int chunkelems = Ippl::chunkSize() / sizeof(T);
01645 NDIndex<Dim> chunkblock = chunk_domain(dom, chunkelems, msdim,
01646 offdata[v].isCompressed);
01647
01648 DFDBG(dbgmsg << "Checking size of vnode " << v << " on node ");
01649 DFDBG(dbgmsg << Ippl::myNode() << " with domain " << dom);
01650 DFDBG(dbgmsg << ", compressed = " << offdata[v].isCompressed);
01651 DFDBG(dbgmsg << ", chunkblock = " << chunkblock << endl);
01652
01653
01654 int dsize = chunkblock.size();
01655 if (dsize > maxsize) {
01656 maxsize = dsize;
01657 DFDBG(dbgmsg << " New maxsize = " << maxsize << endl);
01658 }
01659 } else {
01660 DFDBG(dbgmsg << "Skipping vnode " << v << " since it does not ");
01661 DFDBG(dbgmsg << "intersect with readDomain = " << readDomain);
01662 DFDBG(dbgmsg << "; keeping maxsize = " << maxsize << endl);
01663 }
01664 }
01665 }
01666
01668
01669
01670
01671 template <class T>
01672 bool read_data(int outputDatafd, T* buffer, Offset_t readsize,
01673 Offset_t seekpos) {
01674 TAU_TYPE_STRING(taustr, CT(*this) + " bool (FILE*, " + CT(buffer)
01675 + ", Offset_t, Offset_t )" );
01676 TAU_PROFILE("DiscField::read_data()", taustr,
01677 TAU_UTILITY | TAU_FIELD | TAU_IO);
01678
01679 DFDBG(Inform dbgmsg("DiscField::read_data", INFORM_ALL_NODES));
01680 DFDBG(dbgmsg << "readsize=" << readsize << ", seekpos=" << seekpos);
01681 DFDBG(dbgmsg <<", sizeof(T)=" << sizeof(T) << endl);
01682
01683 PAssert(seekpos >= 0);
01684 PAssert(readsize > 0);
01685 PAssert(buffer != 0);
01686 PAssert(outputDatafd >= 0);
01687 PAssert(readsize % sizeof(T) == 0);
01688
01689 #ifdef IPPL_DIRECTIO
01690 if (openedDirectIO) {
01691 PAssert(readsize % dioinfo.d_miniosz == 0);
01692 PAssert(seekpos % dioinfo.d_miniosz == 0);
01693 }
01694 #endif
01695
01696
01697 off_t seekoffset = seekpos;
01698 size_t nbytes = readsize;
01699 bool seekok = true;
01700
01701 Timer rtimer;
01702 rtimer.clear();
01703 rtimer.start();
01704
01705 #ifdef IPPL_DIRECTIO
01706 long nout = ::pread(outputDatafd, buffer, nbytes, seekoffset);
01707 #else
01708 long nout = 0;
01709 if (::lseek(outputDatafd, seekoffset, SEEK_SET) == seekoffset) {
01710 char *rbuf = (char *)buffer;
01711 nout = ::read(outputDatafd, rbuf, nbytes);
01712 } else {
01713 seekok = false;
01714 }
01715 #endif
01716
01717 rtimer.stop();
01718 DiscBuffer::readtime += rtimer.clock_time();
01719 DiscBuffer::readbytes += readsize;
01720
01721 if (!seekok) {
01722 ERRORMSG("Seek error in DiscField::read_data" << endl);
01723 ERRORMSG("Could not seek to position " << seekoffset << endl);
01724 Ippl::abort("Exiting due to DiscField error.");
01725 }
01726
01727 if (nout != nbytes) {
01728 ERRORMSG("Read error in DiscField::read_data" << endl);
01729 ERRORMSG("Could not read " << nbytes << " bytes." << endl);
01730 Ippl::abort("Exiting due to DiscField error.");
01731 }
01732
01733 DFDBG(size_t nelem = readsize / sizeof(T));
01734 DFDBG(dbgmsg << "Read in block of " << nelem << " elements in ");
01735 DFDBG(dbgmsg << rtimer.clock_time() << " second:" << endl);
01736 DFDBG(for (unsigned int i=0; i < nelem && i < 10; ++i))
01737 DFDBG( dbgmsg << " buffer[" << i << "] = " << buffer[i] << endl);
01738
01739 ADDIPPLSTAT(incDiscBytesRead, readsize);
01740 return true;
01741 }
01742
01744
01745
01746 template<class T>
01747 void offset_data_to_domain(DFOffsetData<Dim,T> &offdata,
01748 NDIndex<Dim> &domain) {
01749 int *dptr = offdata.vnodedata + 1;
01750 for (int i=0; i < Dim; ++i) {
01751 int first = *dptr;
01752 int stride = *(dptr + 1);
01753 int length = *(dptr + 2);
01754 domain[i] = Index(first, first + (length - 1)*stride, stride);
01755 dptr += 6;
01756 }
01757 }
01758
01760
01761 template<class T>
01762 void domain_to_offset_data(const NDIndex<Dim> &domain,
01763 DFOffsetData<Dim,T> &offdata) {
01764 int *dptr = offdata.vnodedata;
01765 for (int i=0; i < Dim; ++i) {
01766 *dptr++ = 0;
01767 *dptr++ = domain[i].first();
01768 *dptr++ = domain[i].stride();
01769 *dptr++ = domain[i].length();
01770 *dptr++ = 0;
01771 *dptr++ = 0;
01772 }
01773 }
01774
01775
01776
01777
01778
01779
01780 DiscField(const DiscField<Dim>&);
01781 DiscField& operator=(const DiscField<Dim>&);
01782 };
01783
01784 #include "Utility/DiscField.cpp"
01785
01786 #endif // DISC_FIELD_H
01787
01788
01789
01790
01791
01792