52 template <
unsigned Dim>
54 unsigned int numFields,
const char* typestr) {
56 initialize(base, config, typestr, numFields);
64 template <
unsigned Dim>
66 const char* typestr) {
68 initialize(base, 0, typestr, numFields);
76 template <
unsigned Dim>
79 initialize(base, config, 0, 0);
87 template <
unsigned Dim>
90 initialize(base, 0, 0, 0);
96 template <
unsigned Dim>
98 const char *typestr,
unsigned int numFields) {
104 TypeString = typestr;
106 TypeString =
"unknown";
117 openedDirectIO =
false;
122 NumFields = numFields;
123 WritingFile = (NumFields > 0);
127 NeedStartRecord = -1;
130 for (
unsigned int i=0; i < NumFields; ++i)
131 ValidField.push_back(
false);
134 ConfigOK = parse_config(config, WritingFile);
139 if (ConfigOK && !WritingFile)
140 ConfigOK = read_meta();
146 template <
unsigned Dim>
153 delete [] VnodeTally;
164 template <
unsigned Dim>
166 std::vector<int>& size)
const {
168 numRecords = NumRecords;
169 numFields = NumFields;
172 for (
int i=1; i < numFiles(); ++i) {
173 for (
int j=0; j < NumVnodes[i].size(); ++j)
174 size[j] += NumVnodes[i][j];
183 template <
unsigned Dim>
186 FILE *f = fopen(fnm.c_str(), mode.c_str());
188 ERRORMSG(
"DiscField: Could not open file '" << fnm.c_str());
189 ERRORMSG(
"' for mode '" << mode.c_str() <<
"' on node ");
200 template <
unsigned Dim>
205 std::string fnamebuf(
"");
206 if (fnm.length() > 0)
208 if (suf.length() > 0)
212 int flags = origflags;
214 openedDirectIO =
false;
217 openedDirectIO =
true;
222 int f = ::open(fnamebuf.c_str(), flags, 0644);
226 f = ::open(fnamebuf.c_str(), origflags, 0644);
227 openedDirectIO = (f >= 0);
232 ERRORMSG(
"DiscField: Could not open file '" << fnamebuf.c_str());
241 if (openedDirectIO) {
242 if (::fcntl(f, F_DIOINFO, &dioinfo) != 0) {
243 ERRORMSG(
"DiscField: Could not get dio info for '"<< fnamebuf.c_str());
244 ERRORMSG(
"' using direct io on node ");
250 DFDBG(std::string dbgmsgname(
"DF:open_df_file_fd"));
252 DFDBG(dbgmsg <<
"Opened file '" << fnamebuf.c_str() <<
"' with direct-io");
253 DFDBG(dbgmsg <<
", dioinfo = (miniosz="<<dioinfo.d_miniosz<<
", maxiosz=");
254 DFDBG(dbgmsg << dioinfo.d_maxiosz <<
", mem=" << dioinfo.d_mem <<
")");
265 template <
unsigned Dim>
267 const std::string& mode) {
269 std::string fnamebuf(
"");
270 if (fnm.length() > 0)
272 if (suf.length() > 0)
285 FILE *f = fopen(fnamebuf.c_str(), mode.c_str());
288 ERRORMSG(
"DiscField: Could not open file '" << fnamebuf);
289 ERRORMSG(
"' for mode '" << mode.c_str() <<
"' on node ");
299 template <
unsigned Dim>
311 unsigned int nfiles = 3;
314 for (
unsigned int i=0; i < nfiles; ++i) {
315 std::string fname(Config->getFilename(0) + suff[i]);
316 if ((f = open_df_file(fname, om)) == 0) {
317 ERRORMSG(
"DiscField: Could not create file '" << fname.c_str());
325 int fd = open_df_file_fd(Config->getFilename(0), suff[3],
326 O_RDWR|O_CREAT|O_TRUNC);
328 std::string fname(Config->getFilename(0) + suff[3]);
329 ERRORMSG(
"DiscField: Could not create data file '"<<fname.c_str());
345 template <
unsigned Dim>
349 if (Size.size() != 0 && !(Size == layout.
getDomain()))
355 globalID.erase(globalID.begin(), globalID.end());
361 for (local = layout.
begin_iv() ; local != layout.
end_iv(); ++local) {
364 int node = (*local).second.get()->getNode();
365 unsigned int nodesmp = Config->getNodeSMPIndex(node);
368 bool foundsmp = (nodesmp == mySMP());
369 unsigned int checksmp = 0;
370 while (!foundsmp && checksmp < Config->getNumOtherSMP()) {
371 foundsmp = (nodesmp == Config->getOtherSMP(checksmp));
379 globalID.insert(vtype(domain, node));
389 int node = (*remote).second->getNode();
390 unsigned int nodesmp = Config->getNodeSMPIndex(node);
393 bool foundsmp = (nodesmp == mySMP());
394 unsigned int checksmp = 0;
395 while (!foundsmp && checksmp < Config->getNumOtherSMP()) {
396 foundsmp = (nodesmp == Config->getOtherSMP(checksmp));
404 globalID.insert(vtype(domain, node));
415 template <
unsigned Dim>
420 Config =
new DiscConfig(fname, BaseFile.c_str(), writing);
425 if (numFiles() > 0 && myBox0() == (
unsigned int)
Ippl::myNode()) {
426 NumVnodes =
new std::vector<int>[numFiles()];
427 VnodeTally =
new std::vector<int>[numFiles()];
445 ERRORMSG(
"DiscField: A problem occurred reading the config file '");
456 template <
unsigned Dim>
459 template <
unsigned Dim>
464 msg <<
"BaseFile = " << BaseFile <<
endl;
465 msg <<
"Field Type = " << TypeString <<
endl;
466 msg <<
"NumRecords = " << NumRecords <<
endl;
467 msg <<
"NumFields = " << NumFields <<
endl;
469 msg <<
"Configuration file information:" <<
endl;
470 Config->printDebug(msg);
489 template <
unsigned Dim>
499 FILE *outputMeta = open_df_file(Config->getFilename(0),
".meta",std::string(
"w"));
504 fprintf(outputMeta,
"Type = %s\n", TypeString.c_str());
505 fprintf(outputMeta,
"Dim = %u\n",
Dim);
506 for (d=0; d <
Dim; ++d)
507 fprintf(outputMeta,
"Domain = %d %d %d\n",
508 Size[d].first(), Size[d].last(), Size[d].stride());
509 fprintf(outputMeta,
"Fields = %u\n", NumFields);
510 fprintf(outputMeta,
"Records = %u\n", NumRecords);
511 fprintf(outputMeta,
"SMPs = %u\n", fileSMPs());
515 fprintf(outputMeta,
"VnodesInRecord = ");
516 for (r=0; r < NumRecords; ++r)
517 fprintf(outputMeta,
" %d", NumVnodes[0][r]);
518 fprintf(outputMeta,
"\n");
520 fprintf(outputMeta,
"VnodeTally= ");
521 for (r=0; r < NumRecords; ++r)
522 fprintf(outputMeta,
" %d", VnodeTally[0][r]);
523 fprintf(outputMeta,
"\n");
535 template <
unsigned Dim>
538 bool iserror =
false;
541 DFDBG(std::string dbgmsgname(
"DF:read_meta:"));
542 DFDBG(dbgmsgname += Config->getConfigFile());
544 DFDBG(dbgmsg <<
"Starting to read meta info: mySMP=" << mySMP());
545 DFDBG(dbgmsg <<
", mybox0=" << myBox0() <<
", numfiles=" << numFiles());
552 for (
unsigned int sf=0; sf < numFiles(); ++sf) {
554 std::string filename = Config->getFilename(sf) +
".meta";
555 DiscMeta outputMeta(filename.c_str());
556 if (outputMeta.size() == 0) {
557 ERRORMSG(
"DiscField: The meta file '" << filename <<
"' is empty ");
564 unsigned int dimread = 0;
565 TypeString =
"unknown";
569 NumVnodes[sf].erase(NumVnodes[sf].begin(), NumVnodes[sf].end());
570 VnodeTally[sf].erase(VnodeTally[sf].begin(), VnodeTally[sf].end());
574 for (metaline = outputMeta.begin(); metaline != metaend; ++metaline) {
576 int linesread = (*metaline).first;
577 int numtokens = (*metaline).second.first;
578 std::string *tokens = (*metaline).second.second;
581 if (tokens[0] ==
"Type") {
583 TypeString = tokens[1];
585 else if (tokens[0] ==
"Dim" && numtokens == 2) {
586 DataDimension = atoi(tokens[1].c_str());
587 if (DataDimension < 1) {
588 ERRORMSG(
"DiscField: The meta file '" << filename <<
"' ");
589 ERRORMSG(
"contains a value for dimension < 1, '");
595 else if (tokens[0] ==
"Fields" && numtokens == 2) {
596 NumFields = atoi(tokens[1].c_str());
598 ERRORMSG(
"DiscField: The meta file '" << filename <<
"' ");
599 ERRORMSG(
"contains a value for Fields < 1, '");
605 else if (tokens[0] ==
"Records" && numtokens == 2) {
606 NumRecords = atoi(tokens[1].c_str());
608 else if (tokens[0] ==
"SMPs" && numtokens == 2) {
609 unsigned int checkfileSMPs = atoi(tokens[1].c_str());
610 if (fileSMPs() != checkfileSMPs) {
611 ERRORMSG(
"DiscField: The meta file '" << filename <<
"' ");
612 ERRORMSG(
"contains a value for the number of filesets that\n");
613 ERRORMSG(
"does not match the number of filesets in the config\n");
614 ERRORMSG(
"file: metafile filesets = " << tokens[1] <<
", ");
615 ERRORMSG(
"config filesets = " << fileSMPs() <<
"." <<
endl);
620 else if (tokens[0] ==
"Domain" && numtokens == 4) {
622 Size[dimread] =
Index(atoi(tokens[1].c_str()),
623 atoi(tokens[2].c_str()),
624 atoi(tokens[3].c_str()));
628 else if (tokens[0] ==
"VnodesInRecord") {
629 for (
int r=1; r < numtokens; ++r)
630 NumVnodes[sf].push_back(atoi(tokens[r].c_str()));
632 else if (tokens[0] ==
"VnodeTally") {
633 for (
int r=1; r < numtokens; ++r)
634 VnodeTally[sf].push_back(atoi(tokens[r].c_str()));
638 ERRORMSG(
"DiscField: Format error on line " << linesread);
639 ERRORMSG(
" in meta file '" << filename <<
"'." <<
endl);
644 DFDBG(dbgmsg <<
"On box0: finished line with tokens[0]='"<<tokens[0]);
645 DFDBG(dbgmsg <<
"' ... iserror = " << iserror <<
endl);
652 if (DataDimension != dimread) {
653 ERRORMSG(
"DiscField: Dim != # Domain lines in meta file '");
654 ERRORMSG(filename <<
"'. (" << DataDimension <<
" != " << dimread);
659 if (NumRecords != NumVnodes[sf].size()) {
660 ERRORMSG(
"DiscField: Records != VnodesInRecord items in meta file '");
661 ERRORMSG(filename <<
"'. (" << NumRecords <<
" != ");
666 if (NumRecords != VnodeTally[sf].size()) {
667 ERRORMSG(
"DiscField: Records != VnodeTally items in meta file '");
668 ERRORMSG(filename <<
"'. (" << NumRecords <<
" != ");
679 DFDBG(dbgmsg <<
"Summary of meta info:" <<
endl);
680 DFDBG(dbgmsg <<
" iserror = " << iserror <<
endl);
681 DFDBG(dbgmsg <<
" DataDimension = " << DataDimension <<
endl);
682 DFDBG(dbgmsg <<
" NumFields = " << NumFields <<
endl);
683 DFDBG(dbgmsg <<
" NumRecords = " << NumRecords <<
endl);
684 DFDBG(dbgmsg <<
" Size = " << Size <<
endl);
685 DFDBG(dbgmsg <<
" TypeString = " << TypeString <<
endl);
688 int numinform = Config->getNumOtherSMP();
689 DFDBG(dbgmsg <<
"Preparing messages to send to other nodes ..." <<
endl);
690 DFDBG(dbgmsg <<
" Must send data to my SMP nodes, and to ");
691 DFDBG(dbgmsg << numinform <<
" other SMP's." <<
endl);
692 for (
int s=0; s <= numinform; ++s) {
695 smp = Config->getOtherSMP(s);
696 DFDBG(dbgmsg <<
" Preparing messages for SMP " << smp <<
" with ");
697 DFDBG(dbgmsg << Config->getNumSMPNodes(smp) <<
" nodes." <<
endl);
698 for (
unsigned int n=0;
n < Config->getNumSMPNodes(smp); ++
n) {
699 int node = Config->getSMPNode(smp,
n);
703 int errint = iserror;
705 msg->
put(DataDimension);
707 msg->
put(NumRecords);
712 DFDBG(dbgmsg <<
" Sending meta info to node " << node);
713 DFDBG(dbgmsg <<
" with tag " << tag <<
", on SMP " << smp <<
endl);
716 DFDBG(dbgmsg <<
" Skipping send to node " << node);
717 DFDBG(dbgmsg <<
" since it is my node." <<
endl);
726 DFDBG(dbgmsg <<
"Waiting for meta info from node " << node);
727 DFDBG(dbgmsg <<
" with tag " << tag <<
endl);
732 DFDBG(dbgmsg <<
"Summary of received meta info:" <<
endl);
735 iserror = (errint != 0);
736 DFDBG(dbgmsg <<
" iserror = " << iserror <<
endl);
737 msg->
get(DataDimension);
738 DFDBG(dbgmsg <<
" DataDimension = " << DataDimension <<
endl);
740 DFDBG(dbgmsg <<
" NumFields = " << NumFields <<
endl);
741 msg->
get(NumRecords);
742 DFDBG(dbgmsg <<
" NumRecords = " << NumRecords <<
endl);
744 DFDBG(dbgmsg <<
" Size = " << Size <<
endl);
746 DFDBG(dbgmsg <<
" TypeString = " << TypeString <<
endl);
758 template <
unsigned Dim>
769 if (fread(ndidata,
sizeof(
int), 6*
Dim, f) != 6*
Dim) {
770 ERRORMSG(
"DiscField: Error reading NDIndex line from data file." <<
endl);
776 int *dptr = ndidata + 1;
777 for (
unsigned int d=0; d <
Dim; ++d) {
779 int stride = *(dptr + 1);
780 int length = *(dptr + 2);
781 ndi[d] =
Index(first, first + (length - 1)*stride, stride);
791 template <
unsigned Dim>
817 for (
unsigned int d=0; d <
Dim; ++d) {
819 *dptr++ = ndi[d].first();
820 *dptr++ = ndi[d].stride();
821 *dptr++ = ndi[d].length();
827 return (fwrite(ndidata,
sizeof(
int), 6*Dim, f) == 6*Dim);
839 template <
unsigned Dim>
847 FILE *outputLayout = open_df_file(Config->getFilename(0),
".layout",
851 int numvnodes = globalID.size();
852 if (fwrite(&numvnodes,
sizeof(
int), 1, outputLayout) != 1) {
853 ERRORMSG(
"Error writing .layout file in DiscField::write_layout." <<
endl);
855 fclose(outputLayout);
861 for (
id = globalID.begin();
id != idend; ++id) {
862 if (!write_NDIndex(outputLayout, (*id).first)) {
863 ERRORMSG(
"Error writing record " << NumRecords-1 <<
" to .layout file.");
865 fclose(outputLayout);
871 fclose(outputLayout);
878 template <
unsigned Dim>
881 DFDBG(std::string dbgmsgname(
"DF:read_layout:"));
882 DFDBG(dbgmsgname += Config->getConfigFile());
884 DFDBG(dbgmsg <<
"Starting to read layout info for record " << record);
885 DFDBG(dbgmsg <<
", file " << sf <<
": mySMP=" << mySMP() <<
", mybox0=");
886 DFDBG(dbgmsg << myBox0() <<
", numfiles=" << numFiles() <<
endl);
889 std::string filename = Config->getFilename(sf) +
".layout";
890 FILE *outputLayout = open_df_file(Config->getFilename(sf),
891 ".layout", std::string(
"r"));
892 if (outputLayout == 0)
895 DFDBG(dbgmsg <<
"On box0: sf=" << sf <<
", opened file '");
896 DFDBG(dbgmsg << filename <<
"' ..." <<
endl);
899 Offset_t seekpos = record*
sizeof(int) +
900 VnodeTally[sf][record]*6*
Dim*
sizeof(
int);
901 DFDBG(dbgmsg <<
" Seeking to position ");
902 DFDBG(dbgmsg << static_cast<long>(seekpos) <<
endl);
904 if (fseek(outputLayout, seekpos, SEEK_SET) != 0) {
905 ERRORMSG(
"Error seeking to position " << static_cast<long>(seekpos));
906 ERRORMSG(
" in file '" << filename <<
"' for DiscField::read_layout.");
909 fclose(outputLayout);
915 if (fread(&numvnodes,
sizeof(
int), 1, outputLayout) != 1) {
916 ERRORMSG(
"Error reading file '" << filename);
917 ERRORMSG(
"' in DiscField::read_layout.");
920 fclose(outputLayout);
929 fclose(outputLayout);
940 template <
unsigned Dim>
945 DFDBG(std::string dbgmsgname(
"DF:compute_expected"));
951 for (; local != f.
end_iv(); ++local) {
955 if (domain.
touches(readDomain)) {
957 DFDBG(dbgmsg <<
"Intersection of " << domain <<
" and " << readDomain);
958 DFDBG(dbgmsg <<
" = " << newdomain <<
endl);
960 expected += newdomain.
size();
962 DFDBG(dbgmsg <<
"No intersection of " << domain <<
" and ");
966 DFDBG(dbgmsg <<
" Size for local domain " << domain <<
" = ");
967 DFDBG(dbgmsg << domain.
size() <<
", readDomain = " << readDomain);
979 template <
unsigned Dim>
988 for (
unsigned int i=0; i <
Dim; ++i)
989 sliceblock[i] =
Index(currblock[i].length());
992 int currsize = currblock.
size();
993 if (chunkelems < 1 || iscompressed)
994 chunkelems = currsize;
998 for (
int d=(Dim-1); d >= 0; --d) {
1000 int axislen = currblock[d].length();
1001 currsize /= axislen;
1004 int numslices = chunkelems / currsize;
1005 if (numslices > axislen)
1006 numslices = axislen;
1009 if (numslices > 0) {
1010 sliceblock[d] =
Index(numslices);
1015 sliceblock[d] =
Index(1);
static void abort(const char *=0, int exitcode=(-1))
bool read_NDIndex(FILE *, NDIndex< Dim > &)
bool make_globalID(FieldLayout< Dim > &)
void initialize(const char *base, const char *config, const char *typestr, unsigned int numFields)
int next_tag(int t, int s=1000)
bool write_NDIndex(FILE *, const NDIndex< Dim > &)
bool parse_config(const char *, bool)
int open_df_file_fd(const std::string &fnm, const std::string &suf, int flags)
iterator_dv end_rdv(const GuardCellSizes< Dim > &gc=gc0())
void query(int &numRecords, int &numFields, std::vector< int > &size) const
ac_id_vnodes::iterator iterator_iv
bool touches(const NDIndex< Dim > &) const
std::pair< NDIndex< Dim >, int > value_type
int compute_expected(const FieldLayout< Dim > &, const NDIndex< Dim > &)
Message & get(const T &cval)
Message & put(const T &val)
FILE * open_df_file(const std::string &fnm, const std::string &mode)
container_t::iterator iterator
void getMessage(Message &m, T &t)
int read_layout(int record, int sf)
NDIndex< Dim > intersect(const NDIndex< Dim > &) const
DiscField(const char *fname, const char *config, unsigned int numFields, const char *typestr=0)
iterator_dv begin_rdv(const GuardCellSizes< Dim > &gc=gc0())
void putMessage(Message &m, const T &t)
NDIndex< Dim > chunk_domain(const NDIndex< Dim > &currblock, int chunkelems, int &msdim, bool iscompressed)
rep_type::iterator iterator
Message * receive_block(int &node, int &tag)
static Communicate * Comm
ac_id_vnodes::const_iterator const_iterator_iv
bool send(Message *, int node, int tag, bool delmsg=true)
const NDIndex< Dim > & getDomain() const
Inform & endl(Inform &inf)