Commit f66a4a0f authored by Hal Finkel's avatar Hal Finkel
Browse files

Add an option to have rank 0 create all files and subfiles

parent 5ff5945f
......@@ -53,6 +53,7 @@ extern "C" {
#include <stdexcept>
#include <iterator>
#include <algorithm>
#include <set>
#include <cassert>
#include <cstddef>
#include <cstring>
......@@ -394,6 +395,17 @@ static int GetSZDT(GenericIO::Variable &Var) {
return -1;
}
void GenericIO::setFH(int R) {
#ifndef GENERICIO_NO_MPI
if (FileIOType == FileIOMPI)
FH.get() = new GenericFileIO_MPI(R);
else if (FileIOType == FileIOMPICollective)
FH.get() = new GenericFileIO_MPICollective(R);
else
#endif
FH.get() = new GenericFileIO_POSIX();
}
#ifndef GENERICIO_NO_MPI
void GenericIO::write() {
if (isBigEndian())
......@@ -423,6 +435,13 @@ void GenericIO::write() {
MPI_Comm_rank(SplitComm, &SplitRank);
MPI_Comm_size(SplitComm, &SplitNRanks);
bool Rank0CreateAll = false;
const char *EnvStr = getenv("GENERICIO_RANK0_CREATE_ALL");
if (EnvStr) {
int Mod = atoi(EnvStr);
Rank0CreateAll = (Mod > 0);
}
string LocalFileName;
if (SplitNRanks != NRanks) {
if (Rank == 0) {
......@@ -466,6 +485,24 @@ void GenericIO::write() {
}
GIO.write();
// On some file systems, it can be very expensive for multiple ranks to
// create files in the same directory. Creating a new file requires a
// kind of exclusive lock that is expensive to obtain.
if (Rank0CreateAll) {
set<int> AllPartitions;
for (int i = 0; i < NRanks; ++i) AllPartitions.insert(MapPartition[i]);
for (set<int>::iterator i = AllPartitions.begin(),
e = AllPartitions.end(); i != e; ++i) {
stringstream ss;
ss << FileName << "#" << *i;
setFH(MPI_COMM_SELF);
FH.get()->open(ss.str());
close();
}
}
} else {
MPI_Gather(&Partition, 1, MPI_INT, 0, 0, MPI_INT, 0, Comm);
}
......@@ -498,7 +535,7 @@ void GenericIO::write() {
RHLocal.GlobalRank = Rank;
bool ShouldCompress = DefaultShouldCompress;
const char *EnvStr = getenv("GENERICIO_COMPRESS");
EnvStr = getenv("GENERICIO_COMPRESS");
if (EnvStr) {
int Mod = atoi(EnvStr);
ShouldCompress = (Mod > 0);
......@@ -754,14 +791,9 @@ nocomp:
uint64_t HeaderCRC = crc64_omp(&Header[0], HeaderSize - CRCSize);
crc64_invert(HeaderCRC, &Header[HeaderSize - CRCSize]);
if (FileIOType == FileIOMPI)
FH.get() = new GenericFileIO_MPI(MPI_COMM_SELF);
else if (FileIOType == FileIOMPICollective)
FH.get() = new GenericFileIO_MPICollective(MPI_COMM_SELF);
else
FH.get() = new GenericFileIO_POSIX();
setFH(MPI_COMM_SELF);
FH.get()->open(LocalFileName);
FH.get()->open(LocalFileName, false, Rank0CreateAll);
FH.get()->setSize(FileSize);
FH.get()->write(&Header[0], HeaderSize, 0, "header");
......@@ -779,12 +811,7 @@ nocomp:
MPI_Barrier(SplitComm);
if (FileIOType == FileIOMPI)
FH.get() = new GenericFileIO_MPI(SplitComm);
else if (FileIOType == FileIOMPICollective)
FH.get() = new GenericFileIO_MPICollective(SplitComm);
else
FH.get() = new GenericFileIO_POSIX();
setFH(SplitComm);
FH.get()->open(LocalFileName, false, true);
......@@ -1021,14 +1048,13 @@ void GenericIO::openAndReadHeader(MismatchBehavior MB, int EffRank, bool CheckPa
vector<char> Header;
if (SplitRank == 0) {
setFH(
#ifndef GENERICIO_NO_MPI
if (FileIOType == FileIOMPI)
FH.get() = new GenericFileIO_MPI(MPI_COMM_SELF);
else if (FileIOType == FileIOMPICollective)
FH.get() = new GenericFileIO_MPICollective(MPI_COMM_SELF);
else
MPI_COMM_SELF
#else
0
#endif
FH.get() = new GenericFileIO_POSIX();
);
#ifndef GENERICIO_NO_MPI
char True = 1, False = 0;
......@@ -1092,12 +1118,7 @@ void GenericIO::openAndReadHeader(MismatchBehavior MB, int EffRank, bool CheckPa
if (!DisableCollErrChecking)
MPI_Barrier(Comm);
if (FileIOType == FileIOMPI)
FH.get() = new GenericFileIO_MPI(SplitComm);
else if (FileIOType == FileIOMPICollective)
FH.get() = new GenericFileIO_MPICollective(SplitComm);
else
FH.get() = new GenericFileIO_POSIX();
setFH(SplitComm);
int OpenErr = 0, TotOpenErr;
try {
......
......@@ -637,6 +637,8 @@ public:
FHWCnt *CountedFH;
} FH;
void setFH(int R);
};
} /* END namespace cosmotk */
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment