Commit 00587dc5 authored by Hal Finkel's avatar Hal Finkel
Browse files

Initial Commit (gio-base-20150317)

parents
This diff is collapsed.
CC = gcc
CXX = g++
MPICC = mpicc
MPICXX = mpicxx
all: fe-progs mpi-progs
sql: fe-sqlite
FEDIR = frontend
FE_CFLAGS := -g -fPIC -O3 -fopenmp
FE_CPPFLAGS := -Ithirdparty/blosc -Ithirdparty/sqlite -DGENERICIO_NO_MPI
MPIDIR = mpi
MPI_CFLAGS := -g -O3
MPI_CPPFLAGS := -Ithirdparty/blosc
$(FEDIR):
mkdir -p $(FEDIR)
$(FEDIR)/%.o: thirdparty/blosc/%.c | $(FEDIR)
$(CC) $(FE_CFLAGS) $(FE_CPPFLAGS) -c -o $@ $<
$(FEDIR)/%.o: %.cxx | $(FEDIR)
$(CXX) $(FE_CFLAGS) $(FE_CPPFLAGS) -c -o $@ $<
$(FEDIR)/GenericIOPrint: $(FEDIR)/GenericIOPrint.o $(FEDIR)/GenericIO.o $(FEDIR)/blosc.o $(FEDIR)/blosclz.o $(FEDIR)/shuffle.o
$(CXX) $(FE_CFLAGS) -o $@ $^
$(FEDIR)/GenericIOVerify: $(FEDIR)/GenericIOVerify.o $(FEDIR)/GenericIO.o $(FEDIR)/blosc.o $(FEDIR)/blosclz.o $(FEDIR)/shuffle.o
$(CXX) $(FE_CFLAGS) -o $@ $^
FE_UNAME := $(shell uname -s)
ifeq ($(FE_UNAME),Darwin)
FE_SHARED := -bundle
else
FE_SHARED := -shared
endif
$(FEDIR)/GenericIOSQLite.so: $(FEDIR)/GenericIOSQLite.o $(FEDIR)/GenericIO.o
$(CXX) $(FE_CFLAGS) $(FE_SHARED) -o $@ $^
SQLITE_CPPFLAGS := -DSQLITE_ENABLE_COLUMN_METADATA=1 -DSQLITE_DISABLE_DIRSYNC=1 -DSQLITE_ENABLE_FTS3=3 -DSQLITE_ENABLE_RTREE=1 -DSQLITE_ENABLE_UNLOCK_NOTIFY=1 -DSQLITE_ENABLE_LOAD_EXTENSION=1 -DHAVE_READLINE=1
$(FEDIR)/sqbuild:
mkdir -p $(FEDIR)/sqbuild
$(FEDIR)/sqbuild/%.o: thirdparty/sqlite/%.c | $(FEDIR)/sqbuild
$(CC) $(FE_CFLAGS) $(FE_CPPFLAGS) $(SQLITE_CPPFLAGS) -c -o $@ $<
$(FEDIR)/sqlite3: $(FEDIR)/sqbuild/sqlite3.o $(FEDIR)/sqbuild/shell.o
$(CC) $(FE_CFLAGS) -o $@ $^ -pthread -lreadline -lrt -ldl
$(MPIDIR):
mkdir -p $(MPIDIR)
$(MPIDIR)/%.o: thirdparty/blosc/%.c | $(MPIDIR)
$(MPICC) $(MPI_CFLAGS) $(MPI_CPPFLAGS) -c -o $@ $<
$(MPIDIR)/%.o: %.cxx | $(MPIDIR)
$(MPICXX) $(MPI_CFLAGS) $(MPI_CPPFLAGS) -c -o $@ $<
$(MPIDIR)/GenericIOPrint: $(MPIDIR)/GenericIOPrint.o $(MPIDIR)/GenericIO.o $(MPIDIR)/blosc.o $(MPIDIR)/blosclz.o $(MPIDIR)/shuffle.o
$(MPICXX) $(MPI_CFLAGS) -o $@ $^
$(MPIDIR)/GenericIOVerify: $(MPIDIR)/GenericIOVerify.o $(MPIDIR)/GenericIO.o $(MPIDIR)/blosc.o $(MPIDIR)/blosclz.o $(MPIDIR)/shuffle.o
$(MPICXX) $(MPI_CFLAGS) -o $@ $^
frontend-progs: $(FEDIR)/GenericIOPrint $(FEDIR)/GenericIOVerify
fe-progs: frontend-progs
mpi-progs: $(MPIDIR)/GenericIOPrint $(MPIDIR)/GenericIOVerify
frontend-sqlite: $(FEDIR)/GenericIOSQLite.so $(FEDIR)/sqlite3
fe-sqlite: frontend-sqlite
clean:
rm -rf frontend mpi
This diff is collapsed.
#ifndef GENERICIO_H
#define GENERICIO_H
#include <cstdlib>
#include <vector>
#include <string>
#include <iostream>
#include <limits>
#include <stdint.h>
#ifndef GENERICIO_NO_MPI
#include <mpi.h>
#else
#include <fstream>
#endif
#include <unistd.h>
namespace gio {
class GenericFileIO {
public:
virtual ~GenericFileIO() {}
public:
virtual void open(const std::string &FN, bool ForReading = false) = 0;
virtual void setSize(size_t sz) = 0;
virtual void read(void *buf, size_t count, off_t offset,
const std::string &D) = 0;
virtual void write(const void *buf, size_t count, off_t offset,
const std::string &D) = 0;
protected:
std::string FileName;
};
#ifndef GENERICIO_NO_MPI
class GenericFileIO_MPI : public GenericFileIO {
public:
GenericFileIO_MPI(const MPI_Comm &C) : FH(MPI_FILE_NULL), Comm(C) {}
virtual ~GenericFileIO_MPI();
public:
virtual void open(const std::string &FN, bool ForReading = false);
virtual void setSize(size_t sz);
virtual void read(void *buf, size_t count, off_t offset, const std::string &D);
virtual void write(const void *buf, size_t count, off_t offset, const std::string &D);
protected:
MPI_File FH;
MPI_Comm Comm;
};
class GenericFileIO_MPICollective : public GenericFileIO_MPI {
public:
GenericFileIO_MPICollective(const MPI_Comm &C) : GenericFileIO_MPI(C) {}
public:
void read(void *buf, size_t count, off_t offset, const std::string &D);
void write(const void *buf, size_t count, off_t offset, const std::string &D);
};
#endif
class GenericFileIO_POSIX : public GenericFileIO {
public:
GenericFileIO_POSIX() : FH(-1) {}
~GenericFileIO_POSIX();
public:
void open(const std::string &FN, bool ForReading = false);
void setSize(size_t sz);
void read(void *buf, size_t count, off_t offset, const std::string &D);
void write(const void *buf, size_t count, off_t offset, const std::string &D);
protected:
int FH;
};
class GenericIO {
public:
enum VariableFlags {
VarHasExtraSpace = (1 << 0), // Note that this flag indicates that the
// extra space is available, but the GenericIO
// implementation is required to
// preserve its contents.
VarIsPhysCoordX = (1 << 1),
VarIsPhysCoordY = (1 << 2),
VarIsPhysCoordZ = (1 << 3),
VarMaybePhysGhost = (1 << 4)
};
struct VariableInfo {
VariableInfo(const std::string &N, std::size_t S, bool IF, bool IS,
bool PCX, bool PCY, bool PCZ, bool PG)
: Name(N), Size(S), IsFloat(IF), IsSigned(IS),
IsPhysCoordX(PCX), IsPhysCoordY(PCY), IsPhysCoordZ(PCZ),
MaybePhysGhost(PG) {}
std::string Name;
std::size_t Size;
bool IsFloat;
bool IsSigned;
bool IsPhysCoordX, IsPhysCoordY, IsPhysCoordZ;
bool MaybePhysGhost;
};
public:
struct Variable {
template <typename T>
Variable(const std::string &N, T* D, unsigned Flags = 0)
: Name(N), Size(sizeof(T)),
IsFloat(!std::numeric_limits<T>::is_integer),
IsSigned(std::numeric_limits<T>::is_signed),
Data((void *) D), HasExtraSpace(Flags & VarHasExtraSpace),
IsPhysCoordX(Flags & VarIsPhysCoordX),
IsPhysCoordY(Flags & VarIsPhysCoordY),
IsPhysCoordZ(Flags & VarIsPhysCoordZ),
MaybePhysGhost(Flags & VarMaybePhysGhost) {}
Variable(const VariableInfo &VI, void *D, unsigned Flags = 0)
: Name(VI.Name), Size(VI.Size), IsFloat(VI.IsFloat),
IsSigned(VI.IsSigned), Data(D),
HasExtraSpace(Flags & VarHasExtraSpace),
IsPhysCoordX((Flags & VarIsPhysCoordX) || VI.IsPhysCoordX),
IsPhysCoordY((Flags & VarIsPhysCoordY) || VI.IsPhysCoordY),
IsPhysCoordZ((Flags & VarIsPhysCoordZ) || VI.IsPhysCoordZ),
MaybePhysGhost((Flags & VarMaybePhysGhost) || VI.MaybePhysGhost) {}
std::string Name;
std::size_t Size;
bool IsFloat;
bool IsSigned;
void *Data;
bool HasExtraSpace;
bool IsPhysCoordX, IsPhysCoordY, IsPhysCoordZ;
bool MaybePhysGhost;
};
public:
enum FileIO {
FileIOMPI,
FileIOPOSIX,
FileIOMPICollective
};
#ifndef GENERICIO_NO_MPI
GenericIO(const MPI_Comm &C, const std::string &FN, unsigned FIOT = -1)
: NElems(0), FileIOType(FIOT == (unsigned) -1 ? DefaultFileIOType : FIOT),
Partition(DefaultPartition), Comm(C), FileName(FN), SplitComm(MPI_COMM_NULL) {
std::fill(PhysOrigin, PhysOrigin + 3, 0.0);
std::fill(PhysScale, PhysScale + 3, 0.0);
}
#else
GenericIO(const std::string &FN, unsigned FIOT = -1)
: NElems(0), FileIOType(FIOT == (unsigned) -1 ? DefaultFileIOType : FIOT),
Partition(DefaultPartition), FileName(FN) {
std::fill(PhysOrigin, PhysOrigin + 3, 0.0);
std::fill(PhysScale, PhysScale + 3, 0.0);
}
#endif
~GenericIO() {
close();
#ifndef GENERICIO_NO_MPI
if (SplitComm != MPI_COMM_NULL)
MPI_Comm_free(&SplitComm);
#endif
}
public:
std::size_t requestedExtraSpace() const {
return 8;
}
void setNumElems(std::size_t E) {
NElems = E;
#ifndef GENERICIO_NO_MPI
int IsLarge = E >= CollectiveMPIIOThreshold;
int AllIsLarge;
MPI_Allreduce(&IsLarge, &AllIsLarge, 1, MPI_INT, MPI_SUM, Comm);
if (!AllIsLarge)
FileIOType = FileIOMPICollective;
#endif
}
void setPhysOrigin(double O, int Dim = -1) {
if (Dim >= 0)
PhysOrigin[Dim] = O;
else
std::fill(PhysOrigin, PhysOrigin + 3, O);
}
void setPhysScale(double S, int Dim = -1) {
if (Dim >= 0)
PhysScale[Dim] = S;
else
std::fill(PhysScale, PhysScale + 3, S);
}
template <typename T>
void addVariable(const std::string &Name, T *Data,
unsigned Flags = 0) {
Vars.push_back(Variable(Name, Data, Flags));
}
template <typename T, typename A>
void addVariable(const std::string &Name, std::vector<T, A> &Data,
unsigned Flags = 0) {
T *D = Data.empty() ? 0 : &Data[0];
addVariable(Name, D, Flags);
}
void addVariable(const VariableInfo &VI, void *Data,
unsigned Flags = 0) {
Vars.push_back(Variable(VI, Data, Flags));
}
#ifndef GENERICIO_NO_MPI
// Writing
void write();
#endif
// Reading
void openAndReadHeader(bool MustMatch = true, int EffRank = -1,
bool CheckPartMap = true);
int readNRanks();
void readDims(int Dims[3]);
// Note: For partitioned inputs, this returns -1.
uint64_t readTotalNumElems();
void readPhysOrigin(double Origin[3]);
void readPhysScale(double Scale[3]);
void clearVariables() { this->Vars.clear(); };
int getNumberOfVariables() { return this->Vars.size(); };
void getVariableInfo(std::vector<VariableInfo> &VI);
std::size_t readNumElems(int EffRank = -1);
void readCoords(int Coords[3], int EffRank = -1);
int readGlobalRankNumber(int EffRank = -1);
void readData(int EffRank = -1, bool PrintStats = true, bool CollStats = true);
void close() {
FH.close();
}
void setPartition(int P) {
Partition = P;
}
static void setDefaultFileIOType(unsigned FIOT) {
DefaultFileIOType = FIOT;
}
static void setDefaultPartition(int P) {
DefaultPartition = P;
}
static void setNaturalDefaultPartition();
static void setDefaultShouldCompress(bool C) {
DefaultShouldCompress = C;
}
#ifndef GENERICIO_NO_MPI
static void setCollectiveMPIIOThreshold(std::size_t T) {
#ifndef GENERICIO_NO_NEVER_USE_COLLECTIVE_IO
CollectiveMPIIOThreshold = T;
#endif
}
#endif
protected:
std::vector<Variable> Vars;
std::size_t NElems;
double PhysOrigin[3], PhysScale[3];
unsigned FileIOType;
int Partition;
#ifndef GENERICIO_NO_MPI
MPI_Comm Comm;
#endif
std::string FileName;
static unsigned DefaultFileIOType;
static int DefaultPartition;
static bool DefaultShouldCompress;
#ifndef GENERICIO_NO_MPI
static std::size_t CollectiveMPIIOThreshold;
#endif
std::vector<int> RankMap;
#ifndef GENERICIO_NO_MPI
MPI_Comm SplitComm;
#endif
std::string OpenFileName;
// This reference counting mechanism allows the the GenericIO class
// to be used in a cursor mode. To do this, make a copy of the class
// after reading the header but prior to adding the variables.
struct FHManager {
FHManager() : CountedFH(0) {
allocate();
}
FHManager(const FHManager& F) {
CountedFH = F.CountedFH;
CountedFH->Cnt += 1;
}
~FHManager() {
close();
}
GenericFileIO *&get() {
if (!CountedFH)
allocate();
return CountedFH->GFIO;
}
std::vector<char> &getHeaderCache() {
if (!CountedFH)
allocate();
return CountedFH->HeaderCache;
}
void allocate() {
close();
CountedFH = new FHWCnt;
};
void close() {
if (CountedFH && CountedFH->Cnt == 1)
delete CountedFH;
else if (CountedFH)
CountedFH->Cnt -= 1;
CountedFH = 0;
}
struct FHWCnt {
FHWCnt() : GFIO(0), Cnt(1) {}
~FHWCnt() {
close();
}
protected:
void close() {
delete GFIO;
GFIO = 0;
}
public:
GenericFileIO *GFIO;
size_t Cnt;
// Used for reading
std::vector<char> HeaderCache;
};
FHWCnt *CountedFH;
} FH;
};
} /* END namespace cosmotk */
#endif // GENERICIO_H
#include <cstdlib>
#include <cstdio>
#include <cstring>
#include <cmath>
#include <iostream>
#include <string>
#include <cassert>
#include "GenericIO.h"
#define POSVEL_T float
#define ID_T int64_t
#define MASK_T uint16_t
using namespace std;
using namespace gio;
// This code is based on restart2cosmo.cxx
int main(int argc, char *argv[]) {
#ifndef GENERICIO_NO_MPI
MPI_Init(&argc, &argv);
#endif
int commRank, commRanks;
#ifndef GENERICIO_NO_MPI
MPI_Comm_rank(MPI_COMM_WORLD, &commRank);
MPI_Comm_size(MPI_COMM_WORLD, &commRanks);
#else
commRank = 0;
commRanks = 1;
#endif
if(argc != 3 + commRanks) {
fprintf(stderr,"USAGE: %s <mpiioName> <cosmoName> <rank0> <rank1> ...\n",argv[0]);
exit(-1);
}
char *mpiioName = argv[1];
char *cosmoName = argv[2];
int rank = atoi(argv[3 + commRank]);
vector<POSVEL_T> xx, yy, zz, vx, vy, vz, phi;
vector<ID_T> id;
vector<MASK_T> mask;
assert(sizeof(ID_T) == 8);
size_t Np = 0;
{ // create GIO
unsigned Method = GenericIO::FileIOPOSIX;
#ifndef GENERICIO_NO_MPI
const char *EnvStr = getenv("GENERICIO_USE_MPIIO");
if (EnvStr && string(EnvStr) == "1")
Method = GenericIO::FileIOMPI;
#endif
GenericIO GIO(
#ifndef GENERICIO_NO_MPI
MPI_COMM_WORLD,
#endif
mpiioName, Method);
GIO.openAndReadHeader(false);
int NR = GIO.readNRanks();
if (rank >= NR) {
fprintf(stderr,"rank %d is invalid: file has data from %d ranks\n", rank, NR);
fflush(stderr);
exit(-1);
}
#ifndef GENERICIO_NO_MPI
MPI_Barrier(MPI_COMM_WORLD);
#endif
Np = GIO.readNumElems(rank);
xx.resize(Np + GIO.requestedExtraSpace()/sizeof(POSVEL_T));
yy.resize(Np + GIO.requestedExtraSpace()/sizeof(POSVEL_T));
zz.resize(Np + GIO.requestedExtraSpace()/sizeof(POSVEL_T));
vx.resize(Np + GIO.requestedExtraSpace()/sizeof(POSVEL_T));
vy.resize(Np + GIO.requestedExtraSpace()/sizeof(POSVEL_T));
vz.resize(Np + GIO.requestedExtraSpace()/sizeof(POSVEL_T));
phi.resize(Np + GIO.requestedExtraSpace()/sizeof(POSVEL_T));
id.resize(Np + GIO.requestedExtraSpace()/sizeof(ID_T));
mask.resize(Np + GIO.requestedExtraSpace()/sizeof(MASK_T));
GIO.addVariable("x", xx, true);
GIO.addVariable("y", yy, true);
GIO.addVariable("z", zz, true);
GIO.addVariable("vx", vx, true);
GIO.addVariable("vy", vy, true);
GIO.addVariable("vz", vz, true);
GIO.addVariable("phi", phi, true);
GIO.addVariable("id", id, true);
GIO.addVariable("mask", mask, true);
GIO.readData(rank);
} // destroy GIO
FILE *cosmoFile =
fopen((string(cosmoName) + "." + string(argv[3 + commRank])).c_str(), "wb");
for(size_t i=0; i<Np; i++) {
fwrite(&xx[i], sizeof(POSVEL_T), 1, cosmoFile);
fwrite(&vx[i], sizeof(POSVEL_T), 1, cosmoFile);
fwrite(&yy[i], sizeof(POSVEL_T), 1, cosmoFile);
fwrite(&vy[i], sizeof(POSVEL_T), 1, cosmoFile);
fwrite(&zz[i], sizeof(POSVEL_T), 1, cosmoFile);
fwrite(&vz[i], sizeof(POSVEL_T), 1, cosmoFile);
fwrite(&phi[i], sizeof(POSVEL_T), 1, cosmoFile);
fwrite(&id[i], sizeof(ID_T), 1, cosmoFile);
}
fclose(cosmoFile);
#ifndef GENERICIO_NO_MPI
MPI_Barrier(MPI_COMM_WORLD);
MPI_Finalize();
#endif
return 0;
}
#include <cstdlib>
#include <iostream>
#include <iomanip>
#include <string>
#include <algorithm>
#include <limits>
#include <stdexcept>
#include <stdint.h>
#include "GenericIO.h"
using namespace std;
using namespace gio;
class PrinterBase {
public:
virtual ~PrinterBase() {}
virtual void print(ostream &os, size_t i) = 0;
};
template <class T>
class Printer : public PrinterBase {
public:
Printer(GenericIO &G, size_t MNE, const string &N)
: Data(MNE + G.requestedExtraSpace()/sizeof(T)) {
G.addVariable(N, Data, true);
}
virtual void print(ostream &os, size_t i) {
os << scientific << setprecision(numeric_limits<T>::digits10) << Data[i];
}
protected:
vector<T> Data;
};