Commit fc4fc2fc authored by Hal Finkel's avatar Hal Finkel
Browse files

add a GIO rewriter (to compress files, etc.)

parent 8ebc79bb
......@@ -180,10 +180,13 @@ $(MPIDIR)/GenericIOBenchmarkRead: $(MPIDIR)/GenericIOBenchmarkRead.o $(MPIDIR)/G
$(MPIDIR)/GenericIOBenchmarkWrite: $(MPIDIR)/GenericIOBenchmarkWrite.o $(MPIDIR)/GenericIO.o $(MPI_BLOSC_O)
$(MPICXX) $(MPI_CFLAGS) -o $@ $^
$(MPIDIR)/GenericIORewrite: $(MPIDIR)/GenericIORewrite.o $(MPIDIR)/GenericIO.o $(MPI_BLOSC_O)
$(MPICXX) $(MPI_CFLAGS) -o $@ $^
frontend-progs: $(FEDIR)/GenericIOPrint $(FEDIR)/GenericIOVerify
fe-progs: frontend-progs
mpi-progs: $(MPIDIR)/GenericIOPrint $(MPIDIR)/GenericIOVerify $(MPIDIR)/GenericIOBenchmarkRead $(MPIDIR)/GenericIOBenchmarkWrite
mpi-progs: $(MPIDIR)/GenericIOPrint $(MPIDIR)/GenericIOVerify $(MPIDIR)/GenericIOBenchmarkRead $(MPIDIR)/GenericIOBenchmarkWrite $(MPIDIR)/GenericIORewrite
frontend-sqlite: $(FEDIR)/GenericIOSQLite.so $(FEDIR)/sqlite3
fe-sqlite: frontend-sqlite
......
#include <cstdlib>
#include <iostream>
#include <iomanip>
#include <string>
#include <algorithm>
#include <limits>
#include <stdexcept>
#include <stdint.h>
#include "GenericIO.h"
using namespace gio;
using namespace std;
int main(int argc, char *argv[]) {
MPI_Init(&argc, &argv);
if (argc < 2) {
cerr << "Usage: " << argv[0] << " <mpiioOld> <mpiioNew>" << endl;
exit(-1);
}
GenericIO::setNaturalDefaultPartition();
GenericIO::setDefaultShouldCompress(true);
{
int arg = 1;
int Rank, NRanks;
MPI_Comm_rank(MPI_COMM_WORLD, &Rank);
MPI_Comm_size(MPI_COMM_WORLD, &NRanks);
string FileName(argv[arg++]);
string NewFileName(argv[arg++]);
unsigned Method = GenericIO::FileIOPOSIX;
const char *EnvStr = getenv("GENERICIO_USE_MPIIO");
if (EnvStr && string(EnvStr) == "1")
Method = GenericIO::FileIOMPI;
GenericIO GIO(MPI_COMM_WORLD, FileName, Method);
GIO.openAndReadHeader(GenericIO::MismatchDisallowed);
vector<GenericIO::VariableInfo> VI;
GIO.getVariableInfo(VI);
size_t NElem = GIO.readNumElems();
double PhysOrigin[3], PhysScale[3];
GIO.readPhysOrigin(PhysOrigin);
GIO.readPhysScale(PhysScale);
vector< vector<char> > Vars(VI.size());
for (size_t i = 0; i < VI.size(); ++i) {
Vars[i].resize(VI[i].Size*NElem + GIO.requestedExtraSpace());
GIO.addVariable(VI[i], &Vars[i][0], GenericIO::VarHasExtraSpace);
}
GIO.readData(-1, false);
GenericIO NewGIO(MPI_COMM_WORLD, NewFileName);
NewGIO.setNumElems(NElem);
for (int d = 0; d < 3; ++d) {
NewGIO.setPhysOrigin(PhysOrigin[d], d);
NewGIO.setPhysScale(PhysScale[d], d);
}
for (size_t i = 0; i < VI.size(); ++i)
NewGIO.addVariable(VI[i], &Vars[i][0], GenericIO::VarHasExtraSpace);
NewGIO.write();
}
MPI_Barrier(MPI_COMM_WORLD);
MPI_Finalize();
return 0;
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment