Commit eeacdada authored by Hal Finkel's avatar Hal Finkel
Browse files

initial SZ integration - it compiles

parent 2c47b734
......@@ -367,6 +367,31 @@ std::size_t GenericIO::CollectiveMPIIOThreshold = 0;
static bool blosc_initialized = false;
static bool sz_initialized = false;
static int GetSZDT(GenericIO::Variable &Var) {
if (Var.hasElementType<float>())
return SZ_FLOAT;
else if (Var.hasElementType<double>())
return SZ_DOUBLE;
else if (Var.hasElementType<uint8_t>())
return SZ_UINT8;
else if (Var.hasElementType<int8_t>())
return SZ_INT8;
else if (Var.hasElementType<uint16_t>())
return SZ_UINT16;
else if (Var.hasElementType<int16_t>())
return SZ_INT16;
else if (Var.hasElementType<uint32_t>())
return SZ_UINT32;
else if (Var.hasElementType<int32_t>())
return SZ_INT32;
else if (Var.hasElementType<uint64_t>())
return SZ_UINT64;
else if (Var.hasElementType<int64_t>())
return SZ_INT64;
else
return -1;
}
#ifndef GENERICIO_NO_MPI
void GenericIO::write() {
if (isBigEndian())
......@@ -500,10 +525,60 @@ void GenericIO::write() {
// calculated by the header-writing rank).
memset(&LocalBlockHeaders[i], 0, sizeof(BlockHeader<IsBigEndian>));
if (ShouldCompress) {
void *OrigData = Vars[i].Data;
bool FreeOrigData = false;
size_t OrigUnitSize = Vars[i].Size;
size_t OrigDataSize = NElems*Vars[i].Size;
int FilterIdx = 0;
if (Vars[i].LCI.Mode != LossyCompressionInfo::LCModeNone) {
int SZDT = GetSZDT(Vars[i]);
if (SZDT == -1)
goto nosz;
int EBM;
switch (Vars[i].LCI.Mode) {
case LossyCompressionInfo::LCModeAbs:
EBM = ABS;
break;
case LossyCompressionInfo::LCModeRel:
EBM = REL;
break;
case LossyCompressionInfo::LCModeAbsAndRel:
EBM = ABS_AND_REL;
break;
case LossyCompressionInfo::LCModeAbsOrRel:
EBM = ABS_OR_REL;
break;
case LossyCompressionInfo::LCModePSNR:
EBM = PSNR;
break;
}
size_t LOutSize;
unsigned char *LCompressedData = SZ_compress_args(SZDT, Vars[i].Data, &LOutSize, EBM,
Vars[i].LCI.AbsErrThreshold, Vars[i].LCI.RelErrThreshold,
Vars[i].LCI.PSNRThreshold, 0, 0, 0, 0, NElems);
if (!LCompressedData)
goto nosz;
if (LOutSize >= NElems*Vars[i].Size) {
free(LCompressedData);
goto nosz;
}
OrigData = LCompressedData;
FreeOrigData = true;
OrigUnitSize = 1;
OrigDataSize = LOutSize;
strncpy(LocalBlockHeaders[i].Filters[FilterIdx++], LossyCompressName, FilterNameSize);
}
nosz:
LocalCData[i].resize(sizeof(CompressHeader<IsBigEndian>));
CompressHeader<IsBigEndian> *CH = (CompressHeader<IsBigEndian>*) &LocalCData[i][0];
CH->OrigCRC = crc64_omp(Vars[i].Data, Vars[i].Size*NElems);
CH->OrigCRC = crc64_omp(OrigData, OrigDataSize);
#ifdef _OPENMP
#pragma omp master
......@@ -525,13 +600,20 @@ void GenericIO::write() {
}
#endif
LocalCData[i].resize(LocalCData[i].size() + NElems*Vars[i].Size);
if (blosc_compress(9, 1, Vars[i].Size, NElems*Vars[i].Size, Vars[i].Data,
LocalCData[i].resize(LocalCData[i].size() + OrigDataSize);
if (blosc_compress(9, 1, OrigUnitSize, OrigDataSize, OrigData,
&LocalCData[i][0] + sizeof(CompressHeader<IsBigEndian>),
NElems*Vars[i].Size) <= 0)
OrigDataSize) <= 0) {
if (FreeOrigData)
free(OrigData);
goto nocomp;
}
strncpy(LocalBlockHeaders[i].Filters[0], CompressName, FilterNameSize);
if (FreeOrigData)
free(OrigData);
strncpy(LocalBlockHeaders[i].Filters[FilterIdx++], CompressName, FilterNameSize);
size_t CNBytes, CCBytes, CBlockSize;
blosc_cbuffer_sizes(&LocalCData[i][0] + sizeof(CompressHeader<IsBigEndian>),
&CNBytes, &CCBytes, &CBlockSize);
......@@ -1434,6 +1516,7 @@ void GenericIO::readData(int EffRank, size_t RowOffset, int Rank,
void *VarData = ((char *) Vars[i].Data) + VarOffset;
vector<unsigned char> LData;
bool HasSZ = false;
void *Data = VarData;
bool HasExtraSpace = Vars[i].HasExtraSpace;
if (offsetof_safe(GH, BlocksStart) < GH->GlobalHeaderSize &&
......@@ -1444,11 +1527,18 @@ void GenericIO::readData(int EffRank, size_t RowOffset, int Rank,
ReadSize = BH->Size + CRCSize;
Offset = BH->Start;
if (strncmp(BH->Filters[0], CompressName, FilterNameSize) == 0) {
int FilterIdx = 0;
if (strncmp(BH->Filters[FilterIdx], LossyCompressName, FilterNameSize) == 0) {
++FilterIdx;
HasSZ = true;
}
if (strncmp(BH->Filters[FilterIdx], CompressName, FilterNameSize) == 0) {
LData.resize(ReadSize);
Data = &LData[0];
HasExtraSpace = true;
} else if (BH->Filters[0][0] != '\0') {
} else if (BH->Filters[FilterIdx][0] != '\0') {
stringstream ss;
ss << "Unknown filter \"" << BH->Filters[0] << "\" on variable " << Vars[i].Name;
throw runtime_error(ss.str());
......@@ -1573,22 +1663,50 @@ void GenericIO::readData(int EffRank, size_t RowOffset, int Rank,
blosc_initialized = true;
}
if (!sz_initialized) {
SZ_Init(NULL);
sz_initialized = true;
}
#ifdef _OPENMP
blosc_set_nthreads(omp_get_max_threads());
}
#endif
void *OrigData = VarData;
size_t OrigDataSize = Vars[i].Size*RH->NElems;
if (HasSZ) {
size_t CNBytes, CCBytes, CBlockSize;
blosc_cbuffer_sizes(&LData[0] + sizeof(CompressHeader<IsBigEndian>),
&CNBytes, &CCBytes, &CBlockSize);
OrigData = malloc(CNBytes);
OrigDataSize = CNBytes;
}
blosc_decompress(&LData[0] + sizeof(CompressHeader<IsBigEndian>),
VarData, Vars[i].Size*RH->NElems);
OrigData, OrigDataSize);
if (CH->OrigCRC != crc64_omp(VarData, Vars[i].Size*RH->NElems)) {
if (CH->OrigCRC != crc64_omp(OrigData, OrigDataSize)) {
++NErrs[2];
break;
}
if (HasSZ) {
int SZDT = GetSZDT(Vars[i]);
size_t LDSz = SZ_decompress_args(SZDT, (unsigned char *)OrigData, OrigDataSize,
VarData, 0, 0, 0, 0, RH->NElems);
free(OrigData);
if (LDSz != Vars[i].Size*RH->NElems)
throw runtime_error("Variable " + Vars[i].Name +
": SZ decompression yielded the wrong amount of data");
}
}
// Byte swap the data if necessary.
if (IsBigEndian != isBigEndian())
if (IsBigEndian != isBigEndian() && !HasSZ)
for (size_t j = 0;
j < RH->NElems*(Vars[i].Size/Vars[i].ElementSize); ++j) {
char *Offset = ((char *) VarData) + j*Vars[i].ElementSize;
......
......@@ -287,6 +287,18 @@ public:
MaybePhysGhost((Flags & VarMaybePhysGhost) || VI.MaybePhysGhost),
ElementSize(VI.ElementSize), LCI(LCI) {}
template <typename ET>
bool hasElementType() {
if (ElementSize != sizeof(ET))
return false;
if (IsFloat != !std::numeric_limits<ET>::is_integer)
return false;
if (IsSigned != std::numeric_limits<ET>::is_signed)
return false;
return true;
}
std::string Name;
std::size_t Size;
bool IsFloat;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment