Commit 6461f57b authored by Hal Finkel's avatar Hal Finkel
Browse files

start working on sz intrgration

parent d6628a65
......@@ -57,7 +57,8 @@ BLOSC_CPPFLAGS := \
-Ithirdparty/blosc/internal-complibs/zstd-0.7.4/compress \
-Ithirdparty/blosc/internal-complibs/zstd-0.7.4/common \
-Ithirdparty/blosc/internal-complibs/zstd-0.7.4/dictBuilder \
-Ithirdparty/blosc/internal-complibs/zstd-0.7.4/decompress
-Ithirdparty/blosc/internal-complibs/zstd-0.7.4/decompress \
-Ithirdparty/SZ/sz/include
BASE_CPPFLAGS := $(BLOSC_CPPFLAGS) -I. -D__STDC_CONSTANT_MACROS
......@@ -125,7 +126,50 @@ BLOSC_O := \
thirdparty/blosc/internal-complibs/snappy-1.1.1/snappy-c.o \
thirdparty/blosc/internal-complibs/snappy-1.1.1/snappy.o \
thirdparty/blosc/internal-complibs/snappy-1.1.1/snappy-sinksource.o \
thirdparty/blosc/internal-complibs/snappy-1.1.1/snappy-stubs-internal.o
thirdparty/blosc/internal-complibs/snappy-1.1.1/snappy-stubs-internal.o \
thirdparty/SZ/sz/src/ByteToolkit.o \
thirdparty/SZ/sz/src/dataCompression.o \
thirdparty/SZ/sz/src/DynamicIntArray.o \
thirdparty/SZ/sz/src/iniparser.o \
thirdparty/SZ/sz/src/CompressElement.o \
thirdparty/SZ/sz/src/DynamicByteArray.o \
thirdparty/SZ/sz/src/rw.o \
thirdparty/SZ/sz/src/TightDataPointStorageI.o \
thirdparty/SZ/sz/src/TightDataPointStorageD.o \
thirdparty/SZ/sz/src/TightDataPointStorageF.o \
thirdparty/SZ/sz/src/conf.o \
thirdparty/SZ/sz/src/DynamicDoubleArray.o \
thirdparty/SZ/sz/src/TypeManager.o \
thirdparty/SZ/sz/src/dictionary.o \
thirdparty/SZ/sz/src/DynamicFloatArray.o \
thirdparty/SZ/sz/src/VarSet.o \
thirdparty/SZ/sz/src/callZlib.o \
thirdparty/SZ/sz/src/Huffman.o \
thirdparty/SZ/sz/src/sz_float.o \
thirdparty/SZ/sz/src/sz_double.o \
thirdparty/SZ/sz/src/sz_int8.o \
thirdparty/SZ/sz/src/sz_int16.o \
thirdparty/SZ/sz/src/sz_int32.o \
thirdparty/SZ/sz/src/sz_int64.o \
thirdparty/SZ/sz/src/sz_uint8.o \
thirdparty/SZ/sz/src/sz_uint16.o \
thirdparty/SZ/sz/src/sz_uint32.o \
thirdparty/SZ/sz/src/sz_uint64.o \
thirdparty/SZ/sz/src/szd_uint8.o \
thirdparty/SZ/sz/src/szd_uint16.o \
thirdparty/SZ/sz/src/szd_uint32.o \
thirdparty/SZ/sz/src/szd_uint64.o \
thirdparty/SZ/sz/src/szd_float.o \
thirdparty/SZ/sz/src/szd_double.o \
thirdparty/SZ/sz/src/szd_int8.o \
thirdparty/SZ/sz/src/szd_int16.o \
thirdparty/SZ/sz/src/szd_int32.o \
thirdparty/SZ/sz/src/szd_int64.o \
thirdparty/SZ/sz/src/sz.o \
thirdparty/SZ/sz/src/sz_float_pwr.o \
thirdparty/SZ/sz/src/sz_double_pwr.o \
thirdparty/SZ/sz/src/szd_float_pwr.o \
thirdparty/SZ/sz/src/szd_double_pwr.o
FE_BLOSC_O := $(addprefix $(FEDIR)/,$(BLOSC_O))
......
......@@ -46,6 +46,7 @@
extern "C" {
#include "blosc.h"
}
#include "sz.h"
#include <sstream>
#include <fstream>
......@@ -351,6 +352,8 @@ struct CompressHeader {
};
const char *CompressName = "BLOSC";
const char *LossyCompressName = "SZ";
#pragma pack()
unsigned GenericIO::DefaultFileIOType = FileIOPOSIX;
......@@ -362,6 +365,7 @@ std::size_t GenericIO::CollectiveMPIIOThreshold = 0;
#endif
static bool blosc_initialized = false;
static bool sz_initialized = false;
#ifndef GENERICIO_NO_MPI
void GenericIO::write() {
......
......@@ -191,6 +191,26 @@ public:
};
public:
struct LossyCompressionInfo {
enum LCMode {
LCModeNone,
LCModeAbs,
LCModeRel,
LCModeAbsAndRel,
LCModeAbsOrRel,
LCModePSNR
};
LCMode Mode;
double AbsErrThreshold;
double RelErrThreshold;
double PSNRThreshold;
LossyCompressionInfo()
: Mode(LCModeNone), AbsErrThreshold(0.0),
RelErrThreshold(0.0), PSNRThreshold(0.0) {}
};
class Variable {
private:
template <typename ET>
......@@ -231,28 +251,33 @@ public:
public:
template <typename T>
Variable(const std::string &N, T* D, unsigned Flags = 0)
Variable(const std::string &N, T* D, unsigned Flags = 0,
const LossyCompressionInfo &LCI = LossyCompressionInfo())
: Name(N), Data((void *) D), HasExtraSpace(Flags & VarHasExtraSpace),
IsPhysCoordX(Flags & VarIsPhysCoordX),
IsPhysCoordY(Flags & VarIsPhysCoordY),
IsPhysCoordZ(Flags & VarIsPhysCoordZ),
MaybePhysGhost(Flags & VarMaybePhysGhost) {
MaybePhysGhost(Flags & VarMaybePhysGhost),
LCI(LCI) {
deduceTypeInfo(D);
}
template <typename T>
Variable(const std::string &N, std::size_t NumElements, T* D,
unsigned Flags = 0)
unsigned Flags = 0,
const LossyCompressionInfo &LCI = LossyCompressionInfo())
: Name(N), Data((void *) D), HasExtraSpace(Flags & VarHasExtraSpace),
IsPhysCoordX(Flags & VarIsPhysCoordX),
IsPhysCoordY(Flags & VarIsPhysCoordY),
IsPhysCoordZ(Flags & VarIsPhysCoordZ),
MaybePhysGhost(Flags & VarMaybePhysGhost) {
MaybePhysGhost(Flags & VarMaybePhysGhost),
LCI(LCI) {
deduceTypeInfoFromElement(D);
Size = ElementSize*NumElements;
}
Variable(const VariableInfo &VI, void *D, unsigned Flags = 0)
Variable(const VariableInfo &VI, void *D, unsigned Flags = 0,
const LossyCompressionInfo &LCI = LossyCompressionInfo())
: Name(VI.Name), Size(VI.Size), IsFloat(VI.IsFloat),
IsSigned(VI.IsSigned), Data(D),
HasExtraSpace(Flags & VarHasExtraSpace),
......@@ -260,7 +285,7 @@ public:
IsPhysCoordY((Flags & VarIsPhysCoordY) || VI.IsPhysCoordY),
IsPhysCoordZ((Flags & VarIsPhysCoordZ) || VI.IsPhysCoordZ),
MaybePhysGhost((Flags & VarMaybePhysGhost) || VI.MaybePhysGhost),
ElementSize(VI.ElementSize) {}
ElementSize(VI.ElementSize), LCI(LCI) {}
std::string Name;
std::size_t Size;
......@@ -271,6 +296,8 @@ public:
bool IsPhysCoordX, IsPhysCoordY, IsPhysCoordZ;
bool MaybePhysGhost;
std::size_t ElementSize;
LossyCompressionInfo LCI;
};
public:
......@@ -340,33 +367,38 @@ public:
template <typename T>
void addVariable(const std::string &Name, T *Data,
unsigned Flags = 0) {
Vars.push_back(Variable(Name, Data, Flags));
unsigned Flags = 0,
const LossyCompressionInfo &LCI = LossyCompressionInfo()) {
Vars.push_back(Variable(Name, Data, Flags, LCI));
}
template <typename T, typename A>
void addVariable(const std::string &Name, std::vector<T, A> &Data,
unsigned Flags = 0) {
unsigned Flags = 0,
const LossyCompressionInfo &LCI = LossyCompressionInfo()) {
T *D = Data.empty() ? 0 : &Data[0];
addVariable(Name, D, Flags);
addVariable(Name, D, Flags, LCI);
}
void addVariable(const VariableInfo &VI, void *Data,
unsigned Flags = 0) {
Vars.push_back(Variable(VI, Data, Flags));
unsigned Flags = 0,
const LossyCompressionInfo &LCI = LossyCompressionInfo()) {
Vars.push_back(Variable(VI, Data, Flags, LCI));
}
template <typename T>
void addScalarizedVariable(const std::string &Name, T *Data,
std::size_t NumElements, unsigned Flags = 0) {
Vars.push_back(Variable(Name, NumElements, Data, Flags));
std::size_t NumElements, unsigned Flags = 0,
const LossyCompressionInfo &LCI = LossyCompressionInfo()) {
Vars.push_back(Variable(Name, NumElements, Data, Flags, LCI));
}
template <typename T, typename A>
void addScalarizedVariable(const std::string &Name, std::vector<T, A> &Data,
std::size_t NumElements, unsigned Flags = 0) {
std::size_t NumElements, unsigned Flags = 0,
const LossyCompressionInfo &LCI = LossyCompressionInfo()) {
T *D = Data.empty() ? 0 : &Data[0];
addScalarizedVariable(Name, D, NumElements, Flags);
addScalarizedVariable(Name, D, NumElements, Flags, LCI);
}
#ifndef GENERICIO_NO_MPI
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment