Commit 9ee2ce3c authored by Hal Finkel's avatar Hal Finkel
Browse files

importing new SZ files

parent abca1570
Copyright © 2016 , UChicago Argonne, LLC
All Rights Reserved
[SZ, Version 1.3]
[SZ, Version 1.4]
Sheng Di
Dingwen Tao
Xin Liang
Franck Cappello
Argonne National Laboratory
......
......@@ -23,6 +23,8 @@ typedef struct TightDataPointStorageD
char reqLength;
char radExpo; //used to compute reqLength based on segmented precisions in "pw_rel_compression"
double minLogValue;
int stateNum;
int allNodes;
......@@ -52,6 +54,10 @@ typedef struct TightDataPointStorageD
unsigned char* pwrErrBoundBytes;
int pwrErrBoundBytes_size;
unsigned char* raBytes;
size_t raBytes_size;
} TightDataPointStorageD;
void new_TightDataPointStorageD_Empty(TightDataPointStorageD **self);
......
......@@ -34,6 +34,8 @@ typedef struct TightDataPointStorageF
unsigned char* rtypeArray;
size_t rtypeArray_size;
float minLogValue;
unsigned char* typeArray; //its size is dataSeriesLength/4 (or xxx/4+1)
size_t typeArray_size;
......@@ -55,6 +57,9 @@ typedef struct TightDataPointStorageF
unsigned char* pwrErrBoundBytes;
int pwrErrBoundBytes_size;
unsigned char* raBytes;
size_t raBytes_size;
} TightDataPointStorageF;
void new_TightDataPointStorageF_Empty(TightDataPointStorageF **self);
......
......@@ -19,8 +19,10 @@ extern "C" {
//TypeManager.c
size_t convertIntArray2ByteArray_fast_1b(unsigned char* intArray, size_t intArrayLength, unsigned char **result);
size_t convertIntArray2ByteArray_fast_1b_to_result(unsigned char* intArray, size_t intArrayLength, unsigned char *result);
void convertByteArray2IntArray_fast_1b(size_t intArrayLength, unsigned char* byteArray, size_t byteArrayLength, unsigned char **intArray);
size_t convertIntArray2ByteArray_fast_2b(unsigned char* timeStepType, size_t timeStepTypeLength, unsigned char **result);
size_t convertIntArray2ByteArray_fast_2b_inplace(unsigned char* timeStepType, size_t timeStepTypeLength, unsigned char *result);
void convertByteArray2IntArray_fast_2b(size_t stepLength, unsigned char* byteArray, size_t byteArrayLength, unsigned char **intArray);
size_t convertIntArray2ByteArray_fast_3b(unsigned char* timeStepType, size_t timeStepTypeLength, unsigned char **result);
void convertByteArray2IntArray_fast_3b(size_t stepLength, unsigned char* byteArray, size_t byteArrayLength, unsigned char **intArray);
......
......@@ -19,6 +19,8 @@ extern "C" {
#include <stdio.h>
int isZlibFormat(unsigned char magic1, unsigned char magic2);
//callZlib.c
unsigned long zlib_compress(unsigned char* data, unsigned long dataLength, unsigned char** compressBytes, int level);
unsigned long zlib_compress2(unsigned char* data, unsigned long dataLength, unsigned char** compressBytes, int level);
......
......@@ -77,6 +77,18 @@ int computeBlockEdgeSize_3D(int segmentSize);
int computeBlockEdgeSize_2D(int segmentSize);
int initRandomAccessBytes(unsigned char* raBytes);
int generateLossyCoefficients_float(float* oriData, double precision, size_t nbEle, int* reqBytesLength, int* resiBitsLength, float* medianValue, float* decData);
int compressExactDataArray_float(float* oriData, double precision, size_t nbEle, unsigned char** leadArray, unsigned char** midArray, unsigned char** resiArray,
int reqLength, int reqBytesLength, int resiBitsLength, float medianValue);
void decompressExactDataArray_float(unsigned char* leadNum, unsigned char* exactMidBytes, unsigned char* residualMidBits, size_t nbEle, int reqLength, float medianValue, float** decData);
int generateLossyCoefficients_double(double* oriData, double precision, size_t nbEle, int* reqBytesLength, int* resiBitsLength, double* medianValue, double* decData);
int compressExactDataArray_double(double* oriData, double precision, size_t nbEle, unsigned char** leadArray, unsigned char** midArray, unsigned char** resiArray,
int reqLength, int reqBytesLength, int resiBitsLength, double medianValue);
void decompressExactDataArray_double(unsigned char* leadNum, unsigned char* exactMidBytes, unsigned char* residualMidBits, size_t nbEle, int reqLength, double medianValue, double** decData);
#ifdef __cplusplus
}
#endif
......
This diff is collapsed.
This diff is collapsed.
......@@ -54,6 +54,7 @@
#include "pastri.h"
#include "sz_float_ts.h"
#include "szd_float_ts.h"
#include "utility.h"
#ifdef _WIN32
#define PATH_SEPARATOR ';'
......@@ -74,11 +75,11 @@ extern "C" {
//typedef long int64_t;
//typedef unsigned long uint64_t;
#define SZ_VERNUM 0x0140
#define SZ_VER_MAJOR 1
#define SZ_VER_MINOR 4
#define SZ_VER_BUILD 13
#define SZ_VER_REVISION 5
#define SZ_VERNUM 0x0200
#define SZ_VER_MAJOR 2
#define SZ_VER_MINOR 0
#define SZ_VER_BUILD 2
#define SZ_VER_REVISION 0
#define PASTRI 103
#define HZ 102
......@@ -130,6 +131,9 @@ extern "C" {
#define SZ_DEFAULT_COMPRESSION 2
#define SZ_TEMPORAL_COMPRESSION 3
#define SZ_NO_REGRESSION 0
#define SZ_WITH_LINEAR_REGRESSION 1
#define SZ_PWR_MIN_TYPE 0
#define SZ_PWR_AVG_TYPE 1
#define SZ_PWR_MAX_TYPE 2
......@@ -151,6 +155,10 @@ extern "C" {
#define MetaDataByteLength 20
#define numOfBufferedSteps 1 //the number of time steps in the buffer
#define GZIP_COMPRESSOR 0 //i.e., ZLIB_COMPRSSOR
#define ZSTD_COMPRESSOR 1
//Note: the following setting should be consistent with stateNum in Huffman.h
//#define intvCapacity 65536
......@@ -236,6 +244,7 @@ typedef struct sz_params
unsigned int quantization_intervals;
unsigned int maxRangeRadius;
int sol_ID;// it's always SZ, unless the setting is PASTRI compression mode (./configure --enable-pastri)
int losslessCompressor;
int sampleDistance; //2 bytes
float predThreshold; // 2 bytes
int szMode; //* 0 (best speed) or 1 (better compression with Gzip) or 3 temporal-dimension based compression
......@@ -278,6 +287,10 @@ typedef struct sz_tsc_metainfo
int currentStep;
char metadata_filename[256];
FILE *metadata_file;
unsigned char* bit_array; //sihuan added
size_t intersect_size; //sihuan added
int64_t* hist_index; //sihuan added: prestep index
} sz_tsc_metadata;
extern int versionNumber[4];
......@@ -289,6 +302,8 @@ extern int sysEndianType; //*sysEndianType is actually set automatically.
extern sz_params *confparams_cpr;
extern sz_params *confparams_dec;
extern sz_exedata *exe_params;
extern int sz_with_regression;
//------------------------------------------------
extern SZ_VarSet* sz_varset;
extern sz_multisteps *multisteps; //compression based on multiple time steps (time-dimension based compression)
......@@ -356,8 +371,6 @@ void filloutDimArray(size_t* dim, size_t r5, size_t r4, size_t r3, size_t r2, si
size_t compute_total_batch_size();
int isZlibFormat(unsigned char magic1, unsigned char magic2);
void SZ_registerVar(char* varName, int dataType, void* data,
int errBoundMode, double absErrBound, double relBoundRatio, double pwRelBoundRatio,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
......
......@@ -75,6 +75,12 @@ size_t r1, size_t r2, size_t r3, size_t s1, size_t s2, size_t s3, size_t e1, siz
TightDataPointStorageD* SZ_compress_double_4D_MDQ_subblock(double *oriData, double realPrecision, double valueRangeSize, double medianValue_d,
size_t r1, size_t r2, size_t r3, size_t r4, size_t s1, size_t s2, size_t s3, size_t s4, size_t e1, size_t e2, size_t e3, size_t e4);
unsigned int optimize_intervals_double_2D_with_freq_and_dense_pos(double *oriData, size_t r1, size_t r2, double realPrecision, double * dense_pos, double * max_freq, double * mean_freq);
unsigned int optimize_intervals_double_3D_with_freq_and_dense_pos(double *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, double * dense_pos, double * max_freq, double * mean_freq);
unsigned char * SZ_compress_double_2D_MDQ_nonblocked_with_blocked_regression(double *oriData, size_t r1, size_t r2, double realPrecision, size_t * comp_size);
unsigned char * SZ_compress_double_3D_MDQ_nonblocked_with_blocked_regression(double *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, size_t * comp_size);
#ifdef __cplusplus
}
#endif
......
......@@ -37,6 +37,10 @@ double absErrBound, double relBoundRatio, double pwrErrRatio, double valueRangeS
void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwrgroup(unsigned char** newByteData, double *oriData,
size_t dataLength, double absErrBound, double relBoundRatio, double pwrErrRatio, double valueRangeSize, double medianValue_f, size_t *outSize);
void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char** newByteData, double *oriData, double globalPrecision, size_t dataLength, size_t *outSize, double min, double max);
void SZ_compress_args_double_NoCkRngeNoGzip_2D_pwr_pre_log(unsigned char** newByteData, double *oriData, double globalPrecision, size_t r1, size_t r2, size_t *outSize, double min, double max);
void SZ_compress_args_double_NoCkRngeNoGzip_3D_pwr_pre_log(unsigned char** newByteData, double *oriData, double globalPrecision, size_t r1, size_t r2, size_t r3, size_t *outSize, double min, double max);
#ifdef __cplusplus
}
#endif
......
......@@ -128,6 +128,14 @@ size_t r1, size_t r2, size_t r3, size_t s1, size_t s2, size_t s3, size_t e1, siz
TightDataPointStorageF* SZ_compress_float_4D_MDQ_subblock(float *oriData, double realPrecision, float valueRangeSize, float medianValue_f,
size_t r1, size_t r2, size_t r3, size_t r4, size_t s1, size_t s2, size_t s3, size_t s4, size_t e1, size_t e2, size_t e3, size_t e4);
unsigned int optimize_intervals_float_2D_with_freq_and_dense_pos(float *oriData, size_t r1, size_t r2, double realPrecision, float * dense_pos, float * max_freq, float * mean_freq);
unsigned int optimize_intervals_float_3D_with_freq_and_dense_pos(float *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, float * dense_pos, float * max_freq, float * mean_freq);
unsigned char * SZ_compress_float_2D_MDQ_nonblocked_with_blocked_regression(float *oriData, size_t r1, size_t r2, double realPrecision, size_t * comp_size);
unsigned char * SZ_compress_float_3D_MDQ_nonblocked_with_blocked_regression(float *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, size_t * comp_size);
unsigned char * SZ_compress_float_3D_MDQ_random_access_with_blocked_regression(float *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, size_t * comp_size);
#ifdef __cplusplus
}
#endif
......
......@@ -44,6 +44,10 @@ double absErrBound, double relBoundRatio, double pwrErrRatio, float valueRangeSi
void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwrgroup(unsigned char** newByteData, float *oriData,
size_t dataLength, double absErrBound, double relBoundRatio, double pwrErrRatio, float valueRangeSize, float medianValue_f, size_t *outSize);
void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char** newByteData, float *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, float min, float max);
void SZ_compress_args_float_NoCkRngeNoGzip_2D_pwr_pre_log(unsigned char** newByteData, float *oriData, double pwrErrRatio, size_t r1, size_t r2, size_t *outSize, float min, float max);
void SZ_compress_args_float_NoCkRngeNoGzip_3D_pwr_pre_log(unsigned char** newByteData, float *oriData, double pwrErrRatio, size_t r1, size_t r2, size_t r3, size_t *outSize, float min, float max);
#ifdef __cplusplus
}
#endif
......
......@@ -24,6 +24,8 @@ void getSnapshotData_double_1D(double** data, size_t dataSeriesLength, TightData
void getSnapshotData_double_2D(double** data, size_t r1, size_t r2, TightDataPointStorageD* tdps, int errBoundMode);
void getSnapshotData_double_3D(double** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageD* tdps, int errBoundMode);
void getSnapshotData_double_4D(double** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageD* tdps, int errBoundMode);
void decompressDataSeries_double_2D_nonblocked_with_blocked_regression(double** data, size_t r1, size_t r2, unsigned char* comp_data);
void decompressDataSeries_double_3D_nonblocked_with_blocked_regression(double** data, size_t r1, size_t r2, size_t r3, unsigned char* comp_data);
int SZ_decompress_args_double(double** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, size_t cmpSize);
......
......@@ -21,6 +21,10 @@ double* extractRealPrecision_3D_double(size_t R1, size_t R2, size_t R3, int bloc
void decompressDataSeries_double_3D_pwr(double** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageD* tdps);
void decompressDataSeries_double_1D_pwrgroup(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps);
void decompressDataSeries_double_1D_pwr_pre_log(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps);
void decompressDataSeries_double_2D_pwr_pre_log(double** data, size_t r1, size_t r2, TightDataPointStorageD* tdps);
void decompressDataSeries_double_3D_pwr_pre_log(double** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageD* tdps);
#ifdef __cplusplus
}
#endif
......
......@@ -31,6 +31,11 @@ size_t decompressDataSeries_float_2D_RA_block(float * data, float mean, size_t d
int SZ_decompress_args_float(float** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, size_t cmpSize);
size_t decompressDataSeries_float_3D_RA_block(float * data, float mean, size_t dim_0, size_t dim_1, size_t dim_2, size_t block_dim_0, size_t block_dim_1, size_t block_dim_2, double realPrecision, int * type, float * unpredictable_data);
void decompressDataSeries_float_2D_nonblocked_with_blocked_regression(float** data, size_t r1, size_t r2, unsigned char* comp_data);
void decompressDataSeries_float_3D_nonblocked_with_blocked_regression(float** data, size_t r1, size_t r2, size_t r3, unsigned char* comp_data);
void decompressDataSeries_float_3D_random_access_with_blocked_regression(float** data, size_t r1, size_t r2, size_t r3, unsigned char* comp_data);
#ifdef __cplusplus
}
#endif
......
......@@ -22,6 +22,9 @@ void decompressDataSeries_float_3D_pwr(float** data, size_t r1, size_t r2, size_
char* decompressGroupIDArray(unsigned char* bytes, size_t dataLength);
void decompressDataSeries_float_1D_pwrgroup(float** data, size_t dataSeriesLength, TightDataPointStorageF* tdps);
void decompressDataSeries_float_1D_pwr_pre_log(float** data, size_t dataSeriesLength, TightDataPointStorageF* tdps);
void decompressDataSeries_float_2D_pwr_pre_log(float** data, size_t r1, size_t r2, TightDataPointStorageF* tdps);
void decompressDataSeries_float_3D_pwr_pre_log(float** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageF* tdps);
#ifdef __cplusplus
}
......
/**
* @file utility.h
* @author Sheng Di, Sihuan Li
* @date July, 2018
* @brief Header file for the utility.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _UTILITY_H
#define _UTILITY_H
#include "sz.h"
#ifdef __cplusplus
extern "C" {
#endif
//sihuan added: use a assistant struct to do sorting and swap that are easy to implement: should
//consider optimizing the performance later.
typedef struct sort_ast_particle{
int64_t id;
float var[6];
} sort_ast_particle;
int compare_struct(const void* obj1, const void* obj2);//sihuan added: the compare function in the qsort parameter for 2 structures
void reorder_vars(SZ_VarSet* vset);//sihuan added: reorder the variables increasingly by their index
size_t intersectAndsort(int64_t* preIndex, size_t preLen, SZ_VarSet* curVar, size_t dataLen, unsigned char* bitarray);
//sihuan added: find intersection and keep new var sorted by id
void write_reordered_tofile(SZ_VarSet* curVar, size_t dataLen);
//sihuan added: write the reordered input to files for further decompression validation
float calculate_delta_t(size_t size);//sihuan added
int is_lossless_compressed_data(unsigned char* compressedBytes, size_t cmpSize);
unsigned long sz_lossless_compress(int losslessCompressor, int level, unsigned char* data, unsigned long dataLength, unsigned char** compressBytes);
unsigned long sz_lossless_decompress(int losslessCompressor, unsigned char* compressBytes, unsigned long cmpSize, unsigned char** oriData, unsigned long targetOriSize);
unsigned long sz_lossless_decompress65536bytes(int losslessCompressor, unsigned char* compressBytes, unsigned long cmpSize, unsigned char** oriData);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _UTILITY_H ----- */
......@@ -430,7 +430,7 @@ long bytesToLong(unsigned char* bytes)
}
//the byte to input is in the big-endian format
float bytesToFloat(unsigned char* bytes)
inline float bytesToFloat(unsigned char* bytes)
{
lfloat buf;
memcpy(buf.byte, bytes, 4);
......@@ -439,7 +439,7 @@ float bytesToFloat(unsigned char* bytes)
return buf.value;
}
void floatToBytes(unsigned char *b, float num)
inline void floatToBytes(unsigned char *b, float num)
{
lfloat buf;
buf.value = num;
......@@ -449,7 +449,7 @@ void floatToBytes(unsigned char *b, float num)
}
//the byte to input is in the big-endian format
double bytesToDouble(unsigned char* bytes)
inline double bytesToDouble(unsigned char* bytes)
{
ldouble buf;
memcpy(buf.byte, bytes, 8);
......@@ -458,7 +458,7 @@ double bytesToDouble(unsigned char* bytes)
return buf.value;
}
void doubleToBytes(unsigned char *b, double num)
inline void doubleToBytes(unsigned char *b, double num)
{
ldouble buf;
buf.value = num;
......@@ -507,7 +507,7 @@ int extractBytes(unsigned char* byteArray, size_t k, int validLength)
return result;
}
int getMaskRightCode(int m) {
inline int getMaskRightCode(int m) {
switch (m) {
case 1:
return 0x01;
......@@ -530,16 +530,16 @@ int getMaskRightCode(int m) {
}
}
int getLeftMovingCode(int kMod8)
inline int getLeftMovingCode(int kMod8)
{
return getMaskRightCode(8 - kMod8);
}
int getRightMovingSteps(int kMod8, int resiBitLength) {
inline int getRightMovingSteps(int kMod8, int resiBitLength) {
return 8 - kMod8 - resiBitLength;
}
int getRightMovingCode(int kMod8, int resiBitLength)
inline int getRightMovingCode(int kMod8, int resiBitLength)
{
int rightMovingSteps = 8 - kMod8 - resiBitLength;
if(rightMovingSteps < 0)
......@@ -814,7 +814,7 @@ void convertULongArrayToBytes(uint64_t* states, size_t stateLength, unsigned cha
}
size_t bytesToSize(unsigned char* bytes)
inline size_t bytesToSize(unsigned char* bytes)
{
size_t result = 0;
if(exe_params->SZ_SIZE_TYPE==4)
......@@ -824,7 +824,7 @@ size_t bytesToSize(unsigned char* bytes)
return result;
}
void sizeToBytes(unsigned char* outBytes, size_t size)
inline void sizeToBytes(unsigned char* outBytes, size_t size)
{
if(exe_params->SZ_SIZE_TYPE==4)
intToBytes_bigEndian(outBytes, size);//4
......
......@@ -21,7 +21,7 @@ void new_DDA(DynamicDoubleArray **dda, size_t cap) {
void convertDDAtoDoubles(DynamicDoubleArray *dba, double **data)
{
int size = dba->size;
size_t size = dba->size;
if(size>0)
*data = (double*)malloc(size * sizeof(double));
else
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment