importing new SZ files

thirdparty/SZ/COPYRIGHT.txt

 Copyright © 2016 , UChicago Argonne, LLC
 All Rights Reserved
-[SZ, Version 1.3]
+[SZ, Version 1.4]
 Sheng Di
 Dingwen Tao
+Xin Liang
 Franck Cappello
 Argonne National Laboratory
 

thirdparty/SZ/sz/include/TightDataPointStorageD.h

@@ -23,6 +23,8 @@ typedef struct TightDataPointStorageD
 	char reqLength;	
 	char radExpo; //used to compute reqLength based on segmented precisions in "pw_rel_compression"
 
+	double minLogValue;
+
 	int stateNum;
 	int allNodes;
 
@@ -52,6 +54,10 @@ typedef struct TightDataPointStorageD
 	
 	unsigned char* pwrErrBoundBytes;
 	int pwrErrBoundBytes_size;
+		
+	unsigned char* raBytes;
+	size_t raBytes_size;
+	
 } TightDataPointStorageD;
 
 void new_TightDataPointStorageD_Empty(TightDataPointStorageD **self);

thirdparty/SZ/sz/include/TightDataPointStorageF.h

@@ -34,6 +34,8 @@ typedef struct TightDataPointStorageF
 	unsigned char* rtypeArray;
 	size_t rtypeArray_size;
 	
+	float minLogValue;
+
 	unsigned char* typeArray; //its size is dataSeriesLength/4 (or xxx/4+1) 
 	size_t typeArray_size;
 	
@@ -55,6 +57,9 @@ typedef struct TightDataPointStorageF
 	unsigned char* pwrErrBoundBytes;
 	int pwrErrBoundBytes_size;
 	
+	unsigned char* raBytes;
+	size_t raBytes_size;
+	
 } TightDataPointStorageF;
 
 void new_TightDataPointStorageF_Empty(TightDataPointStorageF **self);

thirdparty/SZ/sz/include/TypeManager.h

@@ -19,8 +19,10 @@ extern "C" {
 
 //TypeManager.c
 size_t convertIntArray2ByteArray_fast_1b(unsigned char* intArray, size_t intArrayLength, unsigned char **result);
+size_t convertIntArray2ByteArray_fast_1b_to_result(unsigned char* intArray, size_t intArrayLength, unsigned char *result);
 void convertByteArray2IntArray_fast_1b(size_t intArrayLength, unsigned char* byteArray, size_t byteArrayLength, unsigned char **intArray);
 size_t convertIntArray2ByteArray_fast_2b(unsigned char* timeStepType, size_t timeStepTypeLength, unsigned char **result);
+size_t convertIntArray2ByteArray_fast_2b_inplace(unsigned char* timeStepType, size_t timeStepTypeLength, unsigned char *result);
 void convertByteArray2IntArray_fast_2b(size_t stepLength, unsigned char* byteArray, size_t byteArrayLength, unsigned char **intArray);
 size_t convertIntArray2ByteArray_fast_3b(unsigned char* timeStepType, size_t timeStepTypeLength, unsigned char **result);
 void convertByteArray2IntArray_fast_3b(size_t stepLength, unsigned char* byteArray, size_t byteArrayLength, unsigned char **intArray);

thirdparty/SZ/sz/include/callZlib.h

@@ -19,6 +19,8 @@ extern "C" {
 
 #include <stdio.h>
 
+int isZlibFormat(unsigned char magic1, unsigned char magic2);
+
 //callZlib.c
 unsigned long zlib_compress(unsigned char* data, unsigned long dataLength, unsigned char** compressBytes, int level);
 unsigned long zlib_compress2(unsigned char* data, unsigned long dataLength, unsigned char** compressBytes, int level);

thirdparty/SZ/sz/include/dataCompression.h

@@ -77,6 +77,18 @@ int computeBlockEdgeSize_3D(int segmentSize);
 int computeBlockEdgeSize_2D(int segmentSize);
 int initRandomAccessBytes(unsigned char* raBytes);
 
+int generateLossyCoefficients_float(float* oriData, double precision, size_t nbEle, int* reqBytesLength, int* resiBitsLength, float* medianValue, float* decData);
+int compressExactDataArray_float(float* oriData, double precision, size_t nbEle, unsigned char** leadArray, unsigned char** midArray, unsigned char** resiArray, 
+int reqLength, int reqBytesLength, int resiBitsLength, float medianValue);
+
+void decompressExactDataArray_float(unsigned char* leadNum, unsigned char* exactMidBytes, unsigned char* residualMidBits, size_t nbEle, int reqLength, float medianValue, float** decData);
+
+int generateLossyCoefficients_double(double* oriData, double precision, size_t nbEle, int* reqBytesLength, int* resiBitsLength, double* medianValue, double* decData);
+int compressExactDataArray_double(double* oriData, double precision, size_t nbEle, unsigned char** leadArray, unsigned char** midArray, unsigned char** resiArray, 
+int reqLength, int reqBytesLength, int resiBitsLength, double medianValue);
+
+void decompressExactDataArray_double(unsigned char* leadNum, unsigned char* exactMidBytes, unsigned char* residualMidBits, size_t nbEle, int reqLength, double medianValue, double** decData);
+
 #ifdef __cplusplus
 }
 #endif

thirdparty/SZ/sz/include/sz.h

@@ -54,6 +54,7 @@
 #include "pastri.h"
 #include "sz_float_ts.h"
 #include "szd_float_ts.h"
+#include "utility.h"
 
 #ifdef _WIN32
 #define PATH_SEPARATOR ';'
@@ -74,11 +75,11 @@ extern "C" {
 //typedef long int64_t;
 //typedef unsigned long uint64_t;
 
-#define SZ_VERNUM 0x0140
-#define SZ_VER_MAJOR 1
-#define SZ_VER_MINOR 4
-#define SZ_VER_BUILD 13
-#define SZ_VER_REVISION 5
+#define SZ_VERNUM 0x0200
+#define SZ_VER_MAJOR 2
+#define SZ_VER_MINOR 0
+#define SZ_VER_BUILD 2
+#define SZ_VER_REVISION 0
 
 #define PASTRI 103
 #define HZ 102
@@ -130,6 +131,9 @@ extern "C" {
 #define SZ_DEFAULT_COMPRESSION 2
 #define SZ_TEMPORAL_COMPRESSION 3
 
+#define SZ_NO_REGRESSION 0
+#define SZ_WITH_LINEAR_REGRESSION 1
+
 #define SZ_PWR_MIN_TYPE 0
 #define SZ_PWR_AVG_TYPE 1
 #define SZ_PWR_MAX_TYPE 2
@@ -151,6 +155,10 @@ extern "C" {
 #define MetaDataByteLength 20	
 	
 #define numOfBufferedSteps 1 //the number of time steps in the buffer	
+
+
+#define GZIP_COMPRESSOR 0 //i.e., ZLIB_COMPRSSOR
+#define ZSTD_COMPRESSOR 1
 	
 //Note: the following setting should be consistent with stateNum in Huffman.h
 //#define intvCapacity 65536
@@ -236,6 +244,7 @@ typedef struct sz_params
 	unsigned int quantization_intervals; 
 	unsigned int maxRangeRadius;
 	int sol_ID;// it's always SZ, unless the setting is PASTRI compression mode (./configure --enable-pastri)
+	int losslessCompressor;
 	int sampleDistance; //2 bytes
 	float predThreshold;  // 2 bytes
 	int szMode; //* 0 (best speed) or 1 (better compression with Gzip) or 3 temporal-dimension based compression
@@ -278,6 +287,10 @@ typedef struct sz_tsc_metainfo
 	int currentStep;
 	char metadata_filename[256];
 	FILE *metadata_file;
+	unsigned char* bit_array; //sihuan added
+	size_t intersect_size; //sihuan added
+	int64_t* hist_index; //sihuan added: prestep index 
+
 } sz_tsc_metadata;
 
 extern int versionNumber[4];
@@ -289,6 +302,8 @@ extern int sysEndianType; //*sysEndianType is actually set automatically.
 extern sz_params *confparams_cpr;
 extern sz_params *confparams_dec;
 extern sz_exedata *exe_params;
+extern int sz_with_regression;
+
 //------------------------------------------------
 extern SZ_VarSet* sz_varset;
 extern sz_multisteps *multisteps; //compression based on multiple time steps (time-dimension based compression)
@@ -356,8 +371,6 @@ void filloutDimArray(size_t* dim, size_t r5, size_t r4, size_t r3, size_t r2, si
 
 size_t compute_total_batch_size();
 
-int isZlibFormat(unsigned char magic1, unsigned char magic2);
-
 void SZ_registerVar(char* varName, int dataType, void* data, 
 			int errBoundMode, double absErrBound, double relBoundRatio, double pwRelBoundRatio, 
 			size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);

thirdparty/SZ/sz/include/sz_double.h

@@ -75,6 +75,12 @@ size_t r1, size_t r2, size_t r3, size_t s1, size_t s2, size_t s3, size_t e1, siz
 TightDataPointStorageD* SZ_compress_double_4D_MDQ_subblock(double *oriData, double realPrecision, double valueRangeSize, double medianValue_d,
 size_t r1, size_t r2, size_t r3, size_t r4, size_t s1, size_t s2, size_t s3, size_t s4, size_t e1, size_t e2, size_t e3, size_t e4);
 
+unsigned int optimize_intervals_double_2D_with_freq_and_dense_pos(double *oriData, size_t r1, size_t r2, double realPrecision, double * dense_pos, double * max_freq, double * mean_freq);
+unsigned int optimize_intervals_double_3D_with_freq_and_dense_pos(double *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, double * dense_pos, double * max_freq, double * mean_freq);
+unsigned char * SZ_compress_double_2D_MDQ_nonblocked_with_blocked_regression(double *oriData, size_t r1, size_t r2, double realPrecision, size_t * comp_size);
+unsigned char * SZ_compress_double_3D_MDQ_nonblocked_with_blocked_regression(double *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, size_t * comp_size);
+
+
 #ifdef __cplusplus
 }
 #endif

thirdparty/SZ/sz/include/sz_double_pwr.h

@@ -37,6 +37,10 @@ double absErrBound, double relBoundRatio, double pwrErrRatio, double valueRangeS
 void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwrgroup(unsigned char** newByteData, double *oriData,
 size_t dataLength, double absErrBound, double relBoundRatio, double pwrErrRatio, double valueRangeSize, double medianValue_f, size_t *outSize);
 
+void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char** newByteData, double *oriData, double globalPrecision, size_t dataLength, size_t *outSize, double min, double max);
+void SZ_compress_args_double_NoCkRngeNoGzip_2D_pwr_pre_log(unsigned char** newByteData, double *oriData, double globalPrecision, size_t r1, size_t r2, size_t *outSize, double min, double max);
+void SZ_compress_args_double_NoCkRngeNoGzip_3D_pwr_pre_log(unsigned char** newByteData, double *oriData, double globalPrecision, size_t r1, size_t r2, size_t r3, size_t *outSize, double min, double max);
+
 #ifdef __cplusplus
 }
 #endif

thirdparty/SZ/sz/include/sz_float.h

@@ -128,6 +128,14 @@ size_t r1, size_t r2, size_t r3, size_t s1, size_t s2, size_t s3, size_t e1, siz
 TightDataPointStorageF* SZ_compress_float_4D_MDQ_subblock(float *oriData, double realPrecision, float valueRangeSize, float medianValue_f,
 size_t r1, size_t r2, size_t r3, size_t r4, size_t s1, size_t s2, size_t s3, size_t s4, size_t e1, size_t e2, size_t e3, size_t e4);
 
+
+unsigned int optimize_intervals_float_2D_with_freq_and_dense_pos(float *oriData, size_t r1, size_t r2, double realPrecision, float * dense_pos, float * max_freq, float * mean_freq);
+unsigned int optimize_intervals_float_3D_with_freq_and_dense_pos(float *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, float * dense_pos, float * max_freq, float * mean_freq);
+
+unsigned char * SZ_compress_float_2D_MDQ_nonblocked_with_blocked_regression(float *oriData, size_t r1, size_t r2, double realPrecision, size_t * comp_size);
+unsigned char * SZ_compress_float_3D_MDQ_nonblocked_with_blocked_regression(float *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, size_t * comp_size);
+unsigned char * SZ_compress_float_3D_MDQ_random_access_with_blocked_regression(float *oriData, size_t r1, size_t r2, size_t r3, double realPrecision, size_t * comp_size);
+
 #ifdef __cplusplus
 }
 #endif

thirdparty/SZ/sz/include/sz_float_pwr.h

@@ -44,6 +44,10 @@ double absErrBound, double relBoundRatio, double pwrErrRatio, float valueRangeSi
 void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwrgroup(unsigned char** newByteData, float *oriData,
 size_t dataLength, double absErrBound, double relBoundRatio, double pwrErrRatio, float valueRangeSize, float medianValue_f, size_t *outSize);
 
+void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char** newByteData, float *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, float min, float max);
+void SZ_compress_args_float_NoCkRngeNoGzip_2D_pwr_pre_log(unsigned char** newByteData, float *oriData, double pwrErrRatio, size_t r1, size_t r2, size_t *outSize, float min, float max);
+void SZ_compress_args_float_NoCkRngeNoGzip_3D_pwr_pre_log(unsigned char** newByteData, float *oriData, double pwrErrRatio, size_t r1, size_t r2, size_t r3, size_t *outSize, float min, float max);
+
 #ifdef __cplusplus
 }
 #endif

thirdparty/SZ/sz/include/szd_double.h

@@ -24,6 +24,8 @@ void getSnapshotData_double_1D(double** data, size_t dataSeriesLength, TightData
 void getSnapshotData_double_2D(double** data, size_t r1, size_t r2, TightDataPointStorageD* tdps, int errBoundMode);
 void getSnapshotData_double_3D(double** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageD* tdps, int errBoundMode);
 void getSnapshotData_double_4D(double** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageD* tdps, int errBoundMode);
+void decompressDataSeries_double_2D_nonblocked_with_blocked_regression(double** data, size_t r1, size_t r2, unsigned char* comp_data);
+void decompressDataSeries_double_3D_nonblocked_with_blocked_regression(double** data, size_t r1, size_t r2, size_t r3, unsigned char* comp_data);
 
 int SZ_decompress_args_double(double** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, size_t cmpSize);
 

thirdparty/SZ/sz/include/szd_double_pwr.h

@@ -21,6 +21,10 @@ double* extractRealPrecision_3D_double(size_t R1, size_t R2, size_t R3, int bloc
 void decompressDataSeries_double_3D_pwr(double** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageD* tdps);
 
 void decompressDataSeries_double_1D_pwrgroup(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps);
+void decompressDataSeries_double_1D_pwr_pre_log(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps);
+void decompressDataSeries_double_2D_pwr_pre_log(double** data, size_t r1, size_t r2, TightDataPointStorageD* tdps);
+void decompressDataSeries_double_3D_pwr_pre_log(double** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageD* tdps);
+
 #ifdef __cplusplus
 }
 #endif

thirdparty/SZ/sz/include/szd_float.h

@@ -31,6 +31,11 @@ size_t decompressDataSeries_float_2D_RA_block(float * data, float mean, size_t d
 int SZ_decompress_args_float(float** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, size_t cmpSize);
 
 size_t decompressDataSeries_float_3D_RA_block(float * data, float mean, size_t dim_0, size_t dim_1, size_t dim_2, size_t block_dim_0, size_t block_dim_1, size_t block_dim_2, double realPrecision, int * type, float * unpredictable_data);
+
+void decompressDataSeries_float_2D_nonblocked_with_blocked_regression(float** data, size_t r1, size_t r2, unsigned char* comp_data);
+void decompressDataSeries_float_3D_nonblocked_with_blocked_regression(float** data, size_t r1, size_t r2, size_t r3, unsigned char* comp_data);
+void decompressDataSeries_float_3D_random_access_with_blocked_regression(float** data, size_t r1, size_t r2, size_t r3, unsigned char* comp_data);
+
 #ifdef __cplusplus
 }
 #endif

thirdparty/SZ/sz/include/szd_float_pwr.h

@@ -22,6 +22,9 @@ void decompressDataSeries_float_3D_pwr(float** data, size_t r1, size_t r2, size_
 
 char* decompressGroupIDArray(unsigned char* bytes, size_t dataLength);
 void decompressDataSeries_float_1D_pwrgroup(float** data, size_t dataSeriesLength, TightDataPointStorageF* tdps);
+void decompressDataSeries_float_1D_pwr_pre_log(float** data, size_t dataSeriesLength, TightDataPointStorageF* tdps);
+void decompressDataSeries_float_2D_pwr_pre_log(float** data, size_t r1, size_t r2, TightDataPointStorageF* tdps);
+void decompressDataSeries_float_3D_pwr_pre_log(float** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageF* tdps);
 
 #ifdef __cplusplus
 }

thirdparty/SZ/sz/include/utility.h

+/**
+ *  @file utility.h
+ *  @author Sheng Di, Sihuan Li
+ *  @date July, 2018
+ *  @brief Header file for the utility.c.
+ *  (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
+ *      See COPYRIGHT in top-level directory.
+ */
+
+#ifndef _UTILITY_H
+#define _UTILITY_H
+
+#include "sz.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+//sihuan added: use a assistant struct to do sorting and swap that are easy to implement: should
+//consider optimizing the performance later.
+typedef struct sort_ast_particle{
+	int64_t id;
+	float var[6];
+} sort_ast_particle;
+
+int compare_struct(const void* obj1, const void* obj2);//sihuan added: the compare function in the qsort parameter for 2 structures
+void reorder_vars(SZ_VarSet* vset);//sihuan added: reorder the variables increasingly by their index
+size_t intersectAndsort(int64_t* preIndex, size_t preLen, SZ_VarSet* curVar, size_t dataLen, unsigned char* bitarray);
+//sihuan added: find intersection and keep new var sorted by id
+void write_reordered_tofile(SZ_VarSet* curVar, size_t dataLen);
+//sihuan added: write the reordered input to files for further decompression validation
+float calculate_delta_t(size_t size);//sihuan added
+
+int is_lossless_compressed_data(unsigned char* compressedBytes, size_t cmpSize);
+unsigned long sz_lossless_compress(int losslessCompressor, int level, unsigned char* data, unsigned long dataLength, unsigned char** compressBytes);
+unsigned long sz_lossless_decompress(int losslessCompressor, unsigned char* compressBytes, unsigned long cmpSize, unsigned char** oriData, unsigned long targetOriSize);
+unsigned long sz_lossless_decompress65536bytes(int losslessCompressor, unsigned char* compressBytes, unsigned long cmpSize, unsigned char** oriData);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* ----- #ifndef _UTILITY_H  ----- */

thirdparty/SZ/sz/src/ByteToolkit.c

@@ -430,7 +430,7 @@ long bytesToLong(unsigned char* bytes)
 }
 
 //the byte to input is in the big-endian format
-float bytesToFloat(unsigned char* bytes)
+inline float bytesToFloat(unsigned char* bytes)
 {
 	lfloat buf;
 	memcpy(buf.byte, bytes, 4);
@@ -439,7 +439,7 @@ float bytesToFloat(unsigned char* bytes)
 	return buf.value;
 }
 
-void floatToBytes(unsigned char *b, float num)
+inline void floatToBytes(unsigned char *b, float num)
 {
 	lfloat buf;
 	buf.value = num;
@@ -449,7 +449,7 @@ void floatToBytes(unsigned char *b, float num)
 }
 
 //the byte to input is in the big-endian format
-double bytesToDouble(unsigned char* bytes)
+inline double bytesToDouble(unsigned char* bytes)
 {
 	ldouble buf;
 	memcpy(buf.byte, bytes, 8);
@@ -458,7 +458,7 @@ double bytesToDouble(unsigned char* bytes)
 	return buf.value;
 }
 
-void doubleToBytes(unsigned char *b, double num)
+inline void doubleToBytes(unsigned char *b, double num)
 {
 	ldouble buf;
 	buf.value = num;
@@ -507,7 +507,7 @@ int extractBytes(unsigned char* byteArray, size_t k, int validLength)
 	return result;
 }
 
-int getMaskRightCode(int m) {
+inline int getMaskRightCode(int m) {
 	switch (m) {
 	case 1:
 		return 0x01;
@@ -530,16 +530,16 @@ int getMaskRightCode(int m) {
 	}
 }
 
-int getLeftMovingCode(int kMod8)
+inline int getLeftMovingCode(int kMod8)
 {
 	return getMaskRightCode(8 - kMod8);
 }
 
-int getRightMovingSteps(int kMod8, int resiBitLength) {
+inline int getRightMovingSteps(int kMod8, int resiBitLength) {
 	return 8 - kMod8 - resiBitLength;
 }
 
-int getRightMovingCode(int kMod8, int resiBitLength)
+inline int getRightMovingCode(int kMod8, int resiBitLength)
 {
 	int rightMovingSteps = 8 - kMod8 - resiBitLength;
 	if(rightMovingSteps < 0)
@@ -814,7 +814,7 @@ void convertULongArrayToBytes(uint64_t* states, size_t stateLength, unsigned cha
 }
 
 
-size_t bytesToSize(unsigned char* bytes)
+inline size_t bytesToSize(unsigned char* bytes)
 {
 	size_t result = 0;
 	if(exe_params->SZ_SIZE_TYPE==4)	
@@ -824,7 +824,7 @@ size_t bytesToSize(unsigned char* bytes)
 	return result;
 }
 
-void sizeToBytes(unsigned char* outBytes, size_t size)
+inline void sizeToBytes(unsigned char* outBytes, size_t size)
 {
 	if(exe_params->SZ_SIZE_TYPE==4)
 		intToBytes_bigEndian(outBytes, size);//4

thirdparty/SZ/sz/src/DynamicDoubleArray.c

@@ -21,7 +21,7 @@ void new_DDA(DynamicDoubleArray **dda, size_t cap) {
 
 void convertDDAtoDoubles(DynamicDoubleArray *dba, double **data)
 {
-	int size = dba->size;
+	size_t size = dba->size;
 	if(size>0)
 		*data = (double*)malloc(size * sizeof(double));
 	else