utility.c 6.71 KB
Newer Older
Hal Finkel's avatar
Hal Finkel committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
/**
 *  @file utility.c
 *  @author Sheng Di, Sihuan Li
 *  @date Aug, 2018
 *  @brief 
 *  (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include "utility.h"
#include "sz.h"
#include "callZlib.h"
#include "zstd.h"

int compare_struct(const void* obj1, const void* obj2){
	struct sort_ast_particle * srt1 = (struct sort_ast_particle*)obj1;
	struct sort_ast_particle * srt2 = (struct sort_ast_particle*)obj2;
	return srt1->id - srt2->id;
}

void reorder_vars(SZ_VarSet* vset){
	SZ_Variable* v[7];
	SZ_Variable* v_tmp;
	int i, j;
	//v[0]
	for (v_tmp = vset->header->next, i = 0; i < 7; i++){
		v[i] = v_tmp;
		v_tmp = v_tmp->next;
	}
	//printf("here");
	size_t dataLen = computeDataLength(v[0]->r5, v[0]->r4, v[0]->r3, v[0]->r2, v[0]->r1);
	//sihuan debug
	//printf("the data length is (in sorting): %u", dataLen);
	struct sort_ast_particle* particle = (struct sort_ast_particle*) malloc(sizeof(struct sort_ast_particle)*dataLen);

	for (i = 0; i < dataLen; i++){
		particle[i].id = ((int64_t*)v[6]->data)[i];
	//	printf("%llu ", particle[i].id);
		for (j = 0; j < 6; j++)
			particle[i].var[j] = ((float*)v[j]->data)[i];
	}

	//sihuan debug
	#if 0
	printf("index before sorting: \n");
	for (i = 0; i < 5; i++){
		printf("%llu  ", particle[i].id);
		printf("%.5f  ", ((float*)v[0]->data)[i]);
	}
	#endif
	//printf("\n");
	//sihuan debug
	//for (i = 0; i < 5; i++)//{
		//for (j = 0; j < 6; j++)
		//	printf("%.5f  ", particle[i].var[j]);
	//		printf("%llu  ", particle[i].id );
	///}
	//printf("\n\n");


	qsort(particle, dataLen, sizeof(struct sort_ast_particle), compare_struct);
	for (i = 0; i < dataLen; i++){
		((int64_t*)v[6]->data)[i] = particle[i].id;
		for (j = 0; j < 6; j++)
			((float*)v[j]->data)[i] = particle[i].var[j];
	}
	free(particle);

	//sihuan debug
	#if 0
	for (i = 0; i < 5; i++){
		printf("%llu  ", particle[i].id);
		printf("%.5f  ", ((float*)v[0]->data)[i]);
	}
	printf("\n");
	#endif
}

size_t intersectAndsort(int64_t* preIndex, size_t preLen, SZ_VarSet* curVar, size_t dataLen, unsigned char* bitarray){
	size_t i, j, k, m, cnt;
	i = j = k = m = cnt = 0;
	SZ_Variable* v[7];
	SZ_Variable* v_tmp;
	//v[0]
	for (v_tmp = curVar->header->next, i = 0; i < 7; i++){
		v[i] = v_tmp;
		v_tmp = v_tmp->next;
	}
	for (i = 0; i < preLen; i++)
		bitarray[i] = '0';
	i = 0;
	while(i < preLen && j < dataLen){
		if (preIndex[i] == ((int64_t*)v[6]->data)[j]){
			cnt++;
			int64_t tmp;
			tmp = ((int64_t*)v[6]->data)[k];
			((int64_t*)v[6]->data)[k] = ((int64_t*)v[6]->data)[j];
			((int64_t*)v[6]->data)[j] = tmp;
			float data_tmp;
			for (m = 0; m < 6; m++){
				data_tmp = ((float*)v[m]->data)[k];
				((float*)v[m]->data)[k] = ((float*)v[m]->data)[j];
				((float*)v[m]->data)[j] = data_tmp;
			}
			k++; i++; j++;
		}
		else if (preIndex[i] < ((int64_t*)v[6]->data)[j]){
			bitarray[i] = '1';
			i++;
		}
		else j++;
	}
	printf("intersect count is: %zu, i j k pre curlen is: %zu, %zu, %zu, %zu, %zu\n\n", cnt, i, j, k, preLen, dataLen);
	return cnt;
}

void write_reordered_tofile(SZ_VarSet* curVar, size_t dataLen){
	int var_index; //0 for x, 1 for y...,3 for vx...5 for vz
	int i;
	char outputfile_name[256];
	SZ_Variable* v[7]; SZ_Variable* v_tmp;
	for (v_tmp = curVar->header->next, i = 0; i < 6; i++){
		v[i] = v_tmp;
		v_tmp = v_tmp->next;
	}
	for (var_index = 0; var_index < 6; var_index++){
		sprintf(outputfile_name, "reordered_input_%d_%d.in", sz_tsc->currentStep, var_index);
		int status_tmp;
		writeFloatData_inBytes((float*)v[var_index]->data, dataLen, outputfile_name, &status_tmp);
	}
}

float calculate_delta_t(size_t size){
	SZ_Variable* v_tmp = sz_varset->header->next;
	while(strcmp(v_tmp->varName, "x")) v_tmp = v_tmp->next;
	float* x1 = (float*) v_tmp->data;
	float* x0 = (float*) v_tmp->multisteps->hist_data;
	while(strcmp(v_tmp->varName, "vx")) v_tmp = v_tmp->next;
	float* vx0 = (float*) v_tmp->multisteps->hist_data;
	int i, j;
	double denom = 0.0;
	double div = 0.0;
	for (i = 0, j = 0; i < size; i++, j++){
		while(sz_tsc->bit_array[j] == '1') j++;
		denom += vx0[j] * (x1[i] - x0[j]);
		div   += vx0[j] * vx0[j];
	}
	printf("the calculated delta_t is: %.10f\n", denom/div);
	return denom/div;
}

int is_lossless_compressed_data(unsigned char* compressedBytes, size_t cmpSize)
{
158
#if ZSTD_VERSION_NUMBER >= 10300
Hal Finkel's avatar
Hal Finkel committed
159
160
161
	int frameContentSize = ZSTD_getFrameContentSize(compressedBytes, cmpSize);
	if(frameContentSize != ZSTD_CONTENTSIZE_ERROR)
		return ZSTD_COMPRESSOR;
162
163
164
165
166
#else
	int frameContentSize = ZSTD_getDecompressedSize(compressedBytes, cmpSize);
	if(frameContentSize == 0)
		return ZSTD_COMPRESSOR;
#endif
Hal Finkel's avatar
Hal Finkel committed
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
	
	int flag = isZlibFormat(compressedBytes[0], compressedBytes[1]);
	if(flag)
		return GZIP_COMPRESSOR;

	return -1; //fast mode (without GZIP or ZSTD)
}

unsigned long sz_lossless_compress(int losslessCompressor, int level, unsigned char* data, unsigned long dataLength, unsigned char** compressBytes)
{
	unsigned long outSize = 0; 
	size_t estimatedCompressedSize = 0;
	switch(losslessCompressor)
	{
	case GZIP_COMPRESSOR:
		outSize = zlib_compress5(data, dataLength, compressBytes, level);
		break;
	case ZSTD_COMPRESSOR:
		estimatedCompressedSize = dataLength*1.2;
		*compressBytes = (unsigned char*)malloc(estimatedCompressedSize);
		outSize = ZSTD_compress(*compressBytes, estimatedCompressedSize, data, dataLength, level); //default setting of level is 3
		break;
	default:
		printf("Error: Unrecognized lossless compressor in sz_lossless_compress()\n");
	}
	return outSize;
}

unsigned long sz_lossless_decompress(int losslessCompressor, unsigned char* compressBytes, unsigned long cmpSize, unsigned char** oriData, unsigned long targetOriSize)
{
	unsigned long outSize = 0;
	switch(losslessCompressor)
	{
	case GZIP_COMPRESSOR:
		outSize = zlib_uncompress5(compressBytes, cmpSize, oriData, targetOriSize);
		break;
	case ZSTD_COMPRESSOR:
		*oriData = (unsigned char*)malloc(targetOriSize);
		ZSTD_decompress(*oriData, targetOriSize, compressBytes, cmpSize);
		outSize = targetOriSize;
		break;
	default:
		printf("Error: Unrecognized lossless compressor in sz_lossless_decompress()\n");
	}
	return outSize;
}

unsigned long sz_lossless_decompress65536bytes(int losslessCompressor, unsigned char* compressBytes, unsigned long cmpSize, unsigned char** oriData)
{
	unsigned long outSize = 0;
	switch(losslessCompressor)
	{
	case GZIP_COMPRESSOR:
		outSize = zlib_uncompress65536bytes(compressBytes, cmpSize, oriData);
		break;
	case ZSTD_COMPRESSOR:
		*oriData = (unsigned char*)malloc(65536);
		memset(*oriData, 0, 65536);
		ZSTD_decompress(*oriData, 65536, compressBytes, cmpSize);	//the first 32768 bytes should be exact the same.
		outSize = 65536;
		break;
	default:
		printf("Error: Unrecognized lossless compressor\n");
	}
	return outSize;
}