Commit 03ba2823 authored by Hal Finkel's avatar Hal Finkel
Browse files

add utility.c

parent e6aa0ebb
......@@ -170,6 +170,7 @@ BLOSC_O := \
thirdparty/SZ/sz/src/szd_int16.o \
thirdparty/SZ/sz/src/szd_int32.o \
thirdparty/SZ/sz/src/szd_int64.o \
thirdparty/SZ/sz/src/utility.o \
thirdparty/SZ/sz/src/sz.o \
thirdparty/SZ/sz/src/sz_float_pwr.o \
thirdparty/SZ/sz/src/sz_double_pwr.o \
......
/**
* @file utility.c
* @author Sheng Di, Sihuan Li
* @date Aug, 2018
* @brief
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include "utility.h"
#include "sz.h"
#include "callZlib.h"
#include "zstd.h"
int compare_struct(const void* obj1, const void* obj2){
struct sort_ast_particle * srt1 = (struct sort_ast_particle*)obj1;
struct sort_ast_particle * srt2 = (struct sort_ast_particle*)obj2;
return srt1->id - srt2->id;
}
void reorder_vars(SZ_VarSet* vset){
SZ_Variable* v[7];
SZ_Variable* v_tmp;
int i, j;
//v[0]
for (v_tmp = vset->header->next, i = 0; i < 7; i++){
v[i] = v_tmp;
v_tmp = v_tmp->next;
}
//printf("here");
size_t dataLen = computeDataLength(v[0]->r5, v[0]->r4, v[0]->r3, v[0]->r2, v[0]->r1);
//sihuan debug
//printf("the data length is (in sorting): %u", dataLen);
struct sort_ast_particle* particle = (struct sort_ast_particle*) malloc(sizeof(struct sort_ast_particle)*dataLen);
for (i = 0; i < dataLen; i++){
particle[i].id = ((int64_t*)v[6]->data)[i];
// printf("%llu ", particle[i].id);
for (j = 0; j < 6; j++)
particle[i].var[j] = ((float*)v[j]->data)[i];
}
//sihuan debug
#if 0
printf("index before sorting: \n");
for (i = 0; i < 5; i++){
printf("%llu ", particle[i].id);
printf("%.5f ", ((float*)v[0]->data)[i]);
}
#endif
//printf("\n");
//sihuan debug
//for (i = 0; i < 5; i++)//{
//for (j = 0; j < 6; j++)
// printf("%.5f ", particle[i].var[j]);
// printf("%llu ", particle[i].id );
///}
//printf("\n\n");
qsort(particle, dataLen, sizeof(struct sort_ast_particle), compare_struct);
for (i = 0; i < dataLen; i++){
((int64_t*)v[6]->data)[i] = particle[i].id;
for (j = 0; j < 6; j++)
((float*)v[j]->data)[i] = particle[i].var[j];
}
free(particle);
//sihuan debug
#if 0
for (i = 0; i < 5; i++){
printf("%llu ", particle[i].id);
printf("%.5f ", ((float*)v[0]->data)[i]);
}
printf("\n");
#endif
}
size_t intersectAndsort(int64_t* preIndex, size_t preLen, SZ_VarSet* curVar, size_t dataLen, unsigned char* bitarray){
size_t i, j, k, m, cnt;
i = j = k = m = cnt = 0;
SZ_Variable* v[7];
SZ_Variable* v_tmp;
//v[0]
for (v_tmp = curVar->header->next, i = 0; i < 7; i++){
v[i] = v_tmp;
v_tmp = v_tmp->next;
}
for (i = 0; i < preLen; i++)
bitarray[i] = '0';
i = 0;
while(i < preLen && j < dataLen){
if (preIndex[i] == ((int64_t*)v[6]->data)[j]){
cnt++;
int64_t tmp;
tmp = ((int64_t*)v[6]->data)[k];
((int64_t*)v[6]->data)[k] = ((int64_t*)v[6]->data)[j];
((int64_t*)v[6]->data)[j] = tmp;
float data_tmp;
for (m = 0; m < 6; m++){
data_tmp = ((float*)v[m]->data)[k];
((float*)v[m]->data)[k] = ((float*)v[m]->data)[j];
((float*)v[m]->data)[j] = data_tmp;
}
k++; i++; j++;
}
else if (preIndex[i] < ((int64_t*)v[6]->data)[j]){
bitarray[i] = '1';
i++;
}
else j++;
}
printf("intersect count is: %zu, i j k pre curlen is: %zu, %zu, %zu, %zu, %zu\n\n", cnt, i, j, k, preLen, dataLen);
return cnt;
}
void write_reordered_tofile(SZ_VarSet* curVar, size_t dataLen){
int var_index; //0 for x, 1 for y...,3 for vx...5 for vz
int i;
char outputfile_name[256];
SZ_Variable* v[7]; SZ_Variable* v_tmp;
for (v_tmp = curVar->header->next, i = 0; i < 6; i++){
v[i] = v_tmp;
v_tmp = v_tmp->next;
}
for (var_index = 0; var_index < 6; var_index++){
sprintf(outputfile_name, "reordered_input_%d_%d.in", sz_tsc->currentStep, var_index);
int status_tmp;
writeFloatData_inBytes((float*)v[var_index]->data, dataLen, outputfile_name, &status_tmp);
}
}
float calculate_delta_t(size_t size){
SZ_Variable* v_tmp = sz_varset->header->next;
while(strcmp(v_tmp->varName, "x")) v_tmp = v_tmp->next;
float* x1 = (float*) v_tmp->data;
float* x0 = (float*) v_tmp->multisteps->hist_data;
while(strcmp(v_tmp->varName, "vx")) v_tmp = v_tmp->next;
float* vx0 = (float*) v_tmp->multisteps->hist_data;
int i, j;
double denom = 0.0;
double div = 0.0;
for (i = 0, j = 0; i < size; i++, j++){
while(sz_tsc->bit_array[j] == '1') j++;
denom += vx0[j] * (x1[i] - x0[j]);
div += vx0[j] * vx0[j];
}
printf("the calculated delta_t is: %.10f\n", denom/div);
return denom/div;
}
int is_lossless_compressed_data(unsigned char* compressedBytes, size_t cmpSize)
{
int frameContentSize = ZSTD_getFrameContentSize(compressedBytes, cmpSize);
if(frameContentSize != ZSTD_CONTENTSIZE_ERROR)
return ZSTD_COMPRESSOR;
int flag = isZlibFormat(compressedBytes[0], compressedBytes[1]);
if(flag)
return GZIP_COMPRESSOR;
return -1; //fast mode (without GZIP or ZSTD)
}
unsigned long sz_lossless_compress(int losslessCompressor, int level, unsigned char* data, unsigned long dataLength, unsigned char** compressBytes)
{
unsigned long outSize = 0;
size_t estimatedCompressedSize = 0;
switch(losslessCompressor)
{
case GZIP_COMPRESSOR:
outSize = zlib_compress5(data, dataLength, compressBytes, level);
break;
case ZSTD_COMPRESSOR:
estimatedCompressedSize = dataLength*1.2;
*compressBytes = (unsigned char*)malloc(estimatedCompressedSize);
outSize = ZSTD_compress(*compressBytes, estimatedCompressedSize, data, dataLength, level); //default setting of level is 3
break;
default:
printf("Error: Unrecognized lossless compressor in sz_lossless_compress()\n");
}
return outSize;
}
unsigned long sz_lossless_decompress(int losslessCompressor, unsigned char* compressBytes, unsigned long cmpSize, unsigned char** oriData, unsigned long targetOriSize)
{
unsigned long outSize = 0;
switch(losslessCompressor)
{
case GZIP_COMPRESSOR:
outSize = zlib_uncompress5(compressBytes, cmpSize, oriData, targetOriSize);
break;
case ZSTD_COMPRESSOR:
*oriData = (unsigned char*)malloc(targetOriSize);
ZSTD_decompress(*oriData, targetOriSize, compressBytes, cmpSize);
outSize = targetOriSize;
break;
default:
printf("Error: Unrecognized lossless compressor in sz_lossless_decompress()\n");
}
return outSize;
}
unsigned long sz_lossless_decompress65536bytes(int losslessCompressor, unsigned char* compressBytes, unsigned long cmpSize, unsigned char** oriData)
{
unsigned long outSize = 0;
switch(losslessCompressor)
{
case GZIP_COMPRESSOR:
outSize = zlib_uncompress65536bytes(compressBytes, cmpSize, oriData);
break;
case ZSTD_COMPRESSOR:
*oriData = (unsigned char*)malloc(65536);
memset(*oriData, 0, 65536);
ZSTD_decompress(*oriData, 65536, compressBytes, cmpSize); //the first 32768 bytes should be exact the same.
outSize = 65536;
break;
default:
printf("Error: Unrecognized lossless compressor\n");
}
return outSize;
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment