Commit bd84570e authored by Hal Finkel's avatar Hal Finkel
Browse files

Add simple python interface from Dan

parent 95f86a3f
......@@ -59,7 +59,7 @@ BLOSC_CPPFLAGS := \
-Ithirdparty/blosc/internal-complibs/zstd-0.7.4/dictBuilder \
-Ithirdparty/blosc/internal-complibs/zstd-0.7.4/decompress
BASE_CPPFLAGS := $(BLOSC_CPPFLAGS) -D__STDC_CONSTANT_MACROS
BASE_CPPFLAGS := $(BLOSC_CPPFLAGS) -I. -D__STDC_CONSTANT_MACROS
FEDIR = frontend
FE_CFLAGS := -g -fPIC -O3 -fopenmp
......@@ -141,7 +141,14 @@ FE_SHARED := -bundle
else
FE_SHARED := -shared
endif
$(FEDIR)/GenericIOSQLite.so: $(FEDIR)/GenericIOSQLite.o $(FEDIR)/GenericIO.o
$(FEDIR)/libpygio.so: $(FEDIR)/GenericIO.o $(FEDIR)/python/lib/gio.o $(FE_BLOSC_O)
$(CXX) $(FE_CFLAGS) $(FE_SHARED) -o $@ $^
$(FEDIR)/gio.py: python/gio.py
cp -f $< $@
$(FEDIR)/GenericIOSQLite.so: $(FEDIR)/GenericIOSQLite.o $(FEDIR)/GenericIO.o $(FE_BLOSC_O)
$(CXX) $(FE_CFLAGS) $(FE_SHARED) -o $@ $^
SQLITE_CPPFLAGS := -DSQLITE_ENABLE_COLUMN_METADATA=1 -DSQLITE_DISABLE_DIRSYNC=1 -DSQLITE_ENABLE_FTS3=3 -DSQLITE_ENABLE_RTREE=1 -DSQLITE_ENABLE_UNLOCK_NOTIFY=1 -DSQLITE_ENABLE_LOAD_EXTENSION=1 -DHAVE_READLINE=1
......@@ -183,7 +190,7 @@ $(MPIDIR)/GenericIOBenchmarkWrite: $(MPIDIR)/GenericIOBenchmarkWrite.o $(MPIDIR)
$(MPIDIR)/GenericIORewrite: $(MPIDIR)/GenericIORewrite.o $(MPIDIR)/GenericIO.o $(MPI_BLOSC_O)
$(MPICXX) $(MPI_CFLAGS) -o $@ $^
frontend-progs: $(FEDIR)/GenericIOPrint $(FEDIR)/GenericIOVerify
frontend-progs: $(FEDIR)/GenericIOPrint $(FEDIR)/GenericIOVerify $(FEDIR)/libpygio.so $(FEDIR)/gio.py
fe-progs: frontend-progs
mpi-progs: $(MPIDIR)/GenericIOPrint $(MPIDIR)/GenericIOVerify $(MPIDIR)/GenericIOBenchmarkRead $(MPIDIR)/GenericIOBenchmarkWrite $(MPIDIR)/GenericIORewrite
......
# Copyright (C) 2015, UChicago Argonne, LLC
# All Rights Reserved
#
# Generic IO (ANL-15-066)
# Hal Finkel, Argonne National Laboratory
#
# OPEN SOURCE LICENSE
#
# Under the terms of Contract No. DE-AC02-06CH11357 with UChicago Argonne,
# LLC, the U.S. Government retains certain rights in this software.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# 3. Neither the names of UChicago Argonne, LLC or the Department of Energy
# nor the names of its contributors may be used to endorse or promote
# products derived from this software without specific prior written
# permission.
#
# *****************************************************************************
#
# DISCLAIMER
# THE SOFTWARE IS SUPPLIED “AS IS” WITHOUT WARRANTY OF ANY KIND. NEITHER THE
# UNTED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR
# UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY,
# EXPRESS OR IMPLIED, OR ASSUMES ANY LEGAL LIABILITY OR RESPONSIBILITY FOR THE
# ACCURACY, COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, DATA, APPARATUS,
# PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD NOT INFRINGE
# PRIVATELY OWNED RIGHTS.
#
# *****************************************************************************
import numpy as np
import ctypes as ct
import os
#Define where the library is and load it
_path = os.path.dirname('__file__')
libpygio = ct.CDLL(os.path.abspath('libpygio.so'))
#we need to define the return type ("restype") and
#the argument types
libpygio.get_elem_num.restype=ct.c_int64
libpygio.get_elem_num.argtypes=[ct.c_char_p]
libpygio.get_variable_type.restype=ct.c_int
libpygio.get_variable_type.argtypes=[ct.c_char_p,ct.c_char_p]
libpygio.read_gio_int32.restype=None
libpygio.read_gio_int32.argtypes=[ct.c_char_p,ct.c_char_p,ct.POINTER(ct.c_int)]
libpygio.read_gio_int64.restype=None
libpygio.read_gio_int64.argtypes=[ct.c_char_p,ct.c_char_p,ct.POINTER(ct.c_int64)]
libpygio.read_gio_float.restype=None
libpygio.read_gio_float.argtypes=[ct.c_char_p,ct.c_char_p,ct.POINTER(ct.c_float)]
libpygio.read_gio_double.restype=None
libpygio.read_gio_double.argtypes=[ct.c_char_p,ct.c_char_p,ct.POINTER(ct.c_double)]
libpygio.inspect_gio.restype=None
libpygio.inspect_gio.argtypes=[ct.c_char_p]
def gio_read(file_name,var_name):
var_size = libpygio.get_elem_num(file_name)
var_type = libpygio.get_variable_type(file_name,var_name)
if(var_type==10):
print "Variable not found"
return
elif(var_type==9):
print "variable type not known (not int32/int64/float/double)"
elif(var_type==0):
#float
result = np.ndarray(var_size,dtype=np.float32)
libpygio.read_gio_float(file_name,var_name,result.ctypes.data_as(ct.POINTER(ct.c_float)))
return result
elif(var_type==1):
#double
result = np.ndarray(var_size,dtype=np.float64)
libpygio.read_gio_double(file_name,var_name,result.ctypes.data_as(ct.POINTER(ct.c_double)))
return result
elif(var_type==2):
#int32
result = np.ndarray(var_size,dtype=np.int32)
libpygio.read_gio_int32(file_name,var_name,result.ctypes.data_as(ct.POINTER(ct.c_int32)))
return result
elif(var_type==3):
#int64
result = np.ndarray(var_size,dtype=np.int64)
libpygio.read_gio_int64(file_name,var_name,result.ctypes.data_as(ct.POINTER(ct.c_int64)))
return result
def gio_inspect(file_name):
libpygio.inspect_gio(file_name)
/*
* Copyright (C) 2015, UChicago Argonne, LLC
* All Rights Reserved
*
* Generic IO (ANL-15-066)
* Hal Finkel, Argonne National Laboratory
*
* OPEN SOURCE LICENSE
*
* Under the terms of Contract No. DE-AC02-06CH11357 with UChicago Argonne,
* LLC, the U.S. Government retains certain rights in this software.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* 3. Neither the names of UChicago Argonne, LLC or the Department of Energy
* nor the names of its contributors may be used to endorse or promote
* products derived from this software without specific prior written
* permission.
*
* *****************************************************************************
*
* DISCLAIMER
* THE SOFTWARE IS SUPPLIED “AS IS” WITHOUT WARRANTY OF ANY KIND. NEITHER THE
* UNTED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR
* UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY,
* EXPRESS OR IMPLIED, OR ASSUMES ANY LEGAL LIABILITY OR RESPONSIBILITY FOR THE
* ACCURACY, COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, DATA, APPARATUS,
* PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD NOT INFRINGE
* PRIVATELY OWNED RIGHTS.
*
* *****************************************************************************
*/
#include "gio.h"
#include <iostream>
void read_gio_float(char* file_name, char* var_name, float* data){
read_gio<float>(file_name,var_name,data);
}
void read_gio_double(char* file_name, char* var_name, double* data){
read_gio<double>(file_name,var_name,data);
}
void read_gio_int32(char* file_name, char* var_name, int* data){
read_gio<int>(file_name,var_name,data);
}
void read_gio_int64(char* file_name, char* var_name, int64_t* data){
read_gio<int64_t>(file_name,var_name,data);
}
int64_t get_elem_num(char* file_name){
gio::GenericIO reader(file_name);
reader.openAndReadHeader(gio::GenericIO::MismatchAllowed);
int num_ranks = reader.readNRanks();
uint64_t size = 0;
for(int i =0;i<num_ranks;++i)
size +=reader.readNumElems(i);
reader.close();
return size;
}
var_type get_variable_type(char* file_name,char* var_name){
gio::GenericIO reader(file_name);
std::vector<gio::GenericIO::VariableInfo> VI;
reader.openAndReadHeader(gio::GenericIO::MismatchAllowed);
reader.getVariableInfo(VI);
int num =VI.size();
for(int i =0;i<num;++i){
gio::GenericIO::VariableInfo vinfo = VI[i];
if(vinfo.Name == var_name){
if(vinfo.IsFloat && vinfo.Size == 4)
return float_type;
else if(vinfo.IsFloat && vinfo.Size == 8)
return double_type;
else if(!vinfo.IsFloat && vinfo.Size == 4)
return int32_type;
else if(!vinfo.IsFloat && vinfo.Size == 8)
return int64_type;
else
return type_not_found;
}
}
return var_not_found;
}
extern "C" void inspect_gio(char* file_name){
int64_t size = get_elem_num(file_name);
gio::GenericIO reader(file_name);
std::vector<gio::GenericIO::VariableInfo> VI;
reader.openAndReadHeader(gio::GenericIO::MismatchAllowed);
reader.getVariableInfo(VI);
std::cout<<"Number of Elements: "<<size<<std::endl;
int num =VI.size();
std::cout<<"[data type] Variable name"<<std::endl;
std::cout<<"---------------------------------------------"<<std::endl;
for(int i =0;i<num;++i){
gio::GenericIO::VariableInfo vinfo = VI[i];
if(vinfo.IsFloat)
std::cout<<"[f";
else
std::cout<<"[i";
std::cout<<" "<<vinfo.Size*8<<"] ";
std::cout<<vinfo.Name<<std::endl;
}
std::cout<<"\n(i=integer,f=floating point, number bits size)"<<std::endl;
}
/*
* Copyright (C) 2015, UChicago Argonne, LLC
* All Rights Reserved
*
* Generic IO (ANL-15-066)
* Hal Finkel, Argonne National Laboratory
*
* OPEN SOURCE LICENSE
*
* Under the terms of Contract No. DE-AC02-06CH11357 with UChicago Argonne,
* LLC, the U.S. Government retains certain rights in this software.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* 3. Neither the names of UChicago Argonne, LLC or the Department of Energy
* nor the names of its contributors may be used to endorse or promote
* products derived from this software without specific prior written
* permission.
*
* *****************************************************************************
*
* DISCLAIMER
* THE SOFTWARE IS SUPPLIED “AS IS” WITHOUT WARRANTY OF ANY KIND. NEITHER THE
* UNTED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR
* UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY,
* EXPRESS OR IMPLIED, OR ASSUMES ANY LEGAL LIABILITY OR RESPONSIBILITY FOR THE
* ACCURACY, COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, DATA, APPARATUS,
* PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD NOT INFRINGE
* PRIVATELY OWNED RIGHTS.
*
* *****************************************************************************
*/
#include <GenericIO.h>
#include <stdint.h>
#include <sstream>
template <class T>
void read_gio(char* file_name, std::string var_name, T*& data){
gio::GenericIO reader(file_name);
reader.openAndReadHeader(gio::GenericIO::MismatchAllowed);
int num_ranks = reader.readNRanks();
uint64_t max_size = 0;
uint64_t rank_size[num_ranks];
for(int i =0;i<num_ranks;++i){
rank_size[i] = reader.readNumElems(i);
if(max_size < rank_size[i])
max_size = rank_size[i];
}
T* rank_data = new T[max_size+reader.requestedExtraSpace()/sizeof(T)];
int64_t offset =0;
reader.addVariable(var_name,rank_data,true);
for(int i=0;i<num_ranks;++i){
reader.readData(i,false);
std::copy(rank_data,rank_data+rank_size[i],data+offset);
offset +=rank_size[i];
}
delete [] rank_data;
reader.close();
}
extern "C" int64_t get_elem_num(char* file_name);
extern "C" void read_gio_float (char* file_name, char* var_name, float* data);
extern "C" void read_gio_double(char* file_name, char* var_name, double* data);
extern "C" void read_gio_int32 (char* file_name, char* var_name, int* data);
extern "C" void read_gio_int64 (char* file_name, char* var_name, int64_t* data);
enum var_type{
float_type=0,
double_type=1,
int32_type=2,
int64_type=3,
type_not_found=9,
var_not_found=10
};
extern "C" var_type get_variable_type(char* file_name,char* var_name);
extern "C" void inspect_gio(char* file_name);
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment