Add simple python interface from Dan

GNUmakefile

@@ -59,7 +59,7 @@ BLOSC_CPPFLAGS := \
 	-Ithirdparty/blosc/internal-complibs/zstd-0.7.4/dictBuilder \
 	-Ithirdparty/blosc/internal-complibs/zstd-0.7.4/decompress
 
-BASE_CPPFLAGS := $(BLOSC_CPPFLAGS) -D__STDC_CONSTANT_MACROS
+BASE_CPPFLAGS := $(BLOSC_CPPFLAGS) -I. -D__STDC_CONSTANT_MACROS
 
 FEDIR = frontend
 FE_CFLAGS := -g -fPIC -O3 -fopenmp
@@ -141,7 +141,14 @@ FE_SHARED := -bundle
 else
 FE_SHARED := -shared
 endif
-$(FEDIR)/GenericIOSQLite.so: $(FEDIR)/GenericIOSQLite.o $(FEDIR)/GenericIO.o
+
+$(FEDIR)/libpygio.so: $(FEDIR)/GenericIO.o $(FEDIR)/python/lib/gio.o $(FE_BLOSC_O)
+	$(CXX) $(FE_CFLAGS) $(FE_SHARED) -o $@ $^
+
+$(FEDIR)/gio.py: python/gio.py
+	cp -f $< $@
+
+$(FEDIR)/GenericIOSQLite.so: $(FEDIR)/GenericIOSQLite.o $(FEDIR)/GenericIO.o $(FE_BLOSC_O)
 	$(CXX) $(FE_CFLAGS) $(FE_SHARED) -o $@ $^
 
 SQLITE_CPPFLAGS := -DSQLITE_ENABLE_COLUMN_METADATA=1 -DSQLITE_DISABLE_DIRSYNC=1 -DSQLITE_ENABLE_FTS3=3 -DSQLITE_ENABLE_RTREE=1 -DSQLITE_ENABLE_UNLOCK_NOTIFY=1 -DSQLITE_ENABLE_LOAD_EXTENSION=1 -DHAVE_READLINE=1
@@ -183,7 +190,7 @@ $(MPIDIR)/GenericIOBenchmarkWrite: $(MPIDIR)/GenericIOBenchmarkWrite.o $(MPIDIR)
 $(MPIDIR)/GenericIORewrite: $(MPIDIR)/GenericIORewrite.o $(MPIDIR)/GenericIO.o $(MPI_BLOSC_O)
 	$(MPICXX) $(MPI_CFLAGS) -o $@ $^ 
 
-frontend-progs: $(FEDIR)/GenericIOPrint $(FEDIR)/GenericIOVerify
+frontend-progs: $(FEDIR)/GenericIOPrint $(FEDIR)/GenericIOVerify $(FEDIR)/libpygio.so $(FEDIR)/gio.py
 fe-progs: frontend-progs
 
 mpi-progs: $(MPIDIR)/GenericIOPrint $(MPIDIR)/GenericIOVerify $(MPIDIR)/GenericIOBenchmarkRead $(MPIDIR)/GenericIOBenchmarkWrite $(MPIDIR)/GenericIORewrite

python/gio.py

+#                    Copyright (C) 2015, UChicago Argonne, LLC
+#                               All Rights Reserved
+# 
+#                               Generic IO (ANL-15-066)
+#                     Hal Finkel, Argonne National Laboratory
+# 
+#                              OPEN SOURCE LICENSE
+# 
+# Under the terms of Contract No. DE-AC02-06CH11357 with UChicago Argonne,
+# LLC, the U.S. Government retains certain rights in this software.
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+#   1. Redistributions of source code must retain the above copyright notice,
+#      this list of conditions and the following disclaimer.
+# 
+#   2. Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in the
+#      documentation and/or other materials provided with the distribution.
+# 
+#   3. Neither the names of UChicago Argonne, LLC or the Department of Energy
+#      nor the names of its contributors may be used to endorse or promote
+#      products derived from this software without specific prior written
+#      permission.
+# 
+# *****************************************************************************
+# 
+#                                  DISCLAIMER
+# THE SOFTWARE IS SUPPLIED “AS IS” WITHOUT WARRANTY OF ANY KIND.  NEITHER THE
+# UNTED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR
+# UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY,
+# EXPRESS OR IMPLIED, OR ASSUMES ANY LEGAL LIABILITY OR RESPONSIBILITY FOR THE
+# ACCURACY, COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, DATA, APPARATUS,
+# PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD NOT INFRINGE
+# PRIVATELY OWNED RIGHTS.
+# 
+# *****************************************************************************
+
+import numpy as np
+import ctypes as ct
+import os
+
+#Define where the library is and load it
+_path = os.path.dirname('__file__')
+libpygio = ct.CDLL(os.path.abspath('libpygio.so'))
+#we need to define the return type ("restype") and
+#the argument types
+libpygio.get_elem_num.restype=ct.c_int64
+libpygio.get_elem_num.argtypes=[ct.c_char_p]
+
+libpygio.get_variable_type.restype=ct.c_int
+libpygio.get_variable_type.argtypes=[ct.c_char_p,ct.c_char_p]
+
+libpygio.read_gio_int32.restype=None
+libpygio.read_gio_int32.argtypes=[ct.c_char_p,ct.c_char_p,ct.POINTER(ct.c_int)]
+
+libpygio.read_gio_int64.restype=None
+libpygio.read_gio_int64.argtypes=[ct.c_char_p,ct.c_char_p,ct.POINTER(ct.c_int64)]
+
+libpygio.read_gio_float.restype=None
+libpygio.read_gio_float.argtypes=[ct.c_char_p,ct.c_char_p,ct.POINTER(ct.c_float)]
+
+libpygio.read_gio_double.restype=None
+libpygio.read_gio_double.argtypes=[ct.c_char_p,ct.c_char_p,ct.POINTER(ct.c_double)]
+
+libpygio.inspect_gio.restype=None
+libpygio.inspect_gio.argtypes=[ct.c_char_p]
+
+def gio_read(file_name,var_name):
+    var_size = libpygio.get_elem_num(file_name)
+    var_type = libpygio.get_variable_type(file_name,var_name)
+    if(var_type==10):
+        print "Variable not found"
+        return
+    elif(var_type==9):
+        print "variable type not known (not int32/int64/float/double)"
+    elif(var_type==0):
+        #float
+        result = np.ndarray(var_size,dtype=np.float32)
+        libpygio.read_gio_float(file_name,var_name,result.ctypes.data_as(ct.POINTER(ct.c_float)))
+        return result
+    elif(var_type==1):
+        #double
+        result = np.ndarray(var_size,dtype=np.float64)
+        libpygio.read_gio_double(file_name,var_name,result.ctypes.data_as(ct.POINTER(ct.c_double)))
+        return result
+    elif(var_type==2):
+        #int32
+        result = np.ndarray(var_size,dtype=np.int32)
+        libpygio.read_gio_int32(file_name,var_name,result.ctypes.data_as(ct.POINTER(ct.c_int32)))
+        return result
+    elif(var_type==3):
+        #int64
+        result = np.ndarray(var_size,dtype=np.int64)
+        libpygio.read_gio_int64(file_name,var_name,result.ctypes.data_as(ct.POINTER(ct.c_int64)))
+        return result        
+        
+def gio_inspect(file_name):
+    libpygio.inspect_gio(file_name)
+

python/lib/gio.cxx

+/*
+ *                    Copyright (C) 2015, UChicago Argonne, LLC
+ *                               All Rights Reserved
+ * 
+ *                               Generic IO (ANL-15-066)
+ *                     Hal Finkel, Argonne National Laboratory
+ * 
+ *                              OPEN SOURCE LICENSE
+ * 
+ * Under the terms of Contract No. DE-AC02-06CH11357 with UChicago Argonne,
+ * LLC, the U.S. Government retains certain rights in this software.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ *   1. Redistributions of source code must retain the above copyright notice,
+ *      this list of conditions and the following disclaimer.
+ * 
+ *   2. Redistributions in binary form must reproduce the above copyright
+ *      notice, this list of conditions and the following disclaimer in the
+ *      documentation and/or other materials provided with the distribution.
+ * 
+ *   3. Neither the names of UChicago Argonne, LLC or the Department of Energy
+ *      nor the names of its contributors may be used to endorse or promote
+ *      products derived from this software without specific prior written
+ *      permission.
+ * 
+ * *****************************************************************************
+ * 
+ *                                  DISCLAIMER
+ * THE SOFTWARE IS SUPPLIED “AS IS” WITHOUT WARRANTY OF ANY KIND.  NEITHER THE
+ * UNTED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR
+ * UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY,
+ * EXPRESS OR IMPLIED, OR ASSUMES ANY LEGAL LIABILITY OR RESPONSIBILITY FOR THE
+ * ACCURACY, COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, DATA, APPARATUS,
+ * PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD NOT INFRINGE
+ * PRIVATELY OWNED RIGHTS.
+ * 
+ * *****************************************************************************
+ */
+
+#include "gio.h"
+#include <iostream>
+
+  void read_gio_float(char* file_name, char* var_name, float* data){
+    read_gio<float>(file_name,var_name,data);
+  }
+  void read_gio_double(char* file_name, char* var_name, double* data){
+    read_gio<double>(file_name,var_name,data);
+  }
+  void read_gio_int32(char* file_name, char* var_name, int* data){
+    read_gio<int>(file_name,var_name,data);
+  }
+  void read_gio_int64(char* file_name, char* var_name, int64_t* data){
+    read_gio<int64_t>(file_name,var_name,data);
+  }
+  
+  int64_t get_elem_num(char* file_name){
+    gio::GenericIO reader(file_name);
+    reader.openAndReadHeader(gio::GenericIO::MismatchAllowed);
+    int num_ranks = reader.readNRanks();
+    uint64_t size = 0;
+    for(int i =0;i<num_ranks;++i)
+      size +=reader.readNumElems(i);
+    reader.close();
+    return size;
+  }
+
+  var_type get_variable_type(char* file_name,char* var_name){
+   gio::GenericIO reader(file_name);
+   std::vector<gio::GenericIO::VariableInfo> VI;
+   reader.openAndReadHeader(gio::GenericIO::MismatchAllowed);
+   reader.getVariableInfo(VI);
+
+   int num =VI.size();
+    for(int i =0;i<num;++i){
+      gio::GenericIO::VariableInfo vinfo = VI[i];
+      if(vinfo.Name == var_name){
+	if(vinfo.IsFloat && vinfo.Size == 4)
+	  return float_type;
+	else if(vinfo.IsFloat && vinfo.Size == 8)
+	  return double_type;
+	else if(!vinfo.IsFloat && vinfo.Size == 4)
+	  return int32_type;
+	else if(!vinfo.IsFloat && vinfo.Size == 8)
+	  return int64_type;
+	else
+	  return type_not_found;
+      }
+    }
+    return var_not_found;
+      
+  }
+
+extern "C" void inspect_gio(char* file_name){
+  int64_t size = get_elem_num(file_name);
+  gio::GenericIO reader(file_name);
+  std::vector<gio::GenericIO::VariableInfo> VI;
+  reader.openAndReadHeader(gio::GenericIO::MismatchAllowed);
+  reader.getVariableInfo(VI);
+  std::cout<<"Number of Elements: "<<size<<std::endl;
+  int num =VI.size();
+  std::cout<<"[data type] Variable name"<<std::endl;
+  std::cout<<"---------------------------------------------"<<std::endl;
+  for(int i =0;i<num;++i){
+    gio::GenericIO::VariableInfo vinfo = VI[i];
+
+    if(vinfo.IsFloat)
+      std::cout<<"[f";
+    else
+      std::cout<<"[i";
+    std::cout<<" "<<vinfo.Size*8<<"] ";
+    std::cout<<vinfo.Name<<std::endl;
+  }
+  std::cout<<"\n(i=integer,f=floating point, number bits size)"<<std::endl;
+}
+

python/lib/gio.h

+/*
+ *                    Copyright (C) 2015, UChicago Argonne, LLC
+ *                               All Rights Reserved
+ * 
+ *                               Generic IO (ANL-15-066)
+ *                     Hal Finkel, Argonne National Laboratory
+ * 
+ *                              OPEN SOURCE LICENSE
+ * 
+ * Under the terms of Contract No. DE-AC02-06CH11357 with UChicago Argonne,
+ * LLC, the U.S. Government retains certain rights in this software.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ *   1. Redistributions of source code must retain the above copyright notice,
+ *      this list of conditions and the following disclaimer.
+ * 
+ *   2. Redistributions in binary form must reproduce the above copyright
+ *      notice, this list of conditions and the following disclaimer in the
+ *      documentation and/or other materials provided with the distribution.
+ * 
+ *   3. Neither the names of UChicago Argonne, LLC or the Department of Energy
+ *      nor the names of its contributors may be used to endorse or promote
+ *      products derived from this software without specific prior written
+ *      permission.
+ * 
+ * *****************************************************************************
+ * 
+ *                                  DISCLAIMER
+ * THE SOFTWARE IS SUPPLIED “AS IS” WITHOUT WARRANTY OF ANY KIND.  NEITHER THE
+ * UNTED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR
+ * UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY,
+ * EXPRESS OR IMPLIED, OR ASSUMES ANY LEGAL LIABILITY OR RESPONSIBILITY FOR THE
+ * ACCURACY, COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, DATA, APPARATUS,
+ * PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD NOT INFRINGE
+ * PRIVATELY OWNED RIGHTS.
+ * 
+ * *****************************************************************************
+ */
+
+#include <GenericIO.h>
+
+#include <stdint.h>
+#include <sstream>
+
+template <class T>
+void read_gio(char* file_name, std::string var_name, T*& data){
+  gio::GenericIO reader(file_name);
+  reader.openAndReadHeader(gio::GenericIO::MismatchAllowed);
+  int num_ranks = reader.readNRanks();
+  uint64_t max_size = 0;
+  uint64_t rank_size[num_ranks];
+  for(int i =0;i<num_ranks;++i){
+    rank_size[i] = reader.readNumElems(i);
+    if(max_size < rank_size[i])
+      max_size = rank_size[i];
+  }
+  T* rank_data = new T[max_size+reader.requestedExtraSpace()/sizeof(T)];
+  int64_t offset =0;
+  reader.addVariable(var_name,rank_data,true);
+  for(int i=0;i<num_ranks;++i){
+    reader.readData(i,false);
+    std::copy(rank_data,rank_data+rank_size[i],data+offset);
+    offset +=rank_size[i];
+  }
+  delete [] rank_data;
+  reader.close();
+}
+extern "C" int64_t get_elem_num(char* file_name);
+
+extern "C" void read_gio_float (char* file_name, char* var_name, float* data);
+extern "C" void read_gio_double(char* file_name, char* var_name, double* data);
+extern "C" void read_gio_int32 (char* file_name, char* var_name, int* data); 
+extern "C" void read_gio_int64 (char* file_name, char* var_name, int64_t* data);
+enum var_type{
+  float_type=0,
+  double_type=1,
+  int32_type=2,
+  int64_type=3,
+  type_not_found=9,
+  var_not_found=10
+};
+extern "C" var_type get_variable_type(char* file_name,char* var_name);
+extern "C" void inspect_gio(char* file_name);