Commit e0b4befb authored by Jakob Luettgau's avatar Jakob Luettgau
Browse files

Reenable eager utils discovery but ensure LD_LIBRARY_PATH takes precedence.

parent 10a7898f
......@@ -7,18 +7,22 @@ import numpy as np
import pandas as pd
from darshan.discover_darshan import discover_darshan
from darshan.api_def_c import load_darshan_header
from darshan.discover_darshan import find_utils
from darshan.discover_darshan import check_version
#DARSHAN_PATH = discover_darshan()
API_def_c = load_darshan_header()
ffi = cffi.FFI()
ffi.cdef(API_def_c)
#libdutil = ffi.dlopen("libdarshan-util.so")
libdutil = ffi.dlopen("//home/pq/p/software/darshan-pydarshan/darshan-util/libdarshan-util.so")
libdutil = None
libdutil = find_utils(ffi, libdutil)
......@@ -33,7 +37,8 @@ def log_open(filename):
log handle
"""
b_fname = filename.encode()
log = {"handle": libdutil.darshan_log_open(b_fname), 'modules': None, 'name_records': None}
handle = libdutil.darshan_log_open(b_fname)
log = {"handle": handle, 'modules': None, 'name_records': None}
return log
......@@ -194,25 +199,19 @@ def log_lookup_name_records(log, ids=[]):
name_records = {}
print(ids)
#cids = ffi.new("darshan_record_id *") * len(ids)
whitelist = (ctypes.c_ulonglong * len(ids))(*ids)
whitelist_cnt = len(ids)
whitelistp = ffi.from_buffer(whitelist)
nrecs = ffi.new("struct darshan_name_record **")
cnt = ffi.new("int *")
libdutil.darshan_log_get_filtered_name_records(log['handle'], nrecs, cnt, ffi.cast("darshan_record_id *", whitelistp), whitelist_cnt)
for i in range(0, cnt[0]):
name_records[nrecs[0][i].id] = ffi.string(nrecs[0][i].name).decode("utf-8")
# add to cache
log['name_records'] = name_records
......@@ -246,7 +245,7 @@ def log_get_dxt_record(log, mod_name, mod_type, reads=True, writes=True, mode='d
"""
modules = log_get_modules(log)
name_records = log_get_name_records(log)
#name_records = log_get_name_records(log)
rec = {}
buf = ffi.new("void **")
......@@ -259,7 +258,7 @@ def log_get_dxt_record(log, mod_name, mod_type, reads=True, writes=True, mode='d
rec['id'] = filerec[0].base_rec.id
rec['rank'] = filerec[0].base_rec.rank
rec['hostname'] = ffi.string(filerec[0].hostname).decode("utf-8")
rec['filename'] = name_records[rec['id']]
#rec['filename'] = name_records[rec['id']]
wcnt = filerec[0].write_count
rcnt = filerec[0].read_count
......
......@@ -8,10 +8,9 @@ import os
def darshanutils_version():
def check_version():
"""
Discovers an existing darshan-util installation and returns the appropriate
path to a shared object for use with Python's CFFI.
Get version from pkg-config and return info.
:return: Path to a darshan-util installation.
"""
......@@ -23,12 +22,7 @@ def darshanutils_version():
out,err = p.communicate()
retval = p.wait()
return retval
if darshan_config:
return os.path.realpath(darshan_config + '/../../')
else:
raise RuntimeError('Could not discover darshan! Is darshan-util installed?')
return out.decode('ascii').strip()
def discover_darshan_pkgconfig():
......@@ -46,10 +40,10 @@ def discover_darshan_pkgconfig():
out,err = p.communicate()
retval = p.wait()
print(retval)
path = os.path.dirname(out.decode('ascii').strip())
if darshan_config:
return os.path.realpath(darshan_config + '/../../')
if path:
return os.path.realpath(path + '/../../')
else:
raise RuntimeError('Could not discover darshan! Is darshan-util installed?')
......@@ -63,26 +57,50 @@ def discover_darshan_shutil():
"""
import shutil
darshan_config = shutil.which('darshan-parser')
path = shutil.which('darshan-parser')
# alternatively via
#pkg-config --path darshan-util
if darshan_config:
return os.path.realpath(darshan_config + '/../../')
if path:
return os.path.realpath(path + '/../../')
else:
raise RuntimeError('Could not discover darshan! Is darshan-util installed and set in your PATH?')
def discover_darshan():
"""
Discovers an existing darshan-util installation and returns the appropriate
path to a shared object for use with Python's CFFI.
:return: Path to a darshan-util installation.
"""
return discover_darshan_shutil()
def find_utils(ffi, libdutil):
if libdutil is None:
try:
libdutil = ffi.dlopen("libdarshan-util.so")
print("dlopen ok")
except:
print("dlopen failed")
libdutil = None
if libdutil is None:
try:
DARSHAN_PATH = discover_darshan_shutil()
print(DARSHAN_PATH)
libdutil = ffi.dlopen(DARSHAN_PATH + "/lib/libdarshan-util.so")
print("shutil ok")
except:
libdutil = None
print("shutil failed")
if libdutil is None:
try:
DARSHAN_PATH = discover_darshan_pkgconfig()
print(DARSHAN_PATH)
libdutil = ffi.dlopen(DARSHAN_PATH + "/lib/libdarshan-util.so")
print("pkg-config ok")
except:
libdutil = None
print("pkg-config failed")
if libdutil is None:
raise RuntimeError('Could not find libdarshan-util.so! Is darshan-util installed? Please ensure one of the the following: 1) export LD_LIBRARY_PATH=<path-to-libdarshan-util.so>, or 2) darshan-parser can found using the PATH variable, or 3) pkg-config can resolve pkg-config --path darshan-util')
return libdutil
def load_darshan_header():
......
......@@ -66,6 +66,7 @@ class DarshanReport(object):
self.modules = {}
self.counters = {}
self.records = {}
self.mounts = {}
self.name_records = {}
# initialize report/summary namespace
......@@ -113,7 +114,7 @@ class DarshanReport(object):
if filename:
self.log = backend.log_open(self.filename)
self.read_metadata()
self.read_metadata(read_all=read_all)
if read_all:
self.read_all()
......@@ -151,7 +152,7 @@ class DarshanReport(object):
return result
def read_metadata(self):
def read_metadata(self, read_all=False):
"""
Read metadata such as the job, the executables and available modules.
......@@ -169,15 +170,53 @@ class DarshanReport(object):
self.end_time = datetime.datetime.fromtimestamp(self.metadata['job']['end_time'])
self.data['mounts'] = backend.log_get_mounts(self.log)
self.mounts = self.data['mounts']
self.data['modules'] = backend.log_get_modules(self.log)
self.modules = self.data['modules']
if self.read_all == True:
if read_all == True:
self.data["name_records"] = backend.log_get_name_records(self.log)
self.name_records = self.data['name_records']
def update_name_records(self, mod=None):
"""
Update (and prune unused) name records from resolve table.
First reindexes all used name record identifiers and then queries
darshan-utils library to compile filtered list of name records.
Args:
None
Return:
None
"""
# sanitize inputs
mods = mod
if mods is None:
mods = self.records
else:
mods = [mod]
# state
ids = set()
for mod in mods:
for rec in self.records[mod]:
ids.add(rec['id'])
self.name_records = backend.log_lookup_name_records(self.log, ids)
def read_all(self):
"""
Read all available records from darshan log and return as dictionary.
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment