Commit 932d6999 authored by Jakob Luettgau's avatar Jakob Luettgau
Browse files

Collapse low-level log_get_*_record into log_get_record which takes modstring....

Collapse low-level log_get_*_record into log_get_record which takes modstring. Low-level lustre record now includes ost_ids.
parent 8db50f35
......@@ -6,7 +6,6 @@ import ctypes
import numpy as np
import pandas as pd
from darshan.api_def_c import load_darshan_header
from darshan.discover_darshan import find_utils
from darshan.discover_darshan import check_version
......@@ -24,6 +23,24 @@ check_version(ffi, libdutil)
_structdefs = {
"BG/Q": "struct darshan_bgq_record **",
"DXT_MPIIO": "struct dxt_file_record **",
"DXT_POSIX": "struct dxt_file_record **",
"H5F": "struct darshan_hdf5_file **",
"H5D": "struct darshan_hdf5_dataset **",
"LUSTRE": "struct darshan_lustre_record **",
"MPI-IO": "struct darshan_mpiio_file **",
"PNETCDF": "struct darshan_pnetcdf_file **",
"POSIX": "struct darshan_posix_file **",
"STDIO": "struct darshan_stdio_file **",
}
def get_lib_version():
"""
Return the version information hardcoded into the shared library.
......@@ -231,83 +248,25 @@ def log_lookup_name_records(log, ids=[]):
def log_get_dxt_record(log, mod_name, mod_type, reads=True, writes=True, dtype='dict'):
def log_get_record(log, mod, dtype='numpy'):
"""
Returns a dictionary holding a dxt darshan log record.
Standard entry point fetch records via mod string.
Args:
log: Handle returned by darshan.open
mod_name (str): Name of the Darshan module
mod_type (str): String containing the C type
Return:
dict: generic log record
Example:
The typical darshan log record provides two arrays, on for integer counters
and one for floating point counters:
>>> darshan.log_get_dxt_record(log, "DXT_POSIX", "struct dxt_file_record **")
{'rank': 0, 'read_count': 11, 'read_segments': array([...]), ...}
log record of type dtype
"""
modules = log_get_modules(log)
#name_records = log_get_name_records(log)
rec = {}
buf = ffi.new("void **")
r = libdutil.darshan_log_get_record(log['handle'], modules[mod_name]['idx'], buf)
if r < 1:
return None
filerec = ffi.cast(mod_type, buf)
clst = []
rec['id'] = filerec[0].base_rec.id
rec['rank'] = filerec[0].base_rec.rank
rec['hostname'] = ffi.string(filerec[0].hostname).decode("utf-8")
#rec['filename'] = name_records[rec['id']]
wcnt = filerec[0].write_count
rcnt = filerec[0].read_count
rec['write_count'] = wcnt
rec['read_count'] = rcnt
rec['write_segments'] = []
rec['read_segments'] = []
size_of = ffi.sizeof("struct dxt_file_record")
segments = ffi.cast("struct segment_info *", buf[0] + size_of )
for i in range(wcnt):
seg = {
"offset": segments[i].offset,
"length": segments[i].length,
"start_time": segments[i].start_time,
"end_time": segments[i].end_time
}
rec['write_segments'].append(seg)
for i in range(rcnt):
i = i + wcnt
seg = {
"offset": segments[i].offset,
"length": segments[i].length,
"start_time": segments[i].start_time,
"end_time": segments[i].end_time
}
rec['read_segments'].append(seg)
if dtype == "pandas":
rec['read_segments'] = pd.DataFrame(rec['read_segments'])
rec['write_segments'] = pd.DataFrame(rec['write_segments'])
if mod in ['LUSTRE']:
rec = _log_get_lustre_record(log, dtype=dtype)
elif mod in ['DXT_POSIX', 'DXT_MPIIO']:
rec = log_get_dxt_record(log, mod, _structdefs[mod], dtype=dtype)
else:
rec = log_get_generic_record(log, mod, _structdefs[mod], dtype=dtype)
return rec
......@@ -369,10 +328,11 @@ def log_get_generic_record(log, mod_name, mod_type, dtype='numpy'):
df_c = pd.DataFrame(cdict, index=[0])
df_fc = pd.DataFrame(fcdict, index=[0])
# flip column order
# flip column order (to prepend id and rank)
df_c = df_c[df_c.columns[::-1]]
df_fc = df_fc[df_fc.columns[::-1]]
# attach id and rank to counters and fcounters
df_c['id'] = rec['id']
df_c['rank'] = rec['rank']
......@@ -449,35 +409,7 @@ def fcounter_names(mod_name):
return counter_names(mod_name, fcnts=True)
def log_get_bgq_record(log):
"""
Returns a darshan log record for BG/Q.
Args:
log: handle returned by darshan.open
"""
return log_get_generic_record(log, "BG/Q", "struct darshan_bgq_record **")
def log_get_hdf5_file_record(log):
"""
Returns a darshan log record for an HDF5 file.
Args:
log: handle returned by darshan.open
"""
return log_get_generic_record(log, "H5F", "struct darshan_hdf5_file **")
def log_get_hdf5_dataset_record(log):
"""
Returns a darshan log record for an HDF5 dataset.
Args:
log: handle returned by darshan.open
"""
return log_get_generic_record(log, "H5D", "struct darshan_hdf5_dataset **")
def log_get_lustre_record(log):
def _log_get_lustre_record(log, dtype='numpy'):
"""
Returns a darshan log record for Lustre.
......@@ -500,87 +432,128 @@ def log_get_lustre_record(log):
for i in range(0, len(rbuf[0].counters)):
clst.append(rbuf[0].counters[i])
rec['counters'] = np.array(clst, dtype=np.int64)
# counters
cdict = dict(zip(counter_names('LUSTRE'), rec['counters']))
# FIXME
# ost_ids
sizeof_64 = ffi.sizeof("int64_t")
sizeof_base = ffi.sizeof("struct darshan_base_record")
offset = sizeof_base + sizeof_64 * len(rbuf[0].counters)
offset = int(offset/sizeof_64)
ost_ids = ffi.cast("int64_t *", rbuf[0])
ostlst = []
for i in range(0, cdict['LUSTRE_STRIPE_WIDTH']):
print(rbuf[0].ost_ids[i])
for i in range(offset, cdict['LUSTRE_STRIPE_WIDTH']+offset):
ostlst.append(ost_ids[i])
rec['ost_ids'] = np.array(ostlst, dtype=np.int64)
print(rec['ost_ids'])
return rec
sys.exit()
# dtype conversion
if dtype == "dict":
rec.update({
'counters': cdict,
'fcounters': fcdict
'ost_ids': ostlst
})
if dtype == "pandas":
df_c = pd.DataFrame(cdict, index=[0])
# prepend id and rank
df_c = df_c[df_c.columns[::-1]] # flip colum order
df_c['id'] = rec['id']
df_c['rank'] = rec['rank']
df_c = df_c[df_c.columns[::-1]] # flip back
rec.update({
'counters': pd.DataFrame(cdict, index=[0]),
'fcounters': pd.DataFrame(fcdict, index=[0])
'counters': df_c,
'ost_ids': pd.DataFrame(rec['ost_ids'], columns=['ost_ids']),
})
return rec
def log_get_mpiio_record(log):
def log_get_dxt_record(log, mod_name, mod_type, reads=True, writes=True, dtype='dict'):
"""
Returns a darshan log record for MPI-IO.
Returns a dictionary holding a dxt darshan log record.
Args:
log: handle returned by darshan.open
log: Handle returned by darshan.open
mod_name (str): Name of the Darshan module
mod_type (str): String containing the C type
Returns:
dict: log record
"""
return log_get_generic_record(log, "MPI-IO", "struct darshan_mpiio_file **")
Return:
dict: generic log record
Example:
def log_get_pnetcdf_record(log):
"""
Returns a darshan log record for PnetCDF.
The typical darshan log record provides two arrays, on for integer counters
and one for floating point counters:
>>> darshan.log_get_dxt_record(log, "DXT_POSIX", "struct dxt_file_record **")
{'rank': 0, 'read_count': 11, 'read_segments': array([...]), ...}
Args:
log: handle returned by darshan.open
Returns:
dict: log record
"""
return log_get_generic_record(log, "PNETCDF", "struct darshan_pnetcdf_file **")
modules = log_get_modules(log)
#name_records = log_get_name_records(log)
def log_get_posix_record(log):
"""
Returns a darshan log record for
rec = {}
buf = ffi.new("void **")
r = libdutil.darshan_log_get_record(log['handle'], modules[mod_name]['idx'], buf)
if r < 1:
return None
filerec = ffi.cast(mod_type, buf)
clst = []
Args:
log: handle returned by darshan.open
rec['id'] = filerec[0].base_rec.id
rec['rank'] = filerec[0].base_rec.rank
rec['hostname'] = ffi.string(filerec[0].hostname).decode("utf-8")
#rec['filename'] = name_records[rec['id']]
Returns:
dict: log record
"""
return log_get_generic_record(log, "POSIX", "struct darshan_posix_file **")
wcnt = filerec[0].write_count
rcnt = filerec[0].read_count
rec['write_count'] = wcnt
rec['read_count'] = rcnt
rec['write_segments'] = []
rec['read_segments'] = []
def log_get_stdio_record(log):
"""
Returns a darshan log record for STDIO.
size_of = ffi.sizeof("struct dxt_file_record")
segments = ffi.cast("struct segment_info *", buf[0] + size_of )
for i in range(wcnt):
seg = {
"offset": segments[i].offset,
"length": segments[i].length,
"start_time": segments[i].start_time,
"end_time": segments[i].end_time
}
rec['write_segments'].append(seg)
for i in range(rcnt):
i = i + wcnt
seg = {
"offset": segments[i].offset,
"length": segments[i].length,
"start_time": segments[i].start_time,
"end_time": segments[i].end_time
}
rec['read_segments'].append(seg)
if dtype == "pandas":
rec['read_segments'] = pd.DataFrame(rec['read_segments'])
rec['write_segments'] = pd.DataFrame(rec['write_segments'])
return rec
Args:
log: handle returned by darshan.open
Returns:
dict: log record
"""
return log_get_generic_record(log, "STDIO", "struct darshan_stdio_file **")
......@@ -38,7 +38,7 @@ class DarshanReportJSONEncoder(json.JSONEncoder):
return json.JSONEncoder.default(self, obj)
structdefs = {
_structdefs = {
"BG/Q": "struct darshan_bgq_record **",
"DXT_MPIIO": "struct dxt_file_record **",
"DXT_POSIX": "struct dxt_file_record **",
......@@ -320,7 +320,7 @@ class DarshanReport(object):
self.counters[mod]['fcounters'] = fcn
rec = backend.log_get_generic_record(self.log, mod, structdefs[mod], dtype=dtype)
rec = backend.log_get_generic_record(self.log, mod, _structdefs[mod], dtype=dtype)
while rec != None:
if dtype == 'pandas':
self.records[mod].append(rec)
......@@ -337,7 +337,7 @@ class DarshanReport(object):
self.modules[mod]['num_records'] += 1
# fetch next
rec = backend.log_get_generic_record(self.log, mod, structdefs[mod], dtype=dtype)
rec = backend.log_get_generic_record(self.log, mod, _structdefs[mod], dtype=dtype)
if self.lookup_name_records:
......@@ -401,7 +401,7 @@ class DarshanReport(object):
return
#structdefs = {
#_structdefs = {
# "DXT_POSIX": "struct dxt_file_record **",
# "DXT_MPIIO": "struct dxt_file_record **",
#}
......@@ -415,7 +415,7 @@ class DarshanReport(object):
self.counters[mod] = {}
rec = backend.log_get_dxt_record(self.log, mod, structdefs[mod], dtype=dtype)
rec = backend.log_get_dxt_record(self.log, mod, _structdefs[mod], dtype=dtype)
while rec != None:
if dtype == 'numpy':
self.records[mod].append(rec)
......@@ -429,7 +429,7 @@ class DarshanReport(object):
self.data['modules'][mod]['num_records'] += 1
# fetch next
rec = backend.log_get_dxt_record(self.log, mod, structdefs[mod], reads=reads, writes=writes, dtype=dtype)
rec = backend.log_get_dxt_record(self.log, mod, _structdefs[mod], reads=reads, writes=writes, dtype=dtype)
pass
......@@ -456,12 +456,12 @@ class DarshanReport(object):
self.counters[mod]['counters'] = cn
self.counters[mod]['fcounters'] = fcn
rec = backend.log_get_generic_record(self.log, mod, structdefs[mod], dtype=dtype)
rec = backend.log_get_generic_record(self.log, mod, _structdefs[mod], dtype=dtype)
while rec != None:
yield rec
# fetch next
rec = backend.log_get_generic_record(self.log, mod, structdefs[mod])
rec = backend.log_get_generic_record(self.log, mod, _structdefs[mod])
def info(self, metadata=False):
......
%% Cell type:markdown id: tags:
# DarshanUtils for Python
This notebook gives an overwiew of features provided by the Python bindings for DarshanUtils.
%% Cell type:markdown id: tags:
By default all records, metadata, available modules and the name records are loaded:
%% Cell type:code id: tags:
``` python
import darshan
report = darshan.DarshanReport("example-logs/example.darshan", read_all=True) # Default behavior
report.info()
```
%%%% Output: stream
POSIX
POSIX
MPI-IO
POSIX
MPI-IO
STDIO
Filename: example-logs/example.darshan
Times: 2017-03-20 10:07:47 to 2017-03-20 10:09:43 (Duration 0:01:56)
Executeable: /global/project/projectdirs/m888/glock/tokio-abc-results/bin.edison/vpicio_uni /scratch2/scratchdirs/glock/tokioabc-s.4478544/vpicio/vpicio.hdf5 32
Processes: 2048
JobID: 4478544
UID: 69615
Modules in Log: ['POSIX', 'MPI-IO', 'LUSTRE', 'STDIO']
Loaded Records: {'POSIX': 1, 'MPI-IO': 1, 'STDIO': 129}
Name Records: 3
Darshan/Hints: {'lib_ver': '3.1.3', 'h': 'romio_no_indep_rw=true;cb_nodes=4'}
DarshanReport: id(140531725932304) (tmp)
%% Cell type:code id: tags:
``` python
report.modules
```
%%%% Output: execute_result
{'POSIX': {'len': 186, 'ver': 3, 'idx': 1, 'num_records': 1},
'MPI-IO': {'len': 154, 'ver': 2, 'idx': 2, 'num_records': 1},
'LUSTRE': {'len': 87, 'ver': 1, 'idx': 7},
'STDIO': {'len': 3234, 'ver': 1, 'idx': 8, 'num_records': 129}}
%% Cell type:markdown id: tags:
A few of the internal data structures explained:
%% Cell type:code id: tags:
``` python
# report.metadata # dictionary with raw metadata from darshan log
# report.modules # dictionary with raw module info from darshan log (need: technical, module idx)
# report.name_records # dictionary for resovling name records: id -> path/name
# report.records # per module "dataframes"/dictionaries holding loaded records
```
%% Cell type:markdown id: tags:
The darshan report holds a variety of namespaces for report related data. All of them are also referenced in `report.data` at the moment, but reliance on this internal organization of the report object is discouraged once the API stabilized. Currently, `report.data` references the following information:
%% Cell type:code id: tags:
``` python
report.data.keys()
```
%%%% Output: execute_result
dict_keys(['version', 'metadata', 'records', 'summary', 'modules', 'counters', 'name_records', 'mounts'])
%% Cell type:code id: tags:
``` python
report.mod_read_all_records('POSIX')
```
%%%% Output: stream
POSIX
MPI-IO
STDIO
%% Cell type:code id: tags:
``` python
report.mod_read_all_records('STDIO')
```
%%%% Output: stream
POSIX
MPI-IO
STDIO
%% Cell type:code id: tags:
``` python
report.update_name_records()
report.info()
```
%%%% Output: stream
POSIX
MPI-IO
STDIO
Filename: example-logs/example.darshan
Times: 2017-03-20 10:07:47 to 2017-03-20 10:09:43 (Duration 0:01:56)
Executeable: /global/project/projectdirs/m888/glock/tokio-abc-results/bin.edison/vpicio_uni /scratch2/scratchdirs/glock/tokioabc-s.4478544/vpicio/vpicio.hdf5 32
Processes: 2048
JobID: 4478544
UID: 69615
Modules in Log: ['POSIX', 'MPI-IO', 'LUSTRE', 'STDIO']
Loaded Records: {'POSIX': 1, 'MPI-IO': 1, 'STDIO': 129}
Name Records: 3
Darshan/Hints: {'lib_ver': '3.1.3', 'h': 'romio_no_indep_rw=true;cb_nodes=4'}
DarshanReport: id(140531725932304) (tmp)
%% Cell type:code id: tags:
``` python
# visualization helper used by different examples in the remainder of this notebook
from IPython.display import display, HTML
# usage: display(obj)
```
%% Cell type:markdown id: tags:
### Record Formats and Selectively Loading Records
For memory efficiant analysis, it is possible to supress records from being loaded automatically. This is useful, for example, when analysis considers only records of a particular layer/module.
%% Cell type:code id: tags:
``` python
import darshan
report = darshan.DarshanReport("example-logs/example.darshan", read_all=False, lookup_name_records=True) # Loads no records!
```
%% Cell type:code id: tags:
``` python
# expected to fail, as no records were loaded
try:
print(len(report.records['STDIO']), "records loaded for STDIO.")
except:
print("No STDIO records loaded for this report yet.")
```
%%%% Output: stream
No STDIO records loaded for this report yet.
%% Cell type:markdown id: tags:
Additional records then can be loaded selectively, for example, on a per module basis:
%% Cell type:markdown id: tags:
#### dtype: pandas
%% Cell type:code id: tags:
``` python
report.mod_read_all_records("STDIO", dtype="pandas")
```
%%%% Output: stream
STDIO
%% Cell type:code id: tags:
``` python
print('id', report.records['STDIO'][0]['id'])
print('rank', report.records['STDIO'][0]['rank'])
display(report.records['STDIO'][0]['counters'])
display(report.records['STDIO'][0]['fcounters'])