Commit 42a6f6de authored by Jakob Luettgau's avatar Jakob Luettgau
Browse files

Prepare seperation of raw data and summary namespaces, add provenance and...

Prepare seperation of raw data and summary namespaces, add provenance and consistency structures for report algebra.
parent 18bfe771
......@@ -20,7 +20,7 @@ def agg_ioops(self, mode='append'):
# convienience
recs = self.report['records']
recs = self.data['records']
ctx = {}
# aggragate
......@@ -106,7 +106,7 @@ def agg_ioops(self, mode='append'):
# reset summary target
if mode == 'append':
self.report['agg_ioops'] = ctx
self.data['agg_ioops'] = ctx
else:
return ctx
......
......@@ -7,14 +7,14 @@ def create_sankey(self):
"""
# convienience
recs = self.report['records']
nrecs = self.report['name_records']
recs = self.data['records']
nrecs = self.data['name_records']
ctx = {}
# check records for module are present
if 'POSIX' not in recs:
self.report['sankey'] = None
self.data['sankey'] = None
return
ranks = {}
......@@ -27,7 +27,7 @@ def create_sankey(self):
# build mnt list
mnts = []
for mnt in self.report['mounts']:
for mnt in self.data['mounts']:
mnts.append(mnt[0])
......@@ -78,7 +78,7 @@ def create_sankey(self):
tmp = json.dumps(ctx, cls=NumpyEncoder)
tmp = json.loads(tmp)
self.report['sankey'] = tmp
self.data['sankey'] = tmp
......
......@@ -14,14 +14,14 @@ def create_timeline(self, group_by='rank'):
self.mod_read_all_dxt_records("DXT_MPIIO")
self.report['timeline'] = {'groups': [], 'items': []}
self.data['timeline'] = {'groups': [], 'items': []}
groups = self.report['timeline']['groups']
items = self.report['timeline']['items']
groups = self.data['timeline']['groups']
items = self.data['timeline']['items']
start_time = datetime.datetime.fromtimestamp( self.report['job']['start_time'] )
start_time = datetime.datetime.fromtimestamp( self.data['metadata']['job']['start_time'] )
......@@ -94,8 +94,8 @@ def create_timeline(self, group_by='rank'):
supported = ['DXT_POSIX', 'DXT_MPIIO']
for mod in supported:
if mod in self.report['records']:
for rec in self.report['records'][mod]:
if mod in self.data['records']:
for rec in self.data['records'][mod]:
groupify(rec, mod)
......
......@@ -12,7 +12,7 @@ def mod_agg_iohist(self, mod, mode='append'):
"""
# convienience
recs = self.report['records']
recs = self.data['records']
ctx = {}
......@@ -56,9 +56,9 @@ def mod_agg_iohist(self, mod, mode='append'):
if mode == 'append':
if 'agg_iohist' not in self.report:
self.report['agg_iohist'] = {}
self.report['agg_iohist'][mod] = ctx
if 'agg_iohist' not in self.data:
self.data['agg_iohist'] = {}
self.data['agg_iohist'][mod] = ctx
else:
return ctx
......
......@@ -35,37 +35,57 @@ class DarshanReport(object):
a number of common aggregations can be performed.
"""
def __init__(self, filename, mode='dict'):
def __init__(self, filename=None, data_format='numpy', automatic_summary=False):
self.filename = filename
# options
self.data_format = data_format # Experimental: preferred internal representation: numpy useful for aggregations, dict good for export/REST
# might require alternative granularity: e.g., records, vs summaries?
self.automatic_summary = automatic_summary
# initialize actual report dictionary
self.report = {'version': 1}
self.report['records'] = {}
# state dependent book-keeping
self.converted_records = False # true if convert_records() was called (unnumpyfy)
self.log = backend.log_open(self.filename)
# initialize data namespace
self.data_revision = 0 # counter for consistency checks
self.data = {'version': 1}
self.data['records'] = {}
self.data['metadata'] = {}
# initialize report/summary namespace
self.summary_revision = 0 # counter to check if summary needs update
self.summary = {}
# state dependent book-keeping
self.converted_records = False # true if convert_records() was called (unnumpyfy)
# when using report algebra this log allows to untangle potentially
# unfair aggregations (e.g., double accounting)
self.provenance_log = []
self.provenance_reports = {}
if filename:
self.log = backend.log_open(self.filename)
self.read_metadata()
self.data["name_records"] = backend.log_get_name_records(self.log)
self.read_metadata()
self.report["name_records"] = backend.log_get_name_records(self.log)
def __add__(self, other):
new_report = self.copy()
#new_report = copy.deepcopy(self)
new_report.provenance_log.append(("add", self, other))
nr = DarshanReport()
nr.provenance_reports[self.filename] = copy.copy(self)
nr.provenance_reports[other.filename] = copy.copy(other)
nr.provenance_log.append(("add", self, other, datetime.datetime.now()))
# pull in records
return new_report
return nr
def read_all(self):
......@@ -85,7 +105,7 @@ class DarshanReport(object):
None
"""
for mod in self.report['modules']:
for mod in self.data['modules']:
self.mod_read_all_records(mod)
pass
......@@ -102,7 +122,7 @@ class DarshanReport(object):
None
"""
for mod in self.report['modules']:
for mod in self.data['modules']:
self.mod_read_all_dxt_records(mod)
pass
......@@ -119,10 +139,11 @@ class DarshanReport(object):
None
"""
self.report["job"] = backend.log_get_job(self.log)
self.report["exe"] = backend.log_get_exe(self.log)
self.report["mounts"] = backend.log_get_mounts(self.log)
self.report["modules"] = backend.log_get_modules(self.log)
self.data['metadata']['job'] = backend.log_get_job(self.log)
self.data['metadata']['exe'] = backend.log_get_exe(self.log)
self.data['metadata']['mounts'] = backend.log_get_mounts(self.log)
self.data['modules'] = backend.log_get_modules(self.log)
def mod_read_all_records(self, mod, mode='numpy'):
......@@ -158,14 +179,14 @@ class DarshanReport(object):
}
self.report['records'][mod] = []
self.data['records'][mod] = []
cn = backend.counter_names(mod)
fcn = backend.fcounter_names(mod)
self.report['modules'][mod]['counters'] = cn
self.report['modules'][mod]['fcounters'] = fcn
self.report['modules'][mod]['num_records'] = 0
self.data['modules'][mod]['counters'] = cn
self.data['modules'][mod]['fcounters'] = fcn
self.data['modules'][mod]['num_records'] = 0
......@@ -176,14 +197,14 @@ class DarshanReport(object):
#rec = json.loads(recs)
if mode == 'numpy':
self.report['records'][mod].append(rec)
self.data['records'][mod].append(rec)
else:
c = dict(zip(cn, rec['counters']))
fc = dict(zip(fcn, rec['fcounters']))
self.report['records'][mod].append([c, fc])
self.data['records'][mod].append([c, fc])
self.report['modules'][mod]['num_records'] += 1
self.data['modules'][mod]['num_records'] += 1
# fetch next
rec = backend.log_get_generic_record(self.log, mod, structdefs[mod])
......@@ -204,7 +225,7 @@ class DarshanReport(object):
"""
if mod not in self.report['modules']:
if mod not in self.data['modules']:
print("Skipping. Log does not contain data for mod:", mod)
return
......@@ -223,8 +244,8 @@ class DarshanReport(object):
}
self.report['records'][mod] = []
self.report['modules'][mod]['num_records'] = 0
self.data['records'][mod] = []
self.data['modules'][mod]['num_records'] = 0
......@@ -235,18 +256,18 @@ class DarshanReport(object):
#rec = json.loads(recs)
if mode == 'numpy':
self.report['records'][mod].append(rec)
self.data['records'][mod].append(rec)
else:
print("Not implemented.")
exit(1)
#c = dict(zip(cn, rec['counters']))
#fc = dict(zip(fcn, rec['fcounters']))
#self.report['records'][mod].append([c, fc])
#self.data['records'][mod].append([c, fc])
pass
self.report['modules'][mod]['num_records'] += 1
self.data['modules'][mod]['num_records'] += 1
# fetch next
rec = backend.log_get_dxt_record(self.log, mod, structdefs[mod])
......@@ -298,10 +319,10 @@ class DarshanReport(object):
None
"""
recs = self.report['records']
recs = self.data['records']
for mod in recs:
for i, rec in enumerate(self.report['records'][mod]):
for i, rec in enumerate(self.data['records'][mod]):
recs[mod][i]['counters'] = rec['counters'].tolist()
recs[mod][i]['fcounters'] = rec['fcounters'].tolist()
......
%% Cell type:code id: tags:
``` python
import darshan
import pprint
```
%% Cell type:code id: tags:
``` python
report = darshan.DarshanReport("example.darshan")
```
%% Cell type:markdown id: tags:
By default only metadata, available modules and the name records are loaded:
%% Cell type:code id: tags:
``` python
report.report
report.data
```
%%%% Output: execute_result
{'version': 1,
'records': {},
'job': {'jobid': 4478544,
'uid': 69615,
'start_time': 1490000867,
'end_time': 1490000983,
'metadata': {'lib_ver': '3.1.3', 'h': 'romio_no_indep_rw=true;cb_nodes=4'}},
'exe': '/global/project/projectdirs/m888/glock/tokio-abc-results/bin.edison/vpicio_uni /scratch2/scratchdirs/glock/tokioabc-s.4478544/vpicio/vpicio.hdf5 32',
'mounts': [('/.shared/base/default/etc/dat.conf', 'dvs'),
('/usr/lib64/libibverbs.so.1.0.0', 'dvs'),
('/usr/lib64/libibumad.so.3.0.2', 'dvs'),
('/usr/lib64/librdmacm.so.1.0.0', 'dvs'),
('/usr/lib64/libibgni.so.1.0.0', 'dvs'),
('/global/cscratch1', 'lustre'),
('/global/projectb', 'dvs'),
('/global/projecta', 'dvs'),
('/usr/sbin/ibstat', 'dvs'),
('/global/project', 'dvs'),
('/global/common', 'dvs'),
('/global/syscom', 'dvs'),
('/global/dna', 'dvs'),
('/opt/slurm', 'dvs'),
('/global/u1', 'dvs'),
('/global/u2', 'dvs'),
('/scratch1', 'lustre'),
('/scratch2', 'lustre'),
('/scratch3', 'lustre'),
('/etc', 'dvs'),
('/', 'rootfs'),
('/', 'dvs')],
'modules': {'POSIX': {'len': 186, 'ver': 3, 'idx': 1},
'MPI-IO': {'len': 154, 'ver': 2, 'idx': 2},
'LUSTRE': {'len': 87, 'ver': 1, 'idx': 6},
'STDIO': {'len': 3234, 'ver': 1, 'idx': 7}},
'metadata': {'job': {'jobid': 4478544,
'uid': 69615,
'start_time': 1490000867,
'end_time': 1490000983,
'metadata': {'lib_ver': '3.1.3', 'h': 'romio_no_indep_rw=true;cb_nodes=4'}},
'exe': '/global/project/projectdirs/m888/glock/tokio-abc-results/bin.edison/vpicio_uni /scratch2/scratchdirs/glock/tokioabc-s.4478544/vpicio/vpicio.hdf5 32',
'mounts': [('/.shared/base/default/etc/dat.conf', 'dvs'),
('/usr/lib64/libibverbs.so.1.0.0', 'dvs'),
('/usr/lib64/libibumad.so.3.0.2', 'dvs'),
('/usr/lib64/librdmacm.so.1.0.0', 'dvs'),
('/usr/lib64/libibgni.so.1.0.0', 'dvs'),
('/global/cscratch1', 'lustre'),
('/global/projectb', 'dvs'),
('/global/projecta', 'dvs'),
('/usr/sbin/ibstat', 'dvs'),
('/global/project', 'dvs'),
('/global/common', 'dvs'),
('/global/syscom', 'dvs'),
('/global/dna', 'dvs'),
('/opt/slurm', 'dvs'),
('/global/u1', 'dvs'),
('/global/u2', 'dvs'),
('/scratch1', 'lustre'),
('/scratch2', 'lustre'),
('/scratch3', 'lustre'),
('/etc', 'dvs'),
('/', 'rootfs'),
('/', 'dvs')],
'modules': {'POSIX': {'len': 186, 'ver': 3, 'idx': 1},
'MPI-IO': {'len': 154, 'ver': 2, 'idx': 2},
'LUSTRE': {'len': 87, 'ver': 1, 'idx': 6},
'STDIO': {'len': 3234, 'ver': 1, 'idx': 7}}},
'name_records': {14734109647742566553: '<STDIN>',
15920181672442173319: '<STDOUT>',
7238257241479193519: '<STDERR>',
6301063301082038805: '/scratch2/scratchdirs/glock/tokioabc-s.4478544/vpicio/vpicio.hdf5'}}
%% Cell type:markdown id: tags:
### Loading Additional Records
Additional records can be loaded for example on a per module basis:
%% Cell type:code id: tags:
``` python
# expected to fail
report.report['records']['POSIX']
report.data['records']['POSIX']
```
%%%% Output: error
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
<ipython-input-4-3a9a112f3a8d> in <module>
<ipython-input-8-1c44728d67db> in <module>
1 # expected to fail
----> 2 report.report['records']['POSIX']
----> 2 report.data['records']['POSIX']
KeyError: 'POSIX'
%% Cell type:code id: tags:
``` python
report.mod_read_all_records("POSIX")
```
%%%% Output: error
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
<ipython-input-9-e0928bbeb0bc> in <module>
----> 1 report.mod_read_all_records("POSIX")
~/ANL/darshan-decaf/darshan-workflow/python-packages/darshan/report.py in mod_read_all_records(self, mod, mode)
170 fcn = backend.fcounter_names(mod)
171
--> 172 self.data['modules'][mod]['counters'] = cn
173 self.data['modules'][mod]['fcounters'] = fcn
174 self.data['modules'][mod]['num_records'] = 0
KeyError: 'modules'
%% Cell type:code id: tags:
``` python
report.report['records']['POSIX']
report.data['records']['POSIX']
```
%%%% Output: execute_result
[{'id': 6301063301082038805,
'rank': -1,
'counters': array([ 2049, 18446744073709551615, 18446744073709551615,
0, 16402, 16404,
0, 0, 0,
0, 18446744073709551615, 18446744073709551615,
0, 0, 0,
2199023259968, 0, 2199023261831,
0, 0, 0,
16384, 0, 0,
8, 16401, 1048576,
0, 134217728, 0,
0, 0, 0,
0, 0, 0,
0, 0, 0,
4, 14, 0,
0, 0, 0,
0, 0, 16384,
0, 274743689216, 274743691264,
0, 0, 10240,
4096, 0, 0,
134217728, 272, 544,
328, 16384, 8,
2], dtype=uint64),
'fcounters': array([ 2.04900000e+03, -1.00000000e+00, -1.00000000e+00, 0.00000000e+00,
1.64020000e+04, 1.64040000e+04, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00, 0.00000000e+00, -1.00000000e+00, -1.00000000e+00,
0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 2.19902326e+12,
0.00000000e+00, 2.19902326e+12, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00, 1.63840000e+04, 0.00000000e+00, 0.00000000e+00,
8.00000000e+00, 1.64010000e+04, 1.04857600e+06, 0.00000000e+00,
1.34217728e+08, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 4.00000000e+00,
1.40000000e+01, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 1.63840000e+04,
0.00000000e+00, 2.74743689e+11, 2.74743691e+11, 0.00000000e+00,
0.00000000e+00, 1.02400000e+04, 4.09600000e+03, 0.00000000e+00,
0.00000000e+00, 1.34217728e+08, 2.72000000e+02, 5.44000000e+02,
3.28000000e+02, 1.63840000e+04, 8.00000000e+00, 2.00000000e+00])}]
[]
%% Cell type:markdown id: tags:
### Aggregation (Experimental)
Darshan log data is routinely aggregated for quick overview. The report object offers a few methods to perform common aggregations:
%% Cell type:code id: tags:
``` python
report = darshan.DarshanReport("example.darshan")
```
%% Cell type:markdown id: tags:
Report aggregations and summarization remains experimental for now, mostly to allow interfaces to stabilize. Experimental features can be switched on easily by invoking:
%% Cell type:code id: tags:
``` python
darshan.enable_experimental()
```
%%%% Output: stream
Added method create_time_summary to DarshanReport.
Added method summarize to DarshanReport.
Added method create_timeline to DarshanReport.
Added method agg_ioops to DarshanReport.
Added method create_sankey to DarshanReport.
Added method mod_agg_iohist to DarshanReport.
%% Cell type:code id: tags:
``` python
# expected to fail
report.report['agg_ioops']
report.data['agg_ioops']
```
%%%% Output: error
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
<ipython-input-9-20253727a280> in <module>
<ipython-input-11-20253727a280> in <module>
1 # expected to fail
----> 2 report.report['agg_ioops']
KeyError: 'agg_ioops'
%% Cell type:code id: tags:
``` python
report.mod_read_all_records("POSIX")
report.summarize()
```
%% Cell type:code id: tags:
``` python
# attaches among other summarizes the aggregatedioo
report.report['agg_ioops']
report.data['agg_ioops']
```
%%%% Output: execute_result
{'POSIX': {'POSIX_OPENS': 2049,
'POSIX_FILENOS': 18446744073709551615,
'POSIX_DUPS': 18446744073709551615,
'POSIX_READS': 0,
'POSIX_WRITES': 16402,
'POSIX_SEEKS': 16404,
'POSIX_STATS': 0,
'POSIX_MMAPS': 0,
'POSIX_FSYNCS': 0,
'POSIX_FDSYNCS': 0,
'POSIX_RENAME_SOURCES': 18446744073709551615,
'POSIX_RENAME_TARGETS': 18446744073709551615,
'POSIX_RENAMED_FROM': 0,
'POSIX_MODE': 0,
'POSIX_BYTES_READ': 0,
'POSIX_BYTES_WRITTEN': 2199023259968,
'POSIX_MAX_BYTE_READ': 0,
'POSIX_MAX_BYTE_WRITTEN': 2199023261831,
'POSIX_CONSEC_READS': 0,
'POSIX_CONSEC_WRITES': 0,
'POSIX_SEQ_READS': 0,
'POSIX_SEQ_WRITES': 16384,
'POSIX_RW_SWITCHES': 0,
'POSIX_MEM_NOT_ALIGNED': 0,
'POSIX_MEM_ALIGNMENT': 8,
'POSIX_FILE_NOT_ALIGNED': 16401,
'POSIX_FILE_ALIGNMENT': 1048576,
'POSIX_MAX_READ_TIME_SIZE': 0,
'POSIX_MAX_WRITE_TIME_SIZE': 134217728,
'POSIX_SIZE_READ_0_100': 0,
'POSIX_SIZE_READ_100_1K': 0,
'POSIX_SIZE_READ_1K_10K': 0,
'POSIX_SIZE_READ_10K_100K': 0,
'POSIX_SIZE_READ_100K_1M': 0,
'POSIX_SIZE_READ_1M_4M': 0,
'POSIX_SIZE_READ_4M_10M': 0,
'POSIX_SIZE_READ_10M_100M': 0,