Commit 8190931a authored by Jakob Luettgau's avatar Jakob Luettgau
Browse files

Have pydarshan use pythons logging facilities.

parent d30ee690
......@@ -6,6 +6,10 @@ import ctypes
import numpy as np
import pandas as pd
import logging
logger = logging.getLogger(__name__)
from darshan.api_def_c import load_darshan_header
from darshan.discover_darshan import find_utils
from darshan.discover_darshan import check_version
......@@ -38,9 +42,6 @@ _structdefs = {
def get_lib_version():
"""
Return the version information hardcoded into the shared library.
......
......@@ -5,6 +5,8 @@
import os
import logging
logger = logging.getLogger(__name__)
......@@ -16,21 +18,24 @@ def check_version(ffi=None, libdutil=None):
"""
lib_version = None
# query library directly (preferred)
if ffi is not None and libdutil is not None:
ver = ffi.new("char **")
ver = libdutil.darshan_log_get_lib_version()
lib_version = ffi.string(ver).decode("utf-8")
logger.debug(f" Found lib_version={lib_version} via ffi.")
# pkgconfig fallback
if lib_version is None:
print("WARNING: Using pk-config fallback.")
logger.warning("WARNING: Using pk-config fallback.")
import subprocess
args = ['pkg-config', '--modversion', 'darshan-util']
p = subprocess.Popen(args, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd='.')
out,err = p.communicate()
retval = p.wait()
lib_version = out.decode('ascii').strip()
logger.debug(f" Found lib_version={lib_version} via pkgconfig.")
import darshan
......@@ -132,7 +137,7 @@ def load_darshan_header():
filepath = os.path.join(curdir, 'data', 'darshan-api.h')
# filepath = os.path.join(curdir, 'data', 'generated.h')
print(filepath)
logger.debug(f" load_darshan_header using filepath={filepath}")
with open(filepath, 'r') as f:
try:
......
......@@ -18,8 +18,8 @@ import sys
import numpy as np
import pandas as pd
import logging
logger = logging.getLogger(__name__)
......@@ -61,7 +61,9 @@ class DarshanReport(object):
a number of common aggregations can be performed.
"""
def __init__(self, filename=None, data_format='pandas', automatic_summary=False,
def __init__(self,
filename=None, data_format='pandas',
automatic_summary=False,
read_all=True, lookup_name_records=True):
self.filename = filename
......@@ -229,7 +231,7 @@ class DarshanReport(object):
ids = set()
for mod in mods:
print(mod)
logger.debug(f" Refreshing name_records for mod={mod}")
for rec in self.records[mod]:
ids.add(rec['id'])
......@@ -247,8 +249,11 @@ class DarshanReport(object):
Return:
None
"""
self.read_all_generic_records()
self.read_all_dxt_records()
self.mod_read_all_lustre_records()
return
......@@ -284,6 +289,9 @@ class DarshanReport(object):
pass
def mod_read_all_records(self, mod, dtype='numpy', warnings=True):
"""
Reads all generic records for module
......@@ -300,7 +308,7 @@ class DarshanReport(object):
if mod in unsupported:
if warnings:
print("Skipping. Currently unsupported:", mod, "in mod_read_all_records().", file=sys.stderr)
logger.warning(f" Skipping. Currently unsupported: {mod} in mod_read_all_records().")
# skip mod
return
......@@ -383,7 +391,7 @@ class DarshanReport(object):
"""
if mod not in self.data['modules']:
if warnings:
print("Skipping. Log does not contain data for mod:", mod, file=sys.stderr)
logger.warning(f"Skipping. Log does not contain data for mod: {mod}")
return
......@@ -391,7 +399,7 @@ class DarshanReport(object):
if mod not in supported:
if warnings:
print("Skipping. Currently unsupported:", mod, 'in mod_read_all_dxt_records().', file=sys.stderr)
logger.warning(f" Skipping. Unsupported module: {mod} in in mod_read_all_dxt_records(). Supported: {supported}")
# skip mod
return
......@@ -423,6 +431,55 @@ class DarshanReport(object):
pass
def mod_read_all_lustre_records(self, mod="LUSTRE", dtype='numpy', warnings=True):
"""
Reads all dxt records for provided module.
Args:
mod (str): Identifier of module to fetch all records
dtype (str): 'numpy' for ndarray (default), 'dict' for python dictionary
Return:
None
"""
if mod not in self.data['modules']:
if warnings:
logger.warning(f" Skipping. Log does not contain data for mod: {mod}")
return
supported = ['LUSTRE']
if mod not in supported:
if warnings:
logger.warning(f" Skipping. Unsupported module: {mod} in in mod_read_all_dxt_records(). Supported: {supported}")
# skip mod
return
self.records[mod] = []
self.modules[mod]['num_records'] = 0
if mod not in self.counters:
self.counters[mod] = {}
rec = backend.log_get_record(self.log, mod, dtype=dtype)
while rec != None:
self.records[mod].append(rec)
self.data['modules'][mod]['num_records'] += 1
# fetch next
rec = backend.log_get_record(self.log, mod, dtype=dtype)
pass
def mod_records(self, mod, dtype='numpy', warnings=True):
"""
Return generator for lazy record loading and traversal.
......@@ -562,7 +619,16 @@ class DarshanReport(object):
recs = data['records']
for mod in recs:
for i, rec in enumerate(data['records'][mod]):
recs[mod][i]['counters'] = rec['counters'].tolist()
recs[mod][i]['fcounters'] = rec['fcounters'].tolist()
try:
recs[mod][i]['counters'] = rec['counters'].tolist()
except KeyError:
logger.debug(f" to_json: mod={mod} does not include counters")
pass
try:
recs[mod][i]['fcounters'] = rec['fcounters'].tolist()
except KeyError:
logger.debug(f" to_json: mod={mod} does not include fcounters")
pass
return json.dumps(data, cls=DarshanReportJSONEncoder)
%% Cell type:markdown id: tags:
# Debugging and Logging
PyDarshan internally uses Pythons `logging` interfaces for troubleshooting and debugging purposes with most outputs supressed by default.
Applications that are using PyDarshan can access and redirect logging streams of various PyDarshan components as documented by the logging module:
* https://docs.python.org/3/library/logging.html
This notebook demonstrates how logging can be activated, as well as more advanced configuration to redirect streams to files or other destinations. To display PyDarshan logging information in this notebook, one way is to set the log_level to DEBUG for the entire application as follows:
%% Cell type:code id: tags:
``` python
# Enable logging (workaround necessary to ensure logging.getLogger().handlers includes stderr stream)
# https://stackoverflow.com/questions/35326814/change-level-logged-to-ipython-jupyter-notebook
%config Application.log_level="this-workaround-ensures-stderr-is-in-handlers"
import logging
logging.getLogger().setLevel(logging.DEBUG)
```
%%%% Output: stream
ERROR:root:The 'log_level' trait of an IPKernelApp instance must be any of (0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL'), but a value of 'this-workaround-ensures-stderr-is-in-handlers' <class 'str'> was specified.
%% Cell type:code id: tags:
``` python
```
%% Cell type:code id: tags:
``` python
import darshan
report = darshan.DarshanReport("example-logs/example.darshan", read_all=True) # Default behavior
report.records['LUSTRE']
```
%%%% Output: stream
DEBUG:darshan.report: Refreshing name_records for mod=POSIX
DEBUG:darshan.report: Refreshing name_records for mod=POSIX
DEBUG:darshan.report: Refreshing name_records for mod=MPI-IO
DEBUG:darshan.report: Refreshing name_records for mod=POSIX
DEBUG:darshan.report: Refreshing name_records for mod=MPI-IO
DEBUG:darshan.report: Refreshing name_records for mod=STDIO
%%%% Output: execute_result
[{'id': 6301063301082038805,
'rank': -1,
'counters': array([ 24, 1, 0, 1048576, 24]),
'ost_ids': array([ 7, 9, 23, 21, 1, 19, 20, 8, 18, 12, 6, 2, 10, 16, 4, 0, 22,
14, 13, 17, 5, 15, 11, 3])}]
%% Cell type:code id: tags:
``` python
# Some commands will always display warnings when used with incompatible requests arguments:
```
%% Cell type:code id: tags:
``` python
import darshan
report.mod_read_all_records(mod='DXT_POSIX')
```
%%%% Output: stream
WARNING:darshan.report: Skipping. Currently unsupported: DXT_POSIX in mod_read_all_records().
%% Cell type:code id: tags:
``` python
import darshan
report.mod_read_all_dxt_records(mod='DXT_POSIX')
```
%%%% Output: stream
WARNING:darshan.report:Skipping. Log does not contain data for mod: DXT_POSIX
%% Cell type:code id: tags:
``` python
import darshan
report.mod_read_all_dxt_records(mod='POSIX')
```
%%%% Output: stream
WARNING:darshan.report: Skipping. Unsupported module: POSIX in in mod_read_all_dxt_records(). Supported: ['DXT_POSIX', 'DXT_MPIIO']
%% Cell type:code id: tags:
``` python
# Some commands such as export to_json offer debug information only if log_level is set to DEBUG
```
%% Cell type:code id: tags:
``` python
import darshan
report = darshan.DarshanReport("example-logs/example.darshan", read_all=True)
report.to_json()
```
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment