Commit ed91ae3c authored by Jakob Luettgau's avatar Jakob Luettgau
Browse files

Have notebooks with some of the expected outputs preserved.

parent e0773049
......@@ -2,8 +2,8 @@
pydarshan
=========
Python utilities to interact with darshan log records of HPC applications.
pydarshan requires darshan-utils (3.2.0+) to be installed.
Python utilities to interact with Darshan log records of HPC applications.
pydarshan requires darshan-utils (3.3.0+) to be installed.
Features
--------
......@@ -17,8 +17,8 @@ Features
Usage
-----
For examples and a jupyter notebook to get started with pydarshan make sure
to check out the `notebooks` subdirectory.
For examples and Jupyter notebooks to get started with pydarshan make sure
to check out the `examples` subdirectory.
A brief examples showing some of the basic functionality is the following::
......
......@@ -11,6 +11,13 @@ from darshan.report import DarshanReport
def enable_experimental(verbose=True):
"""
Enable experimental features such as aggregation methods for reports.
Args:
verbose (bool): Display log of enabled features. (Default: True)
"""
import os
import glob
import importlib
......
......@@ -6,20 +6,14 @@ import numpy
from darshan.discover_darshan import discover_darshan
from darshan.api_def_c import load_darshan_header
DARSHAN_PATH = discover_darshan()
#DARSHAN_PATH = discover_darshan()
API_def_c = load_darshan_header()
ffi = cffi.FFI()
ffi.cdef(API_def_c)
#libdutil = ffi.dlopen(DARSHAN_PATH + "/lib/libdarshan-util.so")
libdutil = ffi.dlopen("libdarshan-util.so")
#libdutil = ffi.dlopen("/home/pq/p/software/darshan-cffi/darshan-util/libdarshan-util.so")
#print(DARSHAN_PATH + "/lib/libdarshan-util.so")
......@@ -184,8 +178,7 @@ def log_get_dxt_record(log, mod_name, mod_type, mode='dict'):
Return:
dict: generic log record
Example
-------
Example:
The typical darshan log record provides two arrays, on for integer counters
and one for floating point counters:
......@@ -260,8 +253,7 @@ def log_get_generic_record(log, mod_name, mod_type, mode='numpy'):
Return:
dict: generic log record
Example
-------
Example:
The typical darshan log record provides two arrays, on for integer counters
and one for floating point counters:
......
darshan.plots package
=====================
Submodules
----------
darshan.plots.matplotlib module
-------------------------------
.. automodule:: darshan.plots.matplotlib
:members:
:undoc-members:
:show-inheritance:
Module contents
---------------
.. automodule:: darshan.plots
:members:
:undoc-members:
:show-inheritance:
......@@ -6,9 +6,7 @@ Subpackages
.. toctree::
darshan.aggregators
darshan.backend
darshan.plots
Submodules
----------
......
......@@ -15,7 +15,10 @@ To use pydarshan in a project::
# read metadata, log records and name records
report.read_all_generic_records()
# Python aggregations are still experimental and have to be activated:
# calculate or update aggregate statistics for currently loaded records
darshan.enable_experimental()
report.summarize()
......@@ -24,7 +27,6 @@ To use pydarshan in a project::
Directly interfacing through the CFFI Interface Wrappers
--------------------------------------------------------
......@@ -34,12 +36,12 @@ If this seems like an unwanted overhead the CFFI interface can be used which all
To use pydarshan.cffi_parser in a project::
import darshan.backends.cffi_backend as darshan
import darshan.backends.cffi_backend as darshanll
log = darshan.log_open("example.darshan")
log = darshanll.log_open("example.darshan")
# Access various job information
darshan.log_get_job(log)
darshanll.log_get_job(log)
# Example Return:
# {'jobid': 4478544,
# 'uid': 69615,
......@@ -49,7 +51,7 @@ To use pydarshan.cffi_parser in a project::
# Access available modules and modules
darshan.log_get_modules(log)
darshanll.log_get_modules(log)
# Example Return:
# {'POSIX': {'len': 186, 'ver': 3, 'idx': 1},
# 'MPI-IO': {'len': 154, 'ver': 2, 'idx': 2},
......@@ -59,10 +61,10 @@ To use pydarshan.cffi_parser in a project::
# Access different record types as numpy arrays, with integer and float counters seperated
# Example Return: {'counters': array([...], dtype=uint64), 'fcounters': array([...])}
posix_record = darshan.log_get_posix_record(log)
mpiio_record = darshan.log_get_mpiio_record(log)
stdio_record = darshan.log_get_stdio_record(log)
posix_record = darshanll.log_get_posix_record(log)
mpiio_record = darshanll.log_get_mpiio_record(log)
stdio_record = darshanll.log_get_stdio_record(log)
# ...
darshan.log_close(log)
darshanll.log_close(log)
......@@ -12,7 +12,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
......@@ -28,9 +28,59 @@
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"execution_count": 3,
"metadata": {
"scrolled": false
},
"outputs": [
{
"data": {
"text/plain": [
"{'version': 1,\n",
" 'records': {},\n",
" 'job': {'jobid': 4478544,\n",
" 'uid': 69615,\n",
" 'start_time': 1490000867,\n",
" 'end_time': 1490000983,\n",
" 'metadata': {'lib_ver': '3.1.3', 'h': 'romio_no_indep_rw=true;cb_nodes=4'}},\n",
" 'exe': '/global/project/projectdirs/m888/glock/tokio-abc-results/bin.edison/vpicio_uni /scratch2/scratchdirs/glock/tokioabc-s.4478544/vpicio/vpicio.hdf5 32',\n",
" 'mounts': [('/.shared/base/default/etc/dat.conf', 'dvs'),\n",
" ('/usr/lib64/libibverbs.so.1.0.0', 'dvs'),\n",
" ('/usr/lib64/libibumad.so.3.0.2', 'dvs'),\n",
" ('/usr/lib64/librdmacm.so.1.0.0', 'dvs'),\n",
" ('/usr/lib64/libibgni.so.1.0.0', 'dvs'),\n",
" ('/global/cscratch1', 'lustre'),\n",
" ('/global/projectb', 'dvs'),\n",
" ('/global/projecta', 'dvs'),\n",
" ('/usr/sbin/ibstat', 'dvs'),\n",
" ('/global/project', 'dvs'),\n",
" ('/global/common', 'dvs'),\n",
" ('/global/syscom', 'dvs'),\n",
" ('/global/dna', 'dvs'),\n",
" ('/opt/slurm', 'dvs'),\n",
" ('/global/u1', 'dvs'),\n",
" ('/global/u2', 'dvs'),\n",
" ('/scratch1', 'lustre'),\n",
" ('/scratch2', 'lustre'),\n",
" ('/scratch3', 'lustre'),\n",
" ('/etc', 'dvs'),\n",
" ('/', 'rootfs'),\n",
" ('/', 'dvs')],\n",
" 'modules': {'POSIX': {'len': 186, 'ver': 3, 'idx': 1},\n",
" 'MPI-IO': {'len': 154, 'ver': 2, 'idx': 2},\n",
" 'LUSTRE': {'len': 87, 'ver': 1, 'idx': 6},\n",
" 'STDIO': {'len': 3234, 'ver': 1, 'idx': 7}},\n",
" 'name_records': {14734109647742566553: '<STDIN>',\n",
" 15920181672442173319: '<STDOUT>',\n",
" 7238257241479193519: '<STDERR>',\n",
" 6301063301082038805: '/scratch2/scratchdirs/glock/tokioabc-s.4478544/vpicio/vpicio.hdf5'}}"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"report.report"
]
......@@ -46,31 +96,94 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 4,
"metadata": {},
"outputs": [],
"outputs": [
{
"ename": "KeyError",
"evalue": "'POSIX'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-4-3a9a112f3a8d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;31m# expected to fail\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mreport\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreport\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'records'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'POSIX'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;31mKeyError\u001b[0m: 'POSIX'"
]
}
],
"source": [
"# expected to fail\n",
"report.report['records']['POSIX']"
]
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"report.mod_read_all_records(\"MPI-IO\") "
"report.mod_read_all_records(\"POSIX\") "
]
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 6,
"metadata": {
"scrolled": false
},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"[{'id': 6301063301082038805,\n",
" 'rank': -1,\n",
" 'counters': array([ 2049, 18446744073709551615, 18446744073709551615,\n",
" 0, 16402, 16404,\n",
" 0, 0, 0,\n",
" 0, 18446744073709551615, 18446744073709551615,\n",
" 0, 0, 0,\n",
" 2199023259968, 0, 2199023261831,\n",
" 0, 0, 0,\n",
" 16384, 0, 0,\n",
" 8, 16401, 1048576,\n",
" 0, 134217728, 0,\n",
" 0, 0, 0,\n",
" 0, 0, 0,\n",
" 0, 0, 0,\n",
" 4, 14, 0,\n",
" 0, 0, 0,\n",
" 0, 0, 16384,\n",
" 0, 274743689216, 274743691264,\n",
" 0, 0, 10240,\n",
" 4096, 0, 0,\n",
" 134217728, 272, 544,\n",
" 328, 16384, 8,\n",
" 2], dtype=uint64),\n",
" 'fcounters': array([ 2.04900000e+03, -1.00000000e+00, -1.00000000e+00, 0.00000000e+00,\n",
" 1.64020000e+04, 1.64040000e+04, 0.00000000e+00, 0.00000000e+00,\n",
" 0.00000000e+00, 0.00000000e+00, -1.00000000e+00, -1.00000000e+00,\n",
" 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 2.19902326e+12,\n",
" 0.00000000e+00, 2.19902326e+12, 0.00000000e+00, 0.00000000e+00,\n",
" 0.00000000e+00, 1.63840000e+04, 0.00000000e+00, 0.00000000e+00,\n",
" 8.00000000e+00, 1.64010000e+04, 1.04857600e+06, 0.00000000e+00,\n",
" 1.34217728e+08, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
" 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
" 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 4.00000000e+00,\n",
" 1.40000000e+01, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
" 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 1.63840000e+04,\n",
" 0.00000000e+00, 2.74743689e+11, 2.74743691e+11, 0.00000000e+00,\n",
" 0.00000000e+00, 1.02400000e+04, 4.09600000e+03, 0.00000000e+00,\n",
" 0.00000000e+00, 1.34217728e+08, 2.72000000e+02, 5.44000000e+02,\n",
" 3.28000000e+02, 1.63840000e+04, 8.00000000e+00, 2.00000000e+00])}]"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"report.report"
"report.report['records']['POSIX']"
]
},
{
......@@ -84,7 +197,7 @@
},
{
"cell_type": "code",
"execution_count": 13,
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
......@@ -109,7 +222,10 @@
"text": [
"Added method create_time_summary to DarshanReport.\n",
"Added method summarize to DarshanReport.\n",
"Added method create_sankey to DarshanReport.\n"
"Added method create_timeline to DarshanReport.\n",
"Added method agg_ioops to DarshanReport.\n",
"Added method create_sankey to DarshanReport.\n",
"Added method mod_agg_iohist to DarshanReport.\n"
]
}
],
......@@ -153,9 +269,9 @@
},
{
"cell_type": "code",
"execution_count": 12,
"execution_count": 11,
"metadata": {
"scrolled": false
"scrolled": true
},
"outputs": [
{
......@@ -234,7 +350,7 @@
" 'Fsync': 0}}"
]
},
"execution_count": 12,
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
......
......@@ -6,9 +6,6 @@ from setuptools import setup, find_packages
with open('README.rst') as readme_file:
readme = readme_file.read()
with open('HISTORY.rst') as history_file:
history = history_file.read()
requirements = ['cffi', 'numpy', 'matplotlib']
setup_requirements = ['pytest-runner', ]
......@@ -32,7 +29,7 @@ setup(
],
description="Python tools to interact with darshan log records of HPC applications.",
install_requires=requirements,
long_description=readme + '\n\n' + history,
long_description=readme,
include_package_data=True,
keywords='darshan',
name='darshan',
......
[tox]
envlist = py27, py34, py35, py36, flake8
envlist = py34, py35, py36, flake8
[travis]
python =
3.6: py36
3.5: py35
3.4: py34
2.7: py27
[testenv:flake8]
basepython = python
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment