Skip to content

GitLab

Commit 9c4ed094 authored by Jakob Luettgau's avatar Jakob Luettgau
Browse files

Update notebooks and part of the documention.

parent 94d3a016
Hide whitespace changes
Inline Side-by-side
Showing with 87 additions and 148 deletions
darshan-util/pydarshan/docs/Makefile
View file @ 9c4ed094
......@@ -6,7 +6,7 @@ SPHINXOPTS =
SPHINXBUILD = python -m sphinx
SPHINXPROJ = pydarshan
SOURCEDIR = .
BUILDDIR = _build
BUILDDIR = build
# Put it first so that "make" without argument is like "make help".
help:
......
darshan-util/pydarshan/docs/conf.py
View file @ 9c4ed094
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# pydarshan documentation build configuration file, created by
# Documentation build configuration file, created by
# sphinx-quickstart on Fri Jun 9 13:47:02 2017.
#
# This file is execfile()d with the current directory set to its
......@@ -63,7 +63,7 @@ source_suffix = '.rst'
master_doc = 'index'
# General information about the project.
project = u'darshan'
project = u'Darshan'
copyright = u"2019, ANL"
author = u""
......@@ -128,9 +128,7 @@ htmlhelp_basename = 'darshandoc'
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
(master_doc, 'pydarshan',
u'pydarshan Documentation',
[author], 1)
(master_doc, 'pydarshan', u'pydarshan Documentation', [author], 1)
]
......@@ -159,9 +157,7 @@ latex_elements = {
# (source start file, target name, title, author, documentclass
# [howto, manual, or own class]).
latex_documents = [
(master_doc, 'pydarshan.tex',
u'pydarshan Documentation',
u'Argonne National Laboratory', 'manual'),
(master_doc, 'pydarshan.tex', u'pydarshan Documentation', u'Argonne National Laboratory', 'manual'),
]
......@@ -171,12 +167,7 @@ latex_documents = [
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
(master_doc, 'pydarshan',
u'pydarshan Documentation',
author,
'pydarshan',
'One line description of project.',
'Miscellaneous'),
(master_doc, 'pydarshan', u'pydarshan Documentation', author, 'pydarshan', 'Python utilities for Darshan HPC I/O instrumentation.', 'Miscellaneous'),
]
......
darshan-util/pydarshan/docs/darshan.backend.rst
View file @ 9c4ed094
......@@ -4,14 +4,6 @@ darshan.backend package
Submodules
----------
darshan.backend.ascii\_backend module
-------------------------------------
.. automodule:: darshan.backend.ascii_backend
:members:
:undoc-members:
:show-inheritance:
darshan.backend.cffi\_backend module
------------------------------------
......
darshan-util/pydarshan/docs/darshan.rst
View file @ 9c4ed094
......@@ -7,6 +7,7 @@ Subpackages
.. toctree::
darshan.backend
darshan.cli
darshan.experimental
Submodules
......
darshan-util/pydarshan/docs/index.rst
View file @ 9c4ed094
......@@ -8,7 +8,7 @@ This documenation is only for the Darshan Python bindings, for documentation for
:caption: Contents:
readme
installation
setup
usage
.. toctree::
......
darshan-util/pydarshan/docs/installation.rst deleted 100644 → 0
View file @ 94d3a016
.. highlight:: shell
============
Installation
============
Stable release
--------------
To install pydarshan, run this command in your terminal:
.. code-block:: console
$ pip install pydarshan
This is the preferred method to install pydarshan, as it will always install the most recent stable release.
If you don't have `pip`_ installed, this `Python installation guide`_ can guide
you through the process.
.. _pip: https://pip.pypa.io
.. _Python installation guide: https://www.mcs.anl.gov/research/projects/darshan/
Pydarshan assumes that a recent 'darshan-utils' is installed as a shared
library. If darshan-util is not installed consult with the darshan
documentation or consider using `Spack`_ to install::
spack install darshan-util
.. _Spack: https://spack.io/
From sources
------------
The sources for pydarshan can be downloaded from the `Github repo`_.
You can either clone the public repository:
.. code-block:: console
$ git clone https://xgitlab.cels.anl.gov/darshan/darshan.git
$ cd darshan/darshan-util/pydarshan
.. code-block:: console
$ python setup.py install
.. _Github repo: https://xgitlab.cels.anl.gov/darshan/darshan
darshan-util/pydarshan/examples/01_darshan-introduction.ipynb
View file @ 9c4ed094
......@@ -18,14 +18,14 @@
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Filename: example.darshan\n",
"Filename: example-logs/example.darshan\n",
"Times: 2017-03-20 10:07:47 to 2017-03-20 10:09:43 (Duration 0:01:56)\n",
"Executeable: /global/project/projectdirs/m888/glock/tokio-abc-results/bin.edison/vpicio_uni /scratch2/scratchdirs/glock/tokioabc-s.4478544/vpicio/vpicio.hdf5 32\n",
"Processes: 2048\n",
......@@ -35,20 +35,20 @@
"Loaded Records: {'POSIX': 1, 'MPI-IO': 1, 'STDIO': 129}\n",
"Name Records: 4\n",
"Darshan/Hints: {'lib_ver': '3.1.3', 'h': 'romio_no_indep_rw=true;cb_nodes=4'}\n",
"DarshanReport: id(140525897881584) (tmp)\n"
"DarshanReport: id(139721554559216) (tmp)\n"
]
}
],
"source": [
"import darshan\n",
"\n",
"report = darshan.DarshanReport(\"example.darshan\", read_all=True) # Default behavior\n",
"report = darshan.DarshanReport(\"example-logs/example.darshan\", read_all=True) # Default behavior\n",
"report.info()"
]
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 9,
"metadata": {},
"outputs": [
{
......@@ -60,7 +60,7 @@
" 'STDIO': {'len': 3234, 'ver': 1, 'idx': 8, 'num_records': 129}}"
]
},
"execution_count": 2,
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
......@@ -78,16 +78,16 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 10,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<bound method DarshanReport.mod_records of <darshan.report.DarshanReport object at 0x7fcebc3997f0>>"
"<bound method DarshanReport.mod_records of <darshan.report.DarshanReport object at 0x7f1375a080f0>>"
]
},
"execution_count": 3,
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
......@@ -105,7 +105,7 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 11,
"metadata": {
"scrolled": true
},
......@@ -126,7 +126,7 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 12,
"metadata": {
"scrolled": true
},
......@@ -137,7 +137,7 @@
"dict_keys(['version', 'metadata', 'records', 'summary', 'modules', 'counters', 'name_records', 'mounts'])"
]
},
"execution_count": 5,
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
......@@ -148,7 +148,7 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
......@@ -157,7 +157,7 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
......@@ -166,7 +166,7 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 15,
"metadata": {},
"outputs": [
{
......@@ -176,7 +176,7 @@
"POSIX\n",
"MPI-IO\n",
"STDIO\n",
"Filename: example.darshan\n",
"Filename: example-logs/example.darshan\n",
"Times: 2017-03-20 10:07:47 to 2017-03-20 10:09:43 (Duration 0:01:56)\n",
"Executeable: /global/project/projectdirs/m888/glock/tokio-abc-results/bin.edison/vpicio_uni /scratch2/scratchdirs/glock/tokioabc-s.4478544/vpicio/vpicio.hdf5 32\n",
"Processes: 2048\n",
......@@ -186,7 +186,7 @@
"Loaded Records: {'POSIX': 1, 'MPI-IO': 1, 'STDIO': 129}\n",
"Name Records: 3\n",
"Darshan/Hints: {'lib_ver': '3.1.3', 'h': 'romio_no_indep_rw=true;cb_nodes=4'}\n",
"DarshanReport: id(140525897881584) (tmp)\n"
"DarshanReport: id(139721554559216) (tmp)\n"
]
}
],
......@@ -197,7 +197,7 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
......@@ -216,16 +216,16 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
"report = darshan.DarshanReport(\"example.darshan\", read_all=False) # Loads no records!"
"report = darshan.DarshanReport(\"example-logs/example.darshan\", read_all=False) # Loads no records!"
]
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 18,
"metadata": {},
"outputs": [
{
......@@ -253,7 +253,7 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 19,
"metadata": {},
"outputs": [],
"source": [
......@@ -262,7 +262,7 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 20,
"metadata": {},
"outputs": [
{
......@@ -271,7 +271,7 @@
"129"
]
},
"execution_count": 8,
"execution_count": 20,
"metadata": {},
"output_type": "execute_result"
}
......@@ -282,14 +282,14 @@
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": 21,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Filename: example.darshan\n",
"Filename: example-logs/example.darshan\n",
"Times: 2017-03-20 10:07:47 to 2017-03-20 10:09:43 (Duration 0:01:56)\n",
"Executeable: /global/project/projectdirs/m888/glock/tokio-abc-results/bin.edison/vpicio_uni /scratch2/scratchdirs/glock/tokioabc-s.4478544/vpicio/vpicio.hdf5 32\n",
"Processes: 2048\n",
......@@ -297,9 +297,9 @@
"UID: 69615\n",
"Modules in Log: ['POSIX', 'MPI-IO', 'LUSTRE', 'STDIO']\n",
"Loaded Records: {'STDIO': 129}\n",
"Name Records: 4\n",
"Name Records: 0\n",
"Darshan/Hints: {'lib_ver': '3.1.3', 'h': 'romio_no_indep_rw=true;cb_nodes=4'}\n",
"DarshanReport: id(139750934822192) (tmp)\n"
"DarshanReport: id(139721553801904) (tmp)\n"
]
}
],
......@@ -309,7 +309,7 @@
},
{
"cell_type": "code",
"execution_count": 13,
"execution_count": 22,
"metadata": {},
"outputs": [
{
......@@ -317,16 +317,15 @@
"text/plain": [
"{'id': 15920181672442173319,\n",
" 'rank': 0,\n",
" 'counters': array([ 1, 18446744073709551615, 0,\n",
" 6, 0, 0,\n",
" 280, 0, 0,\n",
" 279, 0, 0,\n",
" 0], dtype=uint64),\n",
" 'fcounters': array([ 1., -1., 0., 6., 0., 0., 280., 0., 0., 279., 0.,\n",
" 0., 0.])}"
" 'counters': array([ 1, -1, 0, 6, 0, 0, 280, 0, 0, 279, 0, 0, 0,\n",
" 0]),\n",
" 'fcounters': array([0.00000000e+00, 6.79492950e-05, 0.00000000e+00, 0.00000000e+00,\n",
" 0.00000000e+00, 7.75279999e-02, 0.00000000e+00, 0.00000000e+00,\n",
" 0.00000000e+00, 1.16283583e+02, 0.00000000e+00, 0.00000000e+00,\n",
" 0.00000000e+00, 0.00000000e+00, 0.00000000e+00])}"
]
},
"execution_count": 13,
"execution_count": 22,
"metadata": {},
"output_type": "execute_result"
}
......@@ -353,7 +352,7 @@
},
{
"cell_type": "code",
"execution_count": 22,
"execution_count": 23,
"metadata": {},
"outputs": [
{
......@@ -439,7 +438,7 @@
"import darshan\n",
"darshan.enable_experimental()\n",
"\n",
"report = darshan.DarshanReport(\"example.darshan\", read_all=True)\n",
"report = darshan.DarshanReport(\"example-logs/example.darshan\", read_all=True)\n",
"report.name_records"
]
},
......@@ -870,8 +869,8 @@
"from darshan.experimental.plots.matplotlib import plot_access_histogram\n",
"from darshan.experimental.plots.matplotlib import plot_opcounts\n",
"\n",
"r1 = darshan.DarshanReport(\"example.darshan\", read_all=True)\n",
"r2 = darshan.DarshanReport(\"example2.darshan\", read_all=True)\n",
"r1 = darshan.DarshanReport(\"example-logs/example.darshan\", read_all=True)\n",
"r2 = darshan.DarshanReport(\"example-logs/example2.darshan\", read_all=True)\n",
"rx = r1 + r2\n",
"\n",
"for r in [r1, r2, rx]:\n",
......@@ -926,7 +925,7 @@
},
{
"cell_type": "code",
"execution_count": 18,
"execution_count": 7,
"metadata": {},
"outputs": [
{
......@@ -942,7 +941,7 @@
"<module 'matplotlib.pyplot' from '/home/pq/ANL/darshan-decaf/testbed/conda/lib/python3.6/site-packages/matplotlib/pyplot.py'>"
]
},
"execution_count": 18,
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
},
......@@ -963,7 +962,7 @@
"import darshan\n",
"darshan.enable_experimental(verbose=False)\n",
"\n",
"r3 = darshan.DarshanReport(\"example.darshan\")\n",
"r3 = darshan.DarshanReport(\"example-logs/example.darshan\")\n",
"r3.mod_read_all_records('POSIX')\n",
"\n",
"from darshan.experimental.plots.matplotlib import plot_access_histogram\n",
......@@ -1009,7 +1008,7 @@
"import darshan\n",
"darshan.enable_experimental(verbose=False)\n",
"\n",
"r3 = darshan.DarshanReport(\"example.darshan\")\n",
"r3 = darshan.DarshanReport(\"example-logs/example.darshan\")\n",
"r3.mod_read_all_records('MPI-IO')\n",
"\n",
"from darshan.experimental.plots.matplotlib import plot_access_histogram\n",
......@@ -1018,7 +1017,7 @@
},
{
"cell_type": "code",
"execution_count": 20,
"execution_count": 6,
"metadata": {
"scrolled": true
},
......@@ -1042,7 +1041,7 @@
"<module 'matplotlib.pyplot' from '/home/pq/ANL/darshan-decaf/testbed/conda/lib/python3.6/site-packages/matplotlib/pyplot.py'>"
]
},
"execution_count": 20,
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
},
......@@ -1063,7 +1062,7 @@
"import darshan\n",
"darshan.enable_experimental(verbose=False)\n",
"\n",
"r3 = darshan.DarshanReport(\"example.darshan\")\n",
"r3 = darshan.DarshanReport(\"example-logs/example.darshan\")\n",
"r3.read_all()\n",
"\n",
"from darshan.experimental.plots.matplotlib import plot_opcounts\n",
......@@ -1081,7 +1080,7 @@
},
{
"cell_type": "code",
"execution_count": 21,
"execution_count": 5,
"metadata": {},
"outputs": [
{
......@@ -1093,7 +1092,7 @@
}
],
"source": [
"report2 = darshan.DarshanReport(\"example.darshan\")\n",
"report2 = darshan.DarshanReport(\"example-logs/example.darshan\")\n",
"report2.mod_read_all_dxt_records(\"ABC\") "
]
},
......@@ -1122,7 +1121,7 @@
"outputs": [],
"source": [
"import darshan\n",
"report = darshan.DarshanReport(\"example.darshan\", read_all=True)\n",
"report = darshan.DarshanReport(\"example-logs/example.darshan\", read_all=True)\n",
"report.to_json()"
]
},
......@@ -1141,30 +1140,41 @@
},
{
"cell_type": "code",
"execution_count": 24,
"execution_count": 1,
"metadata": {
"scrolled": true
},
"outputs": [],
"outputs": [
{
"ename": "NameError",
"evalue": "name 'darshan' is not defined",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-1-0be0e3f32711>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mreport\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdarshan\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDarshanReport\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"example-logs/example.darshan\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;31mNameError\u001b[0m: name 'darshan' is not defined"
]
}
],
"source": [
"report = darshan.DarshanReport(\"example.darshan\")"
"report = darshan.DarshanReport(\"example-logs/example.darshan\")"
]
},
{
"cell_type": "code",
"execution_count": 25,
"execution_count": 2,
"metadata": {},
"outputs": [
{
"ename": "KeyError",
"evalue": "'ABC'",
"ename": "NameError",
"evalue": "name 'report' is not defined",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-25-ad9bffc53533>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mreport\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmod_read_all_records\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"ABC\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# Expect KeyError\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;32m~/ANL/darshan-decaf/darshan-workflow/python-packages/darshan/report.py\u001b[0m in \u001b[0;36mmod_read_all_records\u001b[0;34m(self, mod, mode, warnings)\u001b[0m\n\u001b[1;32m 305\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 306\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 307\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmod\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'num_records'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 308\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 309\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mKeyError\u001b[0m: 'ABC'"
"\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-2-ad9bffc53533>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mreport\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmod_read_all_records\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"ABC\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# Expect KeyError\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;31mNameError\u001b[0m: name 'report' is not defined"
]
}
],
......
darshan-util/pydarshan/examples/02_darshan-plotting-matplotlib.ipynb
View file @ 9c4ed094
......@@ -31,8 +31,8 @@
"from darshan.experimental.plots.matplotlib import plot_opcounts\n",
"\n",
"# prepare some reports\n",
"r1 = darshan.DarshanReport(\"example.darshan\", read_all=True)\n",
"r2 = darshan.DarshanReport(\"example2.darshan\", read_all=True)\n",
"r1 = darshan.DarshanReport(\"example-logs/example.darshan\", read_all=True)\n",
"r2 = darshan.DarshanReport(\"example-logs/example2.darshan\", read_all=True)\n",
"rx = r1 + r2"
]
},
......
darshan-util/pydarshan/examples/04_darshan-low-level-cffi-interface.ipynb
View file @ 9c4ed094
......@@ -15,7 +15,7 @@
"metadata": {},
"outputs": [],
"source": [
"logname = \"example.darshan\""
"logpath = \"example-logs/example.darshan\""
]
},
{
......@@ -41,7 +41,7 @@
"metadata": {},
"outputs": [],
"source": [
"log1 = backend.log_open(logname)"
"log1 = backend.log_open(logpath)"
]
},
{
......
darshan-util/pydarshan/examples/99_darshan-experimental_report-algebra.ipynb
View file @ 9c4ed094
......@@ -72,9 +72,9 @@
"metadata": {},
"outputs": [],
"source": [
"r1 = darshan.DarshanReport(\"sample.darshan\", read_all=True)\n",
"r2 = darshan.DarshanReport(\"noposix.darshan\", read_all=True)\n",
"r3 = darshan.DarshanReport(\"sample-badost.darshan\", read_all=True)"
"r1 = darshan.DarshanReport(\"example-logs/sample.darshan\", read_all=True)\n",
"r2 = darshan.DarshanReport(\"example-logs/noposix.darshan\", read_all=True)\n",
"r3 = darshan.DarshanReport(\"example-logs/sample-badost.darshan\", read_all=True)"
]
},
{
......@@ -506,7 +506,7 @@
"import darshan\n",
"darshan.enable_experimental()\n",
"\n",
"r3 = darshan.DarshanReport(\"example.darshan\")\n",
"r3 = darshan.DarshanReport(\"example-logs/example.darshan\")\n",
"r3.mod_read_all_records('POSIX')\n",
"r3.mod_read_all_records('MPI-IO')\n",
"\n",
......