Update notebooks and part of the documention.

darshan-util/pydarshan/docs/Makefile

@@ -6,7 +6,7 @@ SPHINXOPTS    =
 SPHINXBUILD   = python -m sphinx
 SPHINXPROJ    = pydarshan
 SOURCEDIR     = .
-BUILDDIR      = _build
+BUILDDIR      = build
 
 # Put it first so that "make" without argument is like "make help".
 help:

darshan-util/pydarshan/docs/conf.py

 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
-# pydarshan documentation build configuration file, created by
+# Documentation build configuration file, created by
 # sphinx-quickstart on Fri Jun  9 13:47:02 2017.
 #
 # This file is execfile()d with the current directory set to its
@@ -63,7 +63,7 @@ source_suffix = '.rst'
 master_doc = 'index'
 
 # General information about the project.
-project = u'darshan'
+project = u'Darshan'
 copyright = u"2019, ANL"
 author = u""
 
@@ -128,9 +128,7 @@ htmlhelp_basename = 'darshandoc'
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [
-    (master_doc, 'pydarshan',
-     u'pydarshan Documentation',
-     [author], 1)
+    (master_doc, 'pydarshan', u'pydarshan Documentation', [author], 1)
 ]
 
 
@@ -159,9 +157,7 @@ latex_elements = {
 # (source start file, target name, title, author, documentclass
 # [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, 'pydarshan.tex',
-     u'pydarshan Documentation',
-     u'Argonne National Laboratory', 'manual'),
+    (master_doc, 'pydarshan.tex', u'pydarshan Documentation', u'Argonne National Laboratory', 'manual'),
 ]
 
 
@@ -171,12 +167,7 @@ latex_documents = [
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    (master_doc, 'pydarshan',
-     u'pydarshan Documentation',
-     author,
-     'pydarshan',
-     'One line description of project.',
-     'Miscellaneous'),
+    (master_doc, 'pydarshan', u'pydarshan Documentation', author, 'pydarshan', 'Python utilities for Darshan HPC I/O instrumentation.', 'Miscellaneous'),
 ]
 
 

darshan-util/pydarshan/docs/darshan.backend.rst

@@ -4,14 +4,6 @@ darshan.backend package
 Submodules
 ----------
 
-darshan.backend.ascii\_backend module
--------------------------------------
-
-.. automodule:: darshan.backend.ascii_backend
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 darshan.backend.cffi\_backend module
 ------------------------------------
 

darshan-util/pydarshan/docs/darshan.rst

@@ -7,6 +7,7 @@ Subpackages
 .. toctree::
 
     darshan.backend
+    darshan.cli
     darshan.experimental
 
 Submodules

darshan-util/pydarshan/docs/index.rst

@@ -8,7 +8,7 @@ This documenation is only for the Darshan Python bindings, for documentation for
    :caption: Contents:
 
    readme
-   installation
+   setup
    usage
 
 .. toctree::

darshan-util/pydarshan/docs/installation.rst

-.. highlight:: shell
-
-============
-Installation
-============
-
-
-Stable release
---------------
-
-To install pydarshan, run this command in your terminal:
-
-.. code-block:: console
-
-    $ pip install pydarshan
-
-This is the preferred method to install pydarshan, as it will always install the most recent stable release.
-If you don't have `pip`_ installed, this `Python installation guide`_ can guide
-you through the process.
-
-.. _pip: https://pip.pypa.io
-.. _Python installation guide: https://www.mcs.anl.gov/research/projects/darshan/
-
-
-Pydarshan assumes that a recent 'darshan-utils' is installed as a shared 
-library. If darshan-util is not installed consult with the darshan
-documentation or consider using `Spack`_ to install::
-
-    spack install darshan-util
-
-
-.. _Spack: https://spack.io/
-
-
-From sources
-------------
-
-The sources for pydarshan can be downloaded from the `Github repo`_.
-
-You can either clone the public repository:
-
-.. code-block:: console
-
-    $ git clone https://xgitlab.cels.anl.gov/darshan/darshan.git
-    $ cd darshan/darshan-util/pydarshan
-
-
-.. code-block:: console
-
-    $ python setup.py install
-
-
-.. _Github repo: https://xgitlab.cels.anl.gov/darshan/darshan

darshan-util/pydarshan/examples/01_darshan-introduction.ipynb

@@ -18,14 +18,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Filename:       example.darshan\n",
+      "Filename:       example-logs/example.darshan\n",
       "Times:          2017-03-20 10:07:47 to 2017-03-20 10:09:43 (Duration 0:01:56)\n",
       "Executeable:    /global/project/projectdirs/m888/glock/tokio-abc-results/bin.edison/vpicio_uni /scratch2/scratchdirs/glock/tokioabc-s.4478544/vpicio/vpicio.hdf5 32\n",
       "Processes:      2048\n",
@@ -35,20 +35,20 @@
       "Loaded Records: {'POSIX': 1, 'MPI-IO': 1, 'STDIO': 129}\n",
       "Name Records:   4\n",
       "Darshan/Hints:  {'lib_ver': '3.1.3', 'h': 'romio_no_indep_rw=true;cb_nodes=4'}\n",
-      "DarshanReport:  id(140525897881584) (tmp)\n"
+      "DarshanReport:  id(139721554559216) (tmp)\n"
      ]
     }
    ],
    "source": [
     "import darshan\n",
     "\n",
-    "report = darshan.DarshanReport(\"example.darshan\", read_all=True)  # Default behavior\n",
+    "report = darshan.DarshanReport(\"example-logs/example.darshan\", read_all=True)  # Default behavior\n",
     "report.info()"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [
     {
@@ -60,7 +60,7 @@
        " 'STDIO': {'len': 3234, 'ver': 1, 'idx': 8, 'num_records': 129}}"
       ]
      },
-     "execution_count": 2,
+     "execution_count": 9,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -78,16 +78,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "<bound method DarshanReport.mod_records of <darshan.report.DarshanReport object at 0x7fcebc3997f0>>"
+       "<bound method DarshanReport.mod_records of <darshan.report.DarshanReport object at 0x7f1375a080f0>>"
       ]
      },
-     "execution_count": 3,
+     "execution_count": 10,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -105,7 +105,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 11,
    "metadata": {
     "scrolled": true
    },
@@ -126,7 +126,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 12,
    "metadata": {
     "scrolled": true
    },
@@ -137,7 +137,7 @@
        "dict_keys(['version', 'metadata', 'records', 'summary', 'modules', 'counters', 'name_records', 'mounts'])"
       ]
      },
-     "execution_count": 5,
+     "execution_count": 12,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -148,7 +148,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -157,7 +157,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 14,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -166,7 +166,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 15,
    "metadata": {},
    "outputs": [
     {
@@ -176,7 +176,7 @@
       "POSIX\n",
       "MPI-IO\n",
       "STDIO\n",
-      "Filename:       example.darshan\n",
+      "Filename:       example-logs/example.darshan\n",
       "Times:          2017-03-20 10:07:47 to 2017-03-20 10:09:43 (Duration 0:01:56)\n",
       "Executeable:    /global/project/projectdirs/m888/glock/tokio-abc-results/bin.edison/vpicio_uni /scratch2/scratchdirs/glock/tokioabc-s.4478544/vpicio/vpicio.hdf5 32\n",
       "Processes:      2048\n",
@@ -186,7 +186,7 @@
       "Loaded Records: {'POSIX': 1, 'MPI-IO': 1, 'STDIO': 129}\n",
       "Name Records:   3\n",
       "Darshan/Hints:  {'lib_ver': '3.1.3', 'h': 'romio_no_indep_rw=true;cb_nodes=4'}\n",
-      "DarshanReport:  id(140525897881584) (tmp)\n"
+      "DarshanReport:  id(139721554559216) (tmp)\n"
      ]
     }
    ],
@@ -197,7 +197,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 16,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -216,16 +216,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 17,
    "metadata": {},
    "outputs": [],
    "source": [
-    "report = darshan.DarshanReport(\"example.darshan\", read_all=False) # Loads no records!"
+    "report = darshan.DarshanReport(\"example-logs/example.darshan\", read_all=False) # Loads no records!"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 18,
    "metadata": {},
    "outputs": [
     {
@@ -253,7 +253,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 19,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -262,7 +262,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 20,
    "metadata": {},
    "outputs": [
     {
@@ -271,7 +271,7 @@
        "129"
       ]
      },
-     "execution_count": 8,
+     "execution_count": 20,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -282,14 +282,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 21,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Filename:       example.darshan\n",
+      "Filename:       example-logs/example.darshan\n",
       "Times:          2017-03-20 10:07:47 to 2017-03-20 10:09:43 (Duration 0:01:56)\n",
       "Executeable:    /global/project/projectdirs/m888/glock/tokio-abc-results/bin.edison/vpicio_uni /scratch2/scratchdirs/glock/tokioabc-s.4478544/vpicio/vpicio.hdf5 32\n",
       "Processes:      2048\n",
@@ -297,9 +297,9 @@
       "UID:            69615\n",
       "Modules in Log: ['POSIX', 'MPI-IO', 'LUSTRE', 'STDIO']\n",
       "Loaded Records: {'STDIO': 129}\n",
-      "Name Records:   4\n",
+      "Name Records:   0\n",
       "Darshan/Hints:  {'lib_ver': '3.1.3', 'h': 'romio_no_indep_rw=true;cb_nodes=4'}\n",
-      "DarshanReport:  id(139750934822192) (tmp)\n"
+      "DarshanReport:  id(139721553801904) (tmp)\n"
      ]
     }
    ],
@@ -309,7 +309,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 22,
    "metadata": {},
    "outputs": [
     {
@@ -317,16 +317,15 @@
       "text/plain": [
        "{'id': 15920181672442173319,\n",
        " 'rank': 0,\n",
-       " 'counters': array([                   1, 18446744073709551615,                    0,\n",
-       "                           6,                    0,                    0,\n",
-       "                         280,                    0,                    0,\n",
-       "                         279,                    0,                    0,\n",
-       "                           0], dtype=uint64),\n",
-       " 'fcounters': array([  1.,  -1.,   0.,   6.,   0.,   0., 280.,   0.,   0., 279.,   0.,\n",
-       "          0.,   0.])}"
+       " 'counters': array([  1,  -1,   0,   6,   0,   0, 280,   0,   0, 279,   0,   0,   0,\n",
+       "          0]),\n",
+       " 'fcounters': array([0.00000000e+00, 6.79492950e-05, 0.00000000e+00, 0.00000000e+00,\n",
+       "        0.00000000e+00, 7.75279999e-02, 0.00000000e+00, 0.00000000e+00,\n",
+       "        0.00000000e+00, 1.16283583e+02, 0.00000000e+00, 0.00000000e+00,\n",
+       "        0.00000000e+00, 0.00000000e+00, 0.00000000e+00])}"
       ]
      },
-     "execution_count": 13,
+     "execution_count": 22,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -353,7 +352,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 22,
+   "execution_count": 23,
    "metadata": {},
    "outputs": [
     {
@@ -439,7 +438,7 @@
     "import darshan\n",
     "darshan.enable_experimental()\n",
     "\n",
-    "report = darshan.DarshanReport(\"example.darshan\", read_all=True)\n",
+    "report = darshan.DarshanReport(\"example-logs/example.darshan\", read_all=True)\n",
     "report.name_records"
    ]
   },
@@ -870,8 +869,8 @@
     "from darshan.experimental.plots.matplotlib import plot_access_histogram\n",
     "from darshan.experimental.plots.matplotlib import plot_opcounts\n",
     "\n",
-    "r1 = darshan.DarshanReport(\"example.darshan\", read_all=True)\n",
-    "r2 = darshan.DarshanReport(\"example2.darshan\", read_all=True)\n",
+    "r1 = darshan.DarshanReport(\"example-logs/example.darshan\", read_all=True)\n",
+    "r2 = darshan.DarshanReport(\"example-logs/example2.darshan\", read_all=True)\n",
     "rx = r1 + r2\n",
     "\n",
     "for r in [r1, r2, rx]:\n",
@@ -926,7 +925,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {
@@ -942,7 +941,7 @@
        "<module 'matplotlib.pyplot' from '/home/pq/ANL/darshan-decaf/testbed/conda/lib/python3.6/site-packages/matplotlib/pyplot.py'>"
       ]
      },
-     "execution_count": 18,
+     "execution_count": 7,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -963,7 +962,7 @@
     "import darshan\n",
     "darshan.enable_experimental(verbose=False)\n",
     "\n",
-    "r3 = darshan.DarshanReport(\"example.darshan\")\n",
+    "r3 = darshan.DarshanReport(\"example-logs/example.darshan\")\n",
     "r3.mod_read_all_records('POSIX')\n",
     "\n",
     "from darshan.experimental.plots.matplotlib import plot_access_histogram\n",
@@ -1009,7 +1008,7 @@
     "import darshan\n",
     "darshan.enable_experimental(verbose=False)\n",
     "\n",
-    "r3 = darshan.DarshanReport(\"example.darshan\")\n",
+    "r3 = darshan.DarshanReport(\"example-logs/example.darshan\")\n",
     "r3.mod_read_all_records('MPI-IO')\n",
     "\n",
     "from darshan.experimental.plots.matplotlib import plot_access_histogram\n",
@@ -1018,7 +1017,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": 6,
    "metadata": {
     "scrolled": true
    },
@@ -1042,7 +1041,7 @@
        "<module 'matplotlib.pyplot' from '/home/pq/ANL/darshan-decaf/testbed/conda/lib/python3.6/site-packages/matplotlib/pyplot.py'>"
       ]
      },
-     "execution_count": 20,
+     "execution_count": 6,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -1063,7 +1062,7 @@
     "import darshan\n",
     "darshan.enable_experimental(verbose=False)\n",
     "\n",
-    "r3 = darshan.DarshanReport(\"example.darshan\")\n",
+    "r3 = darshan.DarshanReport(\"example-logs/example.darshan\")\n",
     "r3.read_all()\n",
     "\n",
     "from darshan.experimental.plots.matplotlib import plot_opcounts\n",
@@ -1081,7 +1080,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [
     {
@@ -1093,7 +1092,7 @@
     }
    ],
    "source": [
-    "report2 = darshan.DarshanReport(\"example.darshan\")\n",
+    "report2 = darshan.DarshanReport(\"example-logs/example.darshan\")\n",
     "report2.mod_read_all_dxt_records(\"ABC\") "
    ]
   },
@@ -1122,7 +1121,7 @@
    "outputs": [],
    "source": [
     "import darshan\n",
-    "report = darshan.DarshanReport(\"example.darshan\", read_all=True)\n",
+    "report = darshan.DarshanReport(\"example-logs/example.darshan\", read_all=True)\n",
     "report.to_json()"
    ]
   },
@@ -1141,30 +1140,41 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 24,
+   "execution_count": 1,
    "metadata": {
     "scrolled": true
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "ename": "NameError",
+     "evalue": "name 'darshan' is not defined",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
+      "\u001b[0;32m<ipython-input-1-0be0e3f32711>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mreport\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdarshan\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDarshanReport\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"example-logs/example.darshan\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+      "\u001b[0;31mNameError\u001b[0m: name 'darshan' is not defined"
+     ]
+    }
+   ],
    "source": [
-    "report = darshan.DarshanReport(\"example.darshan\")"
+    "report = darshan.DarshanReport(\"example-logs/example.darshan\")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 25,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [
     {
-     "ename": "KeyError",
-     "evalue": "'ABC'",
+     "ename": "NameError",
+     "evalue": "name 'report' is not defined",
      "output_type": "error",
      "traceback": [
       "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
-      "\u001b[0;32m<ipython-input-25-ad9bffc53533>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mreport\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmod_read_all_records\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"ABC\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# Expect KeyError\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
-      "\u001b[0;32m~/ANL/darshan-decaf/darshan-workflow/python-packages/darshan/report.py\u001b[0m in \u001b[0;36mmod_read_all_records\u001b[0;34m(self, mod, mode, warnings)\u001b[0m\n\u001b[1;32m    305\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    306\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 307\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmod\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'num_records'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    308\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    309\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
-      "\u001b[0;31mKeyError\u001b[0m: 'ABC'"
+      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
+      "\u001b[0;32m<ipython-input-2-ad9bffc53533>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mreport\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmod_read_all_records\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"ABC\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# Expect KeyError\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+      "\u001b[0;31mNameError\u001b[0m: name 'report' is not defined"
      ]
     }
    ],