Commit ff8e0a65 authored by Jakob Luettgau's avatar Jakob Luettgau
Browse files

Initial setup for a python package including documenting stubs for

various tests (pytest, coverage, lint/flake8) and make targets for easy
releases.
parent 07e48c11
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
env/
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# pyenv
.python-version
# celery beat schedule file
celerybeat-schedule
# SageMath parsed files
*.sage.py
# dotenv
.env
# virtualenv
.venv
venv/
ENV/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
# Config file for automatic testing at travis-ci.org
language: python
python:
- 3.6
- 3.5
- 3.4
- 2.7
# Command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors
install: pip install -U tox-travis
# Command to run tests, e.g. python setup.py test
script: tox
# Assuming you have installed the travis-ci CLI tool, after you
# create the Github repo and add it to Travis, run the
# following command to finish PyPI deployment setup:
# $ travis encrypt --add deploy.password
deploy:
provider: pypi
distributions: sdist bdist_wheel
user: jakobluettgau
password:
secure: PLEASE_REPLACE_ME
on:
tags: true
repo: jakobluettgau/pydarshan
python: 3.6
=======
History
=======
0.1.0 (2019-07-30)
------------------
* Initial python package.
include HISTORY.rst
include README.rst
recursive-include tests *
recursive-exclude * __pycache__
recursive-exclude * *.py[co]
recursive-include docs *.rst conf.py Makefile make.bat *.jpg *.png *.gif
.PHONY: clean clean-test clean-pyc clean-build docs help
.DEFAULT_GOAL := help
help:
@python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST)
clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts
clean-build: ## remove build artifacts
rm -rf build/
rm -rf dist/
rm -rf .eggs/
rm -rf docs/_build
find . -name '*.egg-info' -exec rm -fr {} +
find . -name '*.egg' -exec rm -f {} +
clean-pyc: ## remove Python file artifacts
find . -name '*.pyc' -exec rm -f {} +
find . -name '*.pyo' -exec rm -f {} +
find . -name '*~' -exec rm -f {} +
find . -name '__pycache__' -exec rm -fr {} +
clean-test: ## remove test and coverage artifacts
rm -rf .tox/
rm -f .coverage
rm -rf htmlcov/
rm -rf .pytest_cache
lint: ## check style with flake8
flake8 darshan tests
test: ## run tests quickly with the default Python
py.test
test-all: ## run tests on every Python version with tox
tox
coverage: ## check code coverage quickly with the default Python
coverage run --source darshan -m pytest
coverage report -m
coverage html
$(BROWSER) htmlcov/index.html
docs: ## generate Sphinx HTML documentation, including API docs
rm -f docs/darshan.rst
rm -f docs/modules.rst
sphinx-apidoc -o docs/ darshan
$(MAKE) -C docs clean
$(MAKE) -C docs html
open-docs: docs
xdg-open docs/_build/html/index.html
servedocs: docs ## compile the docs watching for changes
watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D .
release: dist ## package and upload a release
twine upload dist/*
dist: clean ## builds source and wheel package
python setup.py sdist
python setup.py bdist_wheel
ls -l dist
install: clean ## install the package to the active Python's site-packages
python setup.py install
=========
pydarshan
=========
Python utilities to interact with darshan log records of HPC applications.
pydarshan requires that you have darshan-util
Features
--------
* CFFI bindings to access darshan log files
* Plots typically found in the darshan reports (matplotlib)
* Auto-discover darshan-util.so (via darshan-parser in $PATH)
Usage
-----
For examples and a jupyter notebook to get started with pydarshan make sure
to check out the `examples` subdirectory.
A brief examples showing some of the basic functionality is the following::
import darshan
log = darshan.log_open("example.darshan")
# Access Job Information
darshan.log_get_job(log)
# Example Return:
# {'jobid': 4478544,
# 'uid': 69615,
# 'start_time': 1490000867,
# 'end_time': 1490000983,
# 'metadata': {'lib_ver': '3.1.3', 'h': 'romio_no_indep_rw=true;cb_nodes=4'}}
# Access available modules and modules
darshan.log_get_modules(log)
# Returns:
# {'POSIX': {'len': 186, 'ver': 3, 'idx': 1},
# 'MPI-IO': {'len': 154, 'ver': 2, 'idx': 2},
# 'LUSTRE': {'len': 87, 'ver': 1, 'idx': 6},
# 'STDIO': {'len': 3234, 'ver': 1, 'idx': 7}}
# Access different record types as numpy arrays, with integer and float counters seperated
# Example Return: {'counters': array([...], dtype=uint64), 'fcounters': array([...])}
posix_record = darshan.log_get_posix_record(log)
mpiio_record = darshan.log_get_mpiio_record(log)
stdio_record = darshan.log_get_stdio_record(log)
# ...
darshan.log_close(log)
Installation
------------
To install use either::
make install
Or::
python setup.py install
Testing
-------
Targets for various tests are included in the makefile. To run the normal
test suite use::
make test
Or to test against different version of Python using Tox::
make test-all
Coverage tests can be performed using::
make coverage
Conformance to PEPs can be tested using flake8 via::
make lint
Documentation
-------------
Documentation for the python bindings is generated seperatedly from the
darshan-utils C library in the interest of using Sphinx. After installing the
developement requirements using `pip install -r requirements_dev.txt` the
documentation can be build using make as follows::
pip install -r requirements_dev.txt
make docs
# -*- coding: utf-8 -*-
"""Top-level package for pydarshan."""
__version__ = '0.1.0'
from darshan.parser_cffi import *
# TODO: remove
def dummy_load():
return 42
# -*- coding: utf-8 -*-
header = """/* from darshan-logutils.h */
struct darshan_mnt_info
{
char mnt_type[3031];
char mnt_path[3031];
};
struct darshan_mod_info
{
char *name;
int len;
int ver;
int idx;
};
/* from darshan-log-format.h */
struct darshan_job
{
int64_t uid;
int64_t start_time;
int64_t end_time;
int64_t nprocs;
int64_t jobid;
char metadata[1024];
};
struct darshan_base_record
{
uint64_t id;
int64_t rank;
};
struct darshan_posix_file
{
struct darshan_base_record base_rec;
int64_t counters[64];
double fcounters[17];
};
struct darshan_stdio_file
{
struct darshan_base_record base_rec;
int64_t counters[13];
double fcounters[15];
};
struct darshan_mpiio_file
{
struct darshan_base_record base_rec;
int64_t counters[51];
double fcounters[15];
};
struct darshan_hdf5_file
{
struct darshan_base_record base_rec;
int64_t counters[1];
double fcounters[2];
};
struct darshan_pnetcdf_file
{
struct darshan_base_record base_rec;
int64_t counters[2];
double fcounters[2];
};
struct darshan_bgq_record
{
struct darshan_base_record base_rec;
int64_t counters[11];
double fcounters[1];
};
/* from darshan-apxc-log-format.h */
struct darshan_apxc_header_record
{
struct darshan_base_record base_rec;
int64_t magic;
int nblades;
int nchassis;
int ngroups;
int memory_mode;
int cluster_mode;
};
struct darshan_apxc_perf_record
{
struct darshan_base_record base_rec;
int64_t counters[396];
};
/* counter names */
char *apxc_counter_names[];
char *bgq_counter_names[];
char *bgq_f_counter_names[];
char *hdf5_counter_names[];
char *hdf5_f_counter_names[];
char *mpiio_counter_names[];
char *mpiio_f_counter_names[];
char *pnetcdf_counter_names[];
char *pnetcdf_f_counter_names[];
char *posix_counter_names[];
char *posix_f_counter_names[];
char *stdio_counter_names[];
char *stdio_f_counter_names[];
/* Supported Functions */
void* darshan_log_open(char *);
int darshan_log_get_job(void *, struct darshan_job *);
void darshan_log_close(void*);
int darshan_log_get_exe(void*, char *);
int darshan_log_get_mounts(void*, struct darshan_mnt_info **, int*);
void darshan_log_get_modules(void*, struct darshan_mod_info **, int*);
int darshan_log_get_record(void*, int, void **);"""
def load_darshan_header():
"""
Returns a CFFI compatible header for darshan-utlil as a string.
:return: String with a CFFI compatible header for darshan-util.
"""
return header
all:
@echo "Consult Makefile for available targets."
header:
./generate-header.sh
#./apply-patch.sh
clean:
rm -rf include
rm -f generated.h
rm -f generated.h.org
rm -f generated.h.orig
rm -f generated.h.rej
#!/bin/bash
patch -b --verbose generated.h patch-cffi-incompatible.patch
# 1 "include/darshan-logutils.h"
# 1 "<built-in>"
# 1 "<command-line>"
# 31 "<command-line>"
# 1 "/usr/include/stdc-predef.h" 1 3 4
# 32 "<command-line>" 2
# 1 "include/darshan-logutils.h"
# 15 "include/darshan-logutils.h"
struct UT_hash_handle;
# 1 "include/darshan-log-format.h" 1
# 39 "include/darshan-log-format.h"
enum darshan_comp_type
{
DARSHAN_ZLIB_COMP,
DARSHAN_BZIP2_COMP,
DARSHAN_NO_COMP,
};
typedef uint64_t darshan_record_id;
struct darshan_log_map
{
uint64_t off;
uint64_t len;
};
struct darshan_header
{
char version_string[8];
int64_t magic_nr;
unsigned char comp_type;
uint32_t partial_flag;
struct darshan_log_map name_map;
struct darshan_log_map mod_map[16];
uint32_t mod_ver[16];
};
struct darshan_job
{
int64_t uid;
int64_t start_time;
int64_t end_time;
int64_t nprocs;
int64_t jobid;
char metadata[1024];
};
struct darshan_name_record
{
darshan_record_id id;
char name[1];
};
struct darshan_base_record
{
darshan_record_id id;
int64_t rank;
};
# 1 "include/darshan-null-log-format.h" 1
# 31 "include/darshan-null-log-format.h"
enum darshan_null_indices
{
NULL_FOOS, NULL_FOO_MAX_DAT, NULL_NUM_INDICES,
};
enum darshan_null_f_indices
{
NULL_F_FOO_TIMESTAMP, NULL_F_FOO_MAX_DURATION, NULL_F_NUM_INDICES,
};
# 51 "include/darshan-null-log-format.h"
struct darshan_null_record
{
struct darshan_base_record base_rec;
int64_t counters[NULL_NUM_INDICES];
double fcounters[NULL_F_NUM_INDICES];
};
# 105 "include/darshan-log-format.h" 2
# 1 "include/darshan-posix-log-format.h" 1
# 148 "include/darshan-posix-log-format.h"
enum darshan_posix_indices
{
POSIX_OPENS, POSIX_READS, POSIX_WRITES, POSIX_SEEKS, POSIX_STATS, POSIX_MMAPS, POSIX_FSYNCS, POSIX_FDSYNCS, POSIX_MODE, POSIX_BYTES_READ, POSIX_BYTES_WRITTEN, POSIX_MAX_BYTE_READ, POSIX_MAX_BYTE_WRITTEN, POSIX_CONSEC_READS, POSIX_CONSEC_WRITES, POSIX_SEQ_READS, POSIX_SEQ_WRITES, POSIX_RW_SWITCHES, POSIX_MEM_NOT_ALIGNED, POSIX_MEM_ALIGNMENT, POSIX_FILE_NOT_ALIGNED, POSIX_FILE_ALIGNMENT, POSIX_MAX_READ_TIME_SIZE, POSIX_MAX_WRITE_TIME_SIZE, POSIX_SIZE_READ_0_100, POSIX_SIZE_READ_100_1K, POSIX_SIZE_READ_1K_10K, POSIX_SIZE_READ_10K_100K, POSIX_SIZE_READ_100K_1M, POSIX_SIZE_READ_1M_4M, POSIX_SIZE_READ_4M_10M, POSIX_SIZE_READ_10M_100M, POSIX_SIZE_READ_100M_1G, POSIX_SIZE_READ_1G_PLUS, POSIX_SIZE_WRITE_0_100, POSIX_SIZE_WRITE_100_1K, POSIX_SIZE_WRITE_1K_10K, POSIX_SIZE_WRITE_10K_100K, POSIX_SIZE_WRITE_100K_1M, POSIX_SIZE_WRITE_1M_4M, POSIX_SIZE_WRITE_4M_10M, POSIX_SIZE_WRITE_10M_100M, POSIX_SIZE_WRITE_100M_1G, POSIX_SIZE_WRITE_1G_PLUS, POSIX_STRIDE1_STRIDE, POSIX_STRIDE2_STRIDE, POSIX_STRIDE3_STRIDE, POSIX_STRIDE4_STRIDE, POSIX_STRIDE1_COUNT, POSIX_STRIDE2_COUNT, POSIX_STRIDE3_COUNT, POSIX_STRIDE4_COUNT, POSIX_ACCESS1_ACCESS, POSIX_ACCESS2_ACCESS, POSIX_ACCESS3_ACCESS, POSIX_ACCESS4_ACCESS, POSIX_ACCESS1_COUNT, POSIX_ACCESS2_COUNT, POSIX_ACCESS3_COUNT, POSIX_ACCESS4_COUNT, POSIX_FASTEST_RANK, POSIX_FASTEST_RANK_BYTES, POSIX_SLOWEST_RANK, POSIX_SLOWEST_RANK_BYTES, POSIX_NUM_INDICES,
};
enum darshan_posix_f_indices
{
POSIX_F_OPEN_START_TIMESTAMP, POSIX_F_READ_START_TIMESTAMP, POSIX_F_WRITE_START_TIMESTAMP, POSIX_F_CLOSE_START_TIMESTAMP, POSIX_F_OPEN_END_TIMESTAMP, POSIX_F_READ_END_TIMESTAMP, POSIX_F_WRITE_END_TIMESTAMP, POSIX_F_CLOSE_END_TIMESTAMP, POSIX_F_READ_TIME, POSIX_F_WRITE_TIME, POSIX_F_META_TIME, POSIX_F_MAX_READ_TIME, POSIX_F_MAX_WRITE_TIME, POSIX_F_FASTEST_RANK_TIME, POSIX_F_SLOWEST_RANK_TIME, POSIX_F_VARIANCE_RANK_TIME, POSIX_F_VARIANCE_RANK_BYTES, POSIX_F_NUM_INDICES,
};
# 167 "include/darshan-posix-log-format.h"
struct darshan_posix_file
{
struct darshan_base_record base_rec;
int64_t counters[POSIX_NUM_INDICES];
double fcounters[POSIX_F_NUM_INDICES];
};
# 106 "include/darshan-log-format.h" 2
# 1 "include/darshan-mpiio-log-format.h" 1
# 128 "include/darshan-mpiio-log-format.h"
enum darshan_mpiio_indices
{
MPIIO_INDEP_OPENS, MPIIO_COLL_OPENS, MPIIO_INDEP_READS, MPIIO_INDEP_WRITES, MPIIO_COLL_READS, MPIIO_COLL_WRITES, MPIIO_SPLIT_READS, MPIIO_SPLIT_WRITES, MPIIO_NB_READS, MPIIO_NB_WRITES, MPIIO_SYNCS, MPIIO_HINTS, MPIIO_VIEWS, MPIIO_MODE, MPIIO_BYTES_READ, MPIIO_BYTES_WRITTEN, MPIIO_RW_SWITCHES, MPIIO_MAX_READ_TIME_SIZE, MPIIO_MAX_WRITE_TIME_SIZE, MPIIO_SIZE_READ_AGG_0_100, MPIIO_SIZE_READ_AGG_100_1K, MPIIO_SIZE_READ_AGG_1K_10K, MPIIO_SIZE_READ_AGG_10K_100K, MPIIO_SIZE_READ_AGG_100K_1M, MPIIO_SIZE_READ_AGG_1M_4M, MPIIO_SIZE_READ_AGG_4M_10M, MPIIO_SIZE_READ_AGG_10M_100M, MPIIO_SIZE_READ_AGG_100M_1G, MPIIO_SIZE_READ_AGG_1G_PLUS, MPIIO_SIZE_WRITE_AGG_0_100, MPIIO_SIZE_WRITE_AGG_100_1K, MPIIO_SIZE_WRITE_AGG_1K_10K, MPIIO_SIZE_WRITE_AGG_10K_100K, MPIIO_SIZE_WRITE_AGG_100K_1M, MPIIO_SIZE_WRITE_AGG_1M_4M, MPIIO_SIZE_WRITE_AGG_4M_10M, MPIIO_SIZE_WRITE_AGG_10M_100M, MPIIO_SIZE_WRITE_AGG_100M_1G, MPIIO_SIZE_WRITE_AGG_1G_PLUS, MPIIO_ACCESS1_ACCESS, MPIIO_ACCESS2_ACCESS, MPIIO_ACCESS3_ACCESS, MPIIO_ACCESS4_ACCESS, MPIIO_ACCESS1_COUNT, MPIIO_ACCESS2_COUNT, MPIIO_ACCESS3_COUNT, MPIIO_ACCESS4_COUNT, MPIIO_FASTEST_RANK, MPIIO_FASTEST_RANK_BYTES, MPIIO_SLOWEST_RANK, MPIIO_SLOWEST_RANK_BYTES, MPIIO_NUM_INDICES,
};
enum darshan_mpiio_f_indices