Commit e789f838 authored by Jakob Luettgau's avatar Jakob Luettgau
Browse files

Add scripts to build release wheels with darshan-utils included.

parent 91bdad64
......@@ -9,7 +9,6 @@ import logging
logger = logging.getLogger(__name__)
def check_version(ffi=None, libdutil=None):
"""
Get version from shared library or pkg-config and return info.
......@@ -95,14 +94,44 @@ def discover_darshan_shutil():
raise RuntimeError('Could not discover darshan! Is darshan-util installed and set in your PATH?')
def discover_darshan_wheel():
"""
Discovers darshan-util if installed as as part of the wheel.
:return: Path to a darshan-util installation.
"""
path = os.path.dirname(__file__)
if path:
return os.path.realpath(path + '/../darshan.libs')
else:
raise RuntimeError('Could not discover darshan! Is darshan-util installed and set in your PATH?')
def find_utils(ffi, libdutil):
"""
Try different methods to discover darshan-util:
Precedence:
1) Try if the current environment allows dlopen to load libdarshan-util
2) Try if darshan-parser is exposed via PATH, and attempt loading relative to it.
3) Try if darshan is exposed via pkgconfig
4) Fallback on binary distributed along with Python package
Args:
ffi: existing ffi instance to use
libdutil: reference to libdutil to populate
"""
if libdutil is None:
try:
libdutil = ffi.dlopen("libdarshan-util.so")
except:
libdutil = None
print("ffi.dlopen failed")
if libdutil is None:
try:
......@@ -118,6 +147,19 @@ def find_utils(ffi, libdutil):
libdutil = ffi.dlopen(DARSHAN_PATH + "/lib/libdarshan-util.so")
except:
libdutil = None
print("pkgconfig failed")
if libdutil is None:
try:
DARSHAN_PATH = discover_darshan_wheel()
import glob
DARSHAN_SO = glob.glob(f'{DARSHAN_PATH}/libdarshan-util*.so')[0]
libdutil = ffi.dlopen(DARSHAN_SO)
except:
libdutil = None
print("")
if libdutil is None:
raise RuntimeError('Could not find libdarshan-util.so! Is darshan-util installed? Please ensure one of the the following: 1) export LD_LIBRARY_PATH=<path-to-libdarshan-util.so>, or 2) darshan-parser can found using the PATH variable, or 3) pkg-config can resolve pkg-config --path darshan-util')
......
To build manylinux compliant python packages:
https://github.com/pypa/python-manylinux-demo
Alternatively, consider cibuildwheel, e.g. for mac builds.
https://cibuildwheel.readthedocs.io/en/stable/setup/
# Debugging the Build Process
To debug the build process ensure the docker image is pulled and start in interactive session as follows:
DOCKER_IMAGE=quay.io/pypa/manylinux1_x86_64
PLAT=manylinux1_x86_64
docker pull $DOCKER_IMAGE
docker run -e PLAT=$PLAT -v `pwd`/../:/io -v `pwd`/../../..:/darshan -it $DOCKER_IMAGE /bin/bash
Or to jump into a running docker container:
docker exec -it <NAME> bash
#!/bin/bash
rm -rf wheelhouse
mkdir -p wheelhouse
MNT_PYDARSHAN=`pwd`
MNT_DARSHAN=`pwd`/../..
DOCKER_IMAGE=quay.io/pypa/manylinux1_x86_64
PLAT=manylinux1_x86_64
PRE_CMD=
docker pull $DOCKER_IMAGE
docker run --rm -e PLAT=$PLAT \
-v $MNT_PYDARSHAN:/io \
-v $MNT_DARSHAN:/darshan \
$DOCKER_IMAGE $PRE_CMD /io/devel/build-wheels.sh
exit
DOCKER_IMAGE=quay.io/pypa/manylinux1_i686
PLAT=manylinux1_i686
PRE_CMD=linux32
docker pull $DOCKER_IMAGE
docker run --rm -e PLAT=$PLAT -v $MNT_PYDARSHAN:/io -v $MNT_DARSHAN:/darshan $DOCKER_IMAGE $PRE_CMD /io/devel/build-wheels.sh
DOCKER_IMAGE=quay.io/pypa/manylinux2010_x86_64
PLAT=manylinux2010_x86_64
PRE_CMD=
docker pull $DOCKER_IMAGE
docker run --rm -e PLAT=$PLAT -v $MNT_PYDARSHAN:/io -v $MNT_DARSHAN:/darshan $DOCKER_IMAGE $PRE_CMD /io/devel/build-wheels.sh
#!/bin/bash
set -e -u -x
function repair_wheel {
wheel="$1"
if ! auditwheel show "$wheel"; then
echo "Skipping non-platform wheel $wheel"
else
auditwheel repair "$wheel" --plat "$PLAT" -w /io/wheelhouse/
fi
}
# Install a system package required by darshan-utils
yum install -y zlib zlib-devel
# Build and install darshan-util
cd /darshan/darshan-util
./configure --enable-shared
make install
make distclean
cd /
# Do not build for end-of-life python versions
rm -f /opt/python/cp27-cp27m
rm -f /opt/python/cp27-cp27mu
rm -f /opt/python/cp35-cp35m
ls /opt/python
# Compile wheels
for PYBIN in /opt/python/*/bin; do
"${PYBIN}/pip" install -r /io/requirements_dev.txt
"${PYBIN}/pip" wheel /io/ --no-deps -w /io/wheelhouse/
done
# Bundle external shared libraries into the wheels
for whl in /io/wheelhouse/*.whl; do
repair_wheel "$whl"
done
## Install packages and test
#for PYBIN in /opt/python/*/bin/; do
# "${PYBIN}/pip" install darshan --no-index -f /io/wheelhouse
# (cd "$HOME"; "${PYBIN}/nosetests" darshan)
#done
notifications:
email: false
matrix:
include:
- sudo: required
services:
- docker
env: DOCKER_IMAGE=quay.io/pypa/manylinux1_x86_64
PLAT=manylinux1_x86_64
- sudo: required
services:
- docker
env: DOCKER_IMAGE=quay.io/pypa/manylinux1_i686
PRE_CMD=linux32
PLAT=manylinux1_i686
- sudo: required
services:
- docker
env: DOCKER_IMAGE=quay.io/pypa/manylinux2010_x86_64
PLAT=manylinux2010_x86_64
install:
- docker pull $DOCKER_IMAGE
script:
- docker run --rm -e PLAT=$PLAT -v `pwd`:/io $DOCKER_IMAGE $PRE_CMD /io/devel/build-wheels.sh
- ls wheelhouse/
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment