Commit d951d4cf authored by Valentin Reis's avatar Valentin Reis
Browse files

Merge branch 'develop' into 'master'

Develop branch periodic merge.

See merge request !2
parents 02f80f88 f0bce9ab
Pipeline #10308 passed with stages
in 35 seconds
builds
*.pdf
*/__pycache__
__pycache__
*/.ipynb_checkpoints
......
......@@ -3,33 +3,16 @@ stages:
- lib
- docs
variables:
NIX_PATH: "nixpkgs=./."
ormolu:
stage: source
tags:
- kvm
- nix
script:
- |
nix-shell --pure -E '
let pkgs = (import ./.);
in pkgs.mkShell {
buildInputs = [pkgs.fd pkgs.ormolu];
shellHook =
"export LOCALE_ARCHIVE=${pkgs.glibcLocales}/lib/locale/locale-archive \n" +
"export LANG=en_US.UTF-8";
}
' --run bash <<< '
RETURN=0
cd src
for F in $(fd -e hs); do
ormolu -o -XTypeApplications -m check $F
if [ $? -ne 0 ]; then
echo "[!] $F does not pass ormolu haskell format check." >&2
RETURN=1
fi
done
if [ $RETURN -ne 0 ]; then exit 1; fi
'
- nix-shell -p gnumake --run "make ormolu"
hlint:
stage: source
......@@ -37,18 +20,15 @@ hlint:
- kvm
- nix
script:
- nix-shell --pure -p '(import ./.).hlint' --run bash <<< 'hlint src/ --hint=./.hlint.yaml'
- nix-shell -p gnumake --run "make hlint"
black:
shellcheck:
stage: source
tags:
- kvm
- nix
script:
- |
nix-shell --pure -p '(import ./.).pythonPackages.black' --run bash <<< '
black validation/*.py --check
'
- nix-shell -p gnumake --run "make shellcheck"
dhall-format:
stage: source
......@@ -56,18 +36,7 @@ dhall-format:
- kvm
- nix
script:
- |
nix-shell --pure -p '(import ./.).fd' '(import ./.).haskellPackages.dhall' --run bash <<< '
RETURN=0
for F in $(fd -e dhall); do
dhall format < $F | cmp -s $F -
if [ $? -ne 0 ]; then
echo "[!] $F does not pass dhall-format format check." >&2
RETURN=1
fi
done
if [ $RETURN -ne 0 ]; then exit 1; fi
'
- nix-shell -p gnumake --run "make dhall-format"
nix-build:
stage: lib
......@@ -76,21 +45,6 @@ nix-build:
- nix
script: nix-build -A hbandit --no-build-output
check-vendored-documentation:
stage: docs
tags:
- kvm
- nix
script:
- |
nix-shell nix/default.nix -A hack --run bash --arg useGhcide false <<< '
echo "check: is vendored documentation up to date?"
cp -r doc/ doc2
./shake doc
diff -r doc doc2
'
readthedocs:
stage: docs
when: on_success
......
This diff is collapsed.
Copyright (c) 2019, UChicago Argonne, LLC. All Rights Reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its
contributors may be used to endorse or promote products derived from this
software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# this file was tested using GNUMAKE >= 4.2.1.
# this is necessary for using multi-line strings as command arguments.
SHELL := $(shell which bash)
# this allows omitting newlines.
.ONESHELL:
# "nix-shell -p" constructs an expression that relies on <nixpkgs> for
# selecting attributes, so we override it.
# https://github.com/NixOS/nix/issues/726#issuecomment-161215255
NIX_PATH := nixpkgs=./.
.PHONY: vendor
vendor: hbandit.nix
dhall-to-cabal: default.nix
rm -rf ./dhall-to-cabal
cp -r $$(nix-build -A dhall-to-cabal-resources --no-out-link) ./dhall-to-cabal
chmod -R +rw ./dhall-to-cabal
#generating the vendored cabal file.
.PRECIOUS: hbandit.cabal
hbandit.cabal: hbandit.dhall dhall-to-cabal
@nix-shell --pure -E '
with import <nixpkgs> {};
mkShell {
buildInputs = [ haskellPackages.dhall-to-cabal ];
LOCALE_ARCHIVE="$${pkgs.glibcLocales}/lib/locale/locale-archive";
LANG="en_US.UTF-8";
}
' --run bash <<< '
dhall-to-cabal hbandit.dhall
'
.PRECIOUS: hbandit.nix
hbandit.nix: hbandit.cabal
@nix-shell --pure -p cabal2nix --run bash <<< '
cabal2nix . > hbandit.nix
'
.PHONY: ci
ci:
@nix-shell -p yq -p jq --run bash <<< '
for jobname in $$(yq -r "keys| .[]" .gitlab-ci.yml); do
if [ "$$jobname" != "stages" ]; then
gitlab-runner exec shell "$$jobname"
fi
done
'
ci-%:
@nix-shell --run bash <<< '
gitlab-runner exec shell "$*"
'
.PHONY: ghcid
ghcid: ghcid-hbandit
ghcid-hbandit: hbandit.cabal .hlint.yaml hbandit.nix
@nix-shell -E '
with import <nixpkgs> {};
with haskellPackages;
shellFor {
packages = p: [p.hbandit];
buildInputs = [ghcid cabal-install hlint];
}
' --pure --run bash <<< '
ghcid --command "cabal v2-repl hbandit " \
--restart=hbandit.cabal \
--restart=default.nix \
-l
'
ghcid-test: hbandit.cabal .hlint.yaml hbandit.nix
@nix-shell --pure --run bash <<< '
ghcid --command "cabal v2-repl test " \
--restart=hbandit.cabal \
--restart=default.nix \
-l
'
.PHONY: pre-commit
pre-commit: ormolu dhall-format shellcheck
.PHONY: shellcheck
shellcheck:
@nix-shell --pure -p fd shellcheck --run bash <<< '
for F in $$(fd -e sh); do
shellcheck -s bash $$F
done
'
.PHONY: hlint
dhall-format:
@nix-shell --pure -p hlint --run bash <<< '
hlint src/ --hint=./.hlint.yaml
'
.PHONY: dhall-format
dhall-format:
@nix-shell --pure -p fd haskellPackages.dhall --run bash <<< '
RETURN=0
for F in $$(fd -e dhall); do
dhall format < $$F | cmp -s $$F -
if [ $$? -ne 0 ]; then
echo "[!] $$F does not pass dhall-format format check. Formatting.." >&2
dhall format --inplace $$F
RETURN=1
fi
done
if [ $$RETURN -ne 0 ]; then exit 1; fi
'
.PHONY: ormolu
ormolu:
@nix-shell --pure -E '
let pkgs = import <nixpkgs> {};
in pkgs.mkShell {
buildInputs = [pkgs.fd pkgs.ormolu];
shellHook =
"export LOCALE_ARCHIVE=$${pkgs.glibcLocales}/lib/locale/locale-archive \n" +
"export LANG=en_US.UTF-8";
}
' --run bash <<< '
RETURN=0
for F in $$(fd -E src/HBandit/Tutorial.hs -e hs); do
ormolu -o -XTypeApplications -o -XPatternSynonyms -m check $$F
if [ $$? -ne 0 ]; then
echo "[!] $$F does not pass ormolu format check. Formatting.." >&2
ormolu -o -XTypeApplications -o -XPatternSynonyms -m inplace $$F
RETURN=1
fi
done
if [ $$RETURN -ne 0 ]; then exit 1; fi
'
.PHONY: doc
doc: src/HBandit/Tutorial.hs hbandit.cabal hbandit.nix
@nix-shell -E '
with import <nixpkgs> {};
with haskellPackages;
shellFor {
packages = p: [p.hbandit];
buildInputs = [cabal-install];
}
' --run <<< bash '
cabal v2-haddock hbandit --haddock-internal --builddir=.build
'
.PRECIOUS: src/HBandit/Tutorial.hs
src/HBandit/Tutorial.hs: literate/tutorial.md hbandit.nix src
@nix-shell --pure -E '
with import <nixpkgs> {};
with haskellPackages;
let extra = { mkDerivation, inline-r, pretty-simple, aeson, stdenv }:
mkDerivation {
pname = "extra";
version = "1.0.0";
src = "";
libraryHaskellDepends = [
aeson
inline-r
pretty-simple
];
description = "extra";
license = stdenv.lib.licenses.bsd3;
};
in
shellFor {
packages = p: [
p.hbandit
(haskellPackages.callPackage extra {})
];
buildInputs = [
inline-r
aeson
pretty-simple
panhandle
panpipe
unlit
pandoc
pkgs.which
cabal-install
R];
R_LIBS_SITE = "$${builtins.readFile r-libs-site}";
}
' --run bash <<< '
pandoc --filter $$(which panpipe) --filter $$(which panhandle) -f markdown+lhs -t markdown+lhs $< | unlit -f bird > $@
'
.PHONY:clean
clean:
rm -rf .build
rm -rf dist*
rm -f literate/main.hs
rm -f src/HBandit/Tutorial.hs
rm -f hbandit.nix
rm -f hbandit.cabal
rm -rf dhall-to-cabal
# hbandit
Multi-Armed Bandit implementations: EXP, BwCR (wip), Exp4.R
## Install
Provision using either Cabal or Nix toolchain. Both cabal file and nix packages
are provided in the repository.
Dependencies: hackage packages, glpk
## Notebooks
Provision using Nix (`nix-shell`) and run notebook
[validation/benchmark.ipynb](validation/benchmark.ipynb). Nix one-liner:
```
nix-shell "https://xgitlab.cels.anl.gov/argo/hbandit/-/archive/master/hbandit-master.tar.gz" --pure -A validation --arg useGhcide false --run 'jupyter-notebook $NOTEBOOKS/benchmark.ipynb'
```
## Doc
[haddocks](https://hbandit.readthedocs.io/en/latest/_static/haddocks/)
Safe Multi-Armed Bandit implementations: EXP3, BwCR (wip), Exp4.R
{-# OPTIONS_GHC -fno-warn-missing-signatures #-}
{-# OPTIONS_GHC -fno-warn-orphans #-}
-- |
-- Module : main
-- Copyright : (c) 2019, UChicago Argonne, LLC.
-- License : BSD3
-- Maintainer : fre@freux.fr
module Shared
(
)
where
import Data.List.NonEmpty
import Data.MessagePack
import FFI.TypeUncurry.Msgpack
import Foreign.C
import HBandit.Class
import HBandit.Exp3
import Protolude
import Refined hiding (NonEmpty)
import System.Random
import Prelude (fail)
-- | All FFI exported names in this module must have this opaque type
-- , must be followed by "Export", and must not use reserved symbols
-- like "stdout" or "stdin".
type Ex = CString -> IO CString
----------------------------- Exp3 -------------------------------------------
foreign export ccall initExp3Export :: Ex
foreign export ccall stepExp3Export :: Ex
exp3 :: Exportable (Exp3 Int) Int Int Double
exp3 =
exportBandit
(\count -> Right $ Arms (nub $ fromList [1 .. count]))
refine
initExp3Export = exportIO (initExportable exp3)
stepExp3Export = exportIO (stepExportable exp3)
deriving instance MessagePack (Arms Int)
deriving instance MessagePack (Exp3 Int)
deriving instance MessagePack (Weight Int)
deriving instance MessagePack Probability
deriving instance MessagePack CumulativeLoss
instance (MessagePack a) => MessagePack (NonEmpty a) where
toObject = toObject . Protolude.toList
fromObject x =
fromObject x >>= \y ->
case nonEmpty y of
Nothing -> fail "NonEmpty error in msgpack message"
Just t -> return t
----- typeclass based exporting functions
data Exportable b hyper a loss
= Exportable
{ initExportable :: hyper -> IO (b, a),
stepExportable :: b -> loss -> IO (b, a)
}
exportBandit ::
(Bandit b hyper a loss) =>
(hyper' -> Either e1 hyper) ->
(loss' -> Either e2 loss) ->
Exportable b hyper' a loss'
exportBandit hyperBuilder lossBuilder =
Exportable
{ initExportable = \hyper' -> hyperBuilder hyper' & \case
Left _ -> fail "hyperparameter refinement failed"
Right hyper -> do
g <- liftIO getStdGen
let (b, a, g') = HBandit.Class.init g hyper
liftIO $ setStdGen g'
return (b, a),
stepExportable = \b loss' ->
lossBuilder loss' & \case
Left _ -> fail "loss refinement failed"
Right loss -> do
g <- liftIO getStdGen
let ((a, g'), b') = runState (HBandit.Class.step g loss) b
liftIO $ setStdGen g'
return (b', a)
}
#!/usr/bin/env nix-shell
#! nix-shell -p yq -p jq -i bash
if [ $# -eq 0 ]
then
for jobname in $(yq -r 'keys| .[]' .gitlab-ci.yml); do
if [ $jobname != "stages" ]; then
gitlab-runner exec shell $jobname
fi
done
else
gitlab-runner exec shell $1
fi
import nix/default.nix {}
{ pkgs ? import (builtins.fetchTarball
"http://nixos.org/channels/nixos-20.03/nixexprs.tar.xz") { } }:
with pkgs.lib;
let
ormolu = let
source = pkgs.fetchFromGitHub {
owner = "tweag";
repo = "ormolu";
rev = "f83f6fd1dab5ccbbdf55ee1653b24595c1d653c2";
sha256 = "1hs7ayq5d15m9kxwfmdac3p2i3s6b0cn58cm4rrqc4d447yl426y";
};
in (import source { }).ormolu;
in pkgs // rec {
dhall-to-cabal-resources = pkgs.stdenv.mkDerivation {
name = "dhall-to-cabal-resources";
src = pkgs.haskellPackages.dhall-to-cabal.src;
installPhase = "cp -r dhall $out";
};
haskellPackages = pkgs.haskellPackages.override {
overrides = self: super:
with pkgs.haskell.lib; rec {
hbandit = self.callPackage ./hbandit.nix {};
panpipe = doJailbreak super.panpipe;
refined = unmarkBroken super.refined;
lazysmallcheck2012 = null;
panhandle = doJailbreak (dontCheck (self.callCabal2nix "panhandle"
(builtins.fetchTarball
"https://github.com/freuk/panhandle/archive/master.tar.gz") { }));
};
};
inherit ormolu;
hlint = haskellPackages.hlint;
hbandit = haskellPackages.hbandit;
r-libs-site = pkgs.runCommand "r-libs-site" {
buildInputs = with pkgs; [
R
rPackages.ggplot2
rPackages.svglite
rPackages.plotly
rPackages.latex2exp
];
} "echo $R_LIBS_SITE > $out";
}
# Minimal makefile for Sphinx documentation
#
# You can set these variables from the command line.
SPHINXOPTS =
SPHINXBUILD = sphinx-build
SPHINXPROJ = NRM
SOURCEDIR = .
BUILDDIR = .build
# Put it first so that "make" without argument is like "make help".
help:
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
.PHONY: help Makefile
# Catch-all target: route all unknown targets to Sphinx using the new
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
%: Makefile
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
# -*- coding: utf-8 -*-
#
# Configuration file for the Sphinx documentation builder.
#
# This file does only contain a selection of the most common options. For a
# full list see the documentation:
# http://www.sphinx-doc.org/en/master/config
# -- Path setup --------------------------------------------------------------
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
# import os
# import sys
# sys.path.insert(0, os.path.abspath('.'))
# import os
# import sys
import subprocess
# -- Project information -----------------------------------------------------
project = u'HBandit'
copyright = u'2019, Argonne National Laboratory'
author = u'Argo team, Argonne National Laboratory'
# The short X.Y version
version = u''
# The full version, including alpha/beta/rc tags
release = u''
# -- General configuration ---------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
#
# needs_sphinx = '1.0'
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = []
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'
# The master toctree document.
master_doc = 'index'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path .
exclude_patterns = [u'.build', 'Thumbs.db', '.DS_Store']
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
# -- Options for HTML output -------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
html_theme = 'alabaster'
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
#
# html_theme_options = {}
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static', 'hbandit']
# Custom sidebar templates, must be a dictionary that maps document names
# to template names.