default.nix 14 KB
Newer Older
Valentin Reis's avatar
Valentin Reis committed
1
{ # host package set (unused except for fetching nixpkgs)
Valentin Reis's avatar
Valentin Reis committed
2
  hostPkgs         ? import <nixpkgs> {}
Valentin Reis's avatar
Valentin Reis committed
3 4

, # versioned nixpkgs
Valentin Reis's avatar
Valentin Reis committed
5
  pkgs             ? import (hostPkgs.nix-update-source.fetch ./pkgs.json).src {}
Valentin Reis's avatar
Valentin Reis committed
6

Valentin Reis's avatar
Valentin Reis committed
7
, # versioned nixpkgs-older version for zmcat
Valentin Reis's avatar
Valentin Reis committed
8
  bleeding         ? import (hostPkgs.nix-update-source.fetch ./bleeding.json).src {}
Valentin Reis's avatar
Valentin Reis committed
9

Valentin Reis's avatar
Valentin Reis committed
10
, # fetcher alias for the remaining arguments
Valentin Reis's avatar
Valentin Reis committed
11
  fetched          ? s: (pkgs.nix-update-source.fetch s).src
Valentin Reis's avatar
Valentin Reis committed
12 13

, #source overrides: core nrm components
Valentin Reis's avatar
Valentin Reis committed
14
  aml-src          ? fetched aml/pin.json
Valentin Reis's avatar
Valentin Reis committed
15
, numabench-src    ? fetched numabench/pin.json
Valentin Reis's avatar
Valentin Reis committed
16
, excit-src        ? fetched excit/pin.json
Valentin Reis's avatar
Valentin Reis committed
17 18 19
, libnrm-src       ? fetched libnrm/pin.json
, nrm-src          ? fetched nrm/pin.json
, containers-src   ? fetched containers/pin.json
Valentin Reis's avatar
Valentin Reis committed
20 21

, #source overrides: applications
Valentin Reis's avatar
Valentin Reis committed
22 23 24 25 26
  amg-src          ? fetched amg/pin.json
, qmcpack-src      ? fetched qmcpack/pin.json
, openmc-src       ? fetched openmc/pin.json
, stream-src       ? fetched stream/pin.json
, lammps-src       ? fetched lammps/pin.json
Valentin Reis's avatar
Valentin Reis committed
27

Valentin Reis's avatar
Valentin Reis committed
28
  #source overrides: tooling
29
, repoquality-src  ? builtins.fetchGit {inherit (fetched repoquality/pin.json) url rev;}
Valentin Reis's avatar
Valentin Reis committed
30
, dhrun-src        ? builtins.fetchGit {inherit (fetched dhrun/pin.json) url rev;}
Valentin Reis's avatar
Valentin Reis committed
31
, powerplots-src   ? builtins.fetchGit {inherit (fetched powerplots/pin.json) url rev;}
32
, argonix-src      ? builtins.fetchGit {inherit (fetched argonix/pin.json) url rev;}
Valentin Reis's avatar
Valentin Reis committed
33
, experiments-src  ? fetched experiments/pin.json
34 35

, #used for hardware experiments.
Valentin Reis's avatar
Valentin Reis committed
36
  sudoPath         ? null
Valentin Reis's avatar
Valentin Reis committed
37

Valentin Reis's avatar
Valentin Reis committed
38 39
, #whether to graft apps on libnrm.
  graftLibnrm      ? false
40 41 42

, #name for the generic test builder
  genericTestName  ? "run"
Valentin Reis's avatar
.  
Valentin Reis committed
43 44
}:
let
Valentin Reis's avatar
Valentin Reis committed
45
  callPackage = pkgs.lib.callPackageWith (pkgs // argopkgs);
46

Valentin Reis's avatar
Valentin Reis committed
47 48 49
  addBI = drv: bi: drv.overrideAttrs
    (old:{buildInputs=old.buildInputs ++ [ bi ];});

Valentin Reis's avatar
Valentin Reis committed
50
  argopkgs = rec {
51
    # library of nix expressions
Valentin Reis's avatar
style.  
Valentin Reis committed
52
    argolib       = import ../lib/utils.nix;
53
    # Argo components (source overridabel via argument)
Valentin Reis's avatar
Valentin Reis committed
54
    aml           = callPackage ./aml        {src=aml-src; inherit excit; inherit numabench;};
Valentin Reis's avatar
Valentin Reis committed
55
    numabench     = callPackage ./numabench  {src=numabench-src; hwloc = hwloc2;};
Valentin Reis's avatar
Valentin Reis committed
56
    excit         = callPackage ./excit      {src=excit-src;};
Valentin Reis's avatar
Valentin Reis committed
57 58 59 60 61
    nrm           = callPackage ./nrm        {src=nrm-src;};
    containers    = callPackage ./containers {src=containers-src;};
    libnrm        = callPackage ./libnrm     {src=libnrm-src;};
    libnrm-pinned = callPackage ./libnrm     {src=fetched libnrm/pin.json;};
    libnrm-pmpi   = libnrm.overrideAttrs (oldAttrs: rec {
Valentin Reis's avatar
Valentin Reis committed
62 63 64
      configureFlags = "--enable-pmpi";
      buildInputs = oldAttrs.buildInputs ++ [ pkgs.mpich2 ];
    });
65

66
    nrm-pytest    = nrm.overrideAttrs (old:{checkPhase="py.test";});
Valentin Reis's avatar
Valentin Reis committed
67
    aml-dist      = aml.overrideAttrs (old:{
Valentin Reis's avatar
Valentin Reis committed
68 69
      nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.doxygen ];
      buildPhase = "make dist";
Valentin Reis's avatar
Valentin Reis committed
70 71 72 73
      installPhase = ''
        mkdir -p $out
        cp *.tar.gz $out/
      '';
Valentin Reis's avatar
Valentin Reis committed
74
    });
75

Valentin Reis's avatar
Valentin Reis committed
76 77
    numabench-check = numabench.overrideAttrs(old: {
      buildPhase = "make check";
78
      installPhase =''
79 80 81
        mkdir -p $out/tests
        cp ./tests/*.log $out/tests
        cp ./tests/*.trs $out/tests
Valentin Reis's avatar
Valentin Reis committed
82
        cp -r ./tests/tmp $out/tests
83
        '';
Valentin Reis's avatar
Valentin Reis committed
84 85 86 87 88
    });
    excit-check = excit.overrideAttrs(old: {
      buildPhase = "make check";
    });

Valentin Reis's avatar
Valentin Reis committed
89 90 91
    # applications with replaceDependency for libnrm.
    repLibnrm = path: src: extra:
    let
Valentin Reis's avatar
Valentin Reis committed
92
      drv = callPackage path ({
Valentin Reis's avatar
style.  
Valentin Reis committed
93 94 95
        nrmSupport = true;
        src = src;
        libnrm = libnrm-pinned;
Valentin Reis's avatar
Valentin Reis committed
96
      }//extra);
Valentin Reis's avatar
style.  
Valentin Reis committed
97
      rebuilt = drv.override {libnrm = libnrm;};
Valentin Reis's avatar
Valentin Reis committed
98 99 100 101 102 103
      grafted = pkgs.replaceDependency {
        drv = drv;
        oldDependency = libnrm-pinned;
        newDependency = libnrm;
      };
    in if graftLibnrm then grafted else rebuilt;
Valentin Reis's avatar
Valentin Reis committed
104 105 106 107
    amg          = repLibnrm ./amg     amg-src     {};
    qmcpack      = repLibnrm ./qmcpack qmcpack-src {};
    openmc       = repLibnrm ./openmc  openmc-src  {};
    lammps       = repLibnrm ./lammps  lammps-src  {};
Valentin Reis's avatar
Valentin Reis committed
108
    stream       = repLibnrm ./stream  stream-src  {iterationCount="2000";};
Valentin Reis's avatar
Valentin Reis committed
109

110
    # data (static, non source overridable via arguments)
Valentin Reis's avatar
Valentin Reis committed
111
    openmc-data = fetched openmc/data.json;
Valentin Reis's avatar
Valentin Reis committed
112

113
    # tools
114
    checkpatch = callPackage ./checkpatch {src=pkgs.linux_testing.src;};
115 116 117
    zymake = callPackage ./zymake {
      inherit (pkgs.ocamlPackages) findlib ocamlbuild topkg batteries;
    };
Valentin Reis's avatar
Valentin Reis committed
118 119
    haskellPackages = pkgs.haskellPackages.override {
      overrides = self: super: with pkgs.haskell.lib;
Valentin Reis's avatar
Valentin Reis committed
120
      rec {
Valentin Reis's avatar
Valentin Reis committed
121
        panpipe        = (pkgs.haskell.lib.doJailbreak super.panpipe);
Valentin Reis's avatar
Valentin Reis committed
122
        spdx           = (pkgs.haskell.lib.doJailbreak super.spdx);
Valentin Reis's avatar
Valentin Reis committed
123
        monad-mock     = (pkgs.haskell.lib.doJailbreak super.monad-mock);
Valentin Reis's avatar
Valentin Reis committed
124
        generic-pretty = doJailbreak super.generic-pretty;
Valentin Reis's avatar
Valentin Reis committed
125

Valentin Reis's avatar
Valentin Reis committed
126 127 128
        #minor cabal and version fix
        panhandle    = (self.callCabal2nix "panhandle" (builtins.fetchGit {
                      inherit (panhandle-src) url rev; })) {};
Valentin Reis's avatar
Valentin Reis committed
129 130 131 132 133
        inline-r = dontCheck super.inline-r;
        argonix     = (self.callCabal2nix "argonix"     argonix-src     ) {};
        repoquality = (self.callCabal2nix "repoquality" repoquality-src ) {};
        powerplots  = (self.callCabal2nix "powerplots"  powerplots-src  ) {};

134
      };
Valentin Reis's avatar
Valentin Reis committed
135
    };
Valentin Reis's avatar
Valentin Reis committed
136

Valentin Reis's avatar
Valentin Reis committed
137 138 139
    bleedingHaskellPackages = bleeding.haskellPackages.override {
      overrides = self: super: with bleeding.haskell.lib;
      rec {
140 141 142 143 144 145 146
        #panpipe        = (bleeding.haskell.lib.doJailbreak super.panpipe);
        #spdx           = (bleeding.haskell.lib.doJailbreak super.spdx);
        #monad-mock     = (bleeding.haskell.lib.doJailbreak super.monad-mock);
        #generic-pretty = doJailbreak super.generic-pretty;

        multistate  = doJailbreak super.multistate;
        #hspec       = doJailbreak super.hspec;
Valentin Reis's avatar
Valentin Reis committed
147 148 149 150
        dhrun       = (self.callCabal2nix "dhrun"       dhrun-src ) {};
      };
    };

151
    # re-export from hpkgs and add non-haskell build rule specialization
Valentin Reis's avatar
Valentin Reis committed
152
    inherit (haskellPackages) repoquality;
Valentin Reis's avatar
Valentin Reis committed
153
    inherit (bleedingHaskellPackages) dhall dhall-json;
Valentin Reis's avatar
Valentin Reis committed
154

Valentin Reis's avatar
Valentin Reis committed
155
    dhrun = bleedingHaskellPackages.dhrun.overrideAttrs (old:{
Valentin Reis's avatar
Valentin Reis committed
156
        installPhase = old.installPhase + ''
Valentin Reis's avatar
Valentin Reis committed
157 158
          mkdir -p $out/share/
          cp -r resources $out/share/
Valentin Reis's avatar
Valentin Reis committed
159
        '';
Valentin Reis's avatar
Valentin Reis committed
160 161 162 163
      });

    argonix = haskellPackages.argonix.overrideAttrs (old:{
        installPhase = old.installPhase + ''
Valentin Reis's avatar
Valentin Reis committed
164
          mkdir -p $out/share/
Valentin Reis's avatar
Valentin Reis committed
165 166 167
          $out/bin/argonix --bash-completion-script argonix > $out/share/argonix.sh
        '';
      });
168

Valentin Reis's avatar
Valentin Reis committed
169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189
    powerplots =
    let
      pkl = with pkgs.rPackages; [ docopt pracma tidyr readr magrittr formatR
          purrr wrapr knitr cowplot plotly lubridate dplyr ggplot2 fpp2
          prospectr gridExtra zoo xtable ggthemes data_table
        ];
      myR = (pkgs.rWrapper.override{ packages = [pkl];});
    in
    haskellPackages.powerplots.overrideDerivation (drv: rec {
      buildInputs = (drv.buildInputs or []) ++ [
        pkgs.makeWrapper
        myR
      ];

      installPhase = drv.installPhase + ''
        wrapProgram "$out/bin/powerplots" \
          --set R_LIBS_SITE `Rscript -e "Sys.getenv(\"R_LIBS_SITE\")"`
      '';

    });

190
    # fix for nrm
Valentin Reis's avatar
Valentin Reis committed
191
    pyzmq = pkgs.pythonPackages.pyzmq.override { zeromq3 = pkgs.zeromq; };
Valentin Reis's avatar
Valentin Reis committed
192

Valentin Reis's avatar
Valentin Reis committed
193 194 195 196 197 198 199
    hwloc2 = pkgs.hwloc.overrideAttrs (old:{
      src = pkgs.fetchurl {
        url = "https://download.open-mpi.org/release/hwloc/v2.0/hwloc-2.0.3.tar.gz";
        sha256 = "0707ml34ylrr8zwwhddp64q6q57248m9646f27jadcx5m93g5pk4";
      };
    });

200
    # ci/experimental environments
Valentin Reis's avatar
Valentin Reis committed
201 202
    test = pkgs.stdenv.mkDerivation rec {
      name = "testEnv";
203

Valentin Reis's avatar
Valentin Reis committed
204
      buildInputs = [
Valentin Reis's avatar
Valentin Reis committed
205
        # tooling may run mpiexec or hwloc:
Valentin Reis's avatar
Valentin Reis committed
206 207 208 209 210 211 212
        pkgs.mpich2
        pkgs.hwloc
        # these are needed for some reason:
        pkgs.coreutils
        pkgs.utillinux
        pkgs.git
        # argo tools
Valentin Reis's avatar
Valentin Reis committed
213
        dhrun
Valentin Reis's avatar
Valentin Reis committed
214 215 216
        # argo components
        containers
        nrm
Valentin Reis's avatar
Valentin Reis committed
217
      ];
218

Valentin Reis's avatar
Valentin Reis committed
219
      #required to be able to build the drv.
220
      installPhase = "mkdir $out";
Valentin Reis's avatar
Valentin Reis committed
221
      unpackPhase = "true";
Valentin Reis's avatar
Valentin Reis committed
222

223 224 225 226 227 228 229 230
      __noChroot = true;
      shellHook =''
        echo ""
        echo "ENTERING ENVIRONMENT: Sources of impurity for this build are: /tmp/ /etc/argo/ /var/run/ /var/lock/".
        echo "ENTERING ENVIRONMENT: Moreover, sandboxing is disabled, in particular because of:"
        echo "ENTERING ENVIRONMENT: /tmp/nrm-* sockets, /etc/argo, /var/run/, /var/lock/ which all need read access. "
        echo "ENTERING ENVIRONMENT: until these components are patched to allow for alternative paths, no sandbox use is possible."
        echo ""
231
      '';
232
      #argotk uses this environment variable.
Valentin Reis's avatar
Valentin Reis committed
233
      #OPENMC_PWD = "${argotk}/share/openmc";
Valentin Reis's avatar
Valentin Reis committed
234
    };
Valentin Reis's avatar
Valentin Reis committed
235 236 237 238 239 240

    analysis = pkgs.stdenv.mkDerivation rec {
      name = "analysis";
      src = experiments-src;

      #required to be able to build the drv.
Valentin Reis's avatar
Valentin Reis committed
241 242 243
      installPhase = ''
        mkdir -p $out
      '';
Valentin Reis's avatar
Valentin Reis committed
244 245 246
      unpackPhase = "true";

      buildInputs = [
247
        pkgs.pandoc
Valentin Reis's avatar
Valentin Reis committed
248 249 250 251
        (pkgs.rWrapper.override{ packages = with pkgs.rPackages; [
          pkgs.rlwrap
          pkgs.pandoc
          docopt
252 253 254 255
          pracma
          tidyr
          readr
          magrittr
Valentin Reis's avatar
Valentin Reis committed
256 257
          formatR
          purrr
258
          wrapr
Valentin Reis's avatar
Valentin Reis committed
259 260 261 262 263 264
          knitr
          cowplot
          plotly
          lubridate
          dplyr
          ggplot2
265 266 267 268
          fpp2
          prospectr
          gridExtra
          zoo
Valentin Reis's avatar
Valentin Reis committed
269 270 271 272 273 274 275
          xtable
          ggthemes
          data_table
        ];})
      ];
    };

276 277
    hanalysis = analysis.overrideAttrs (old:{
      buildInputs = old.buildInputs ++ [
Valentin Reis's avatar
Valentin Reis committed
278 279 280 281 282
        (haskellPackages.ghcWithPackages(p:[
          p.protolude
          p.optparse-applicative
          (p.inline-r.overrideAttrs(old:{doCheck=false;}))
        ]))
283 284 285
      ];
    });

Valentin Reis's avatar
Valentin Reis committed
286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332
    f = { mkDerivation, base, protolude, data-default
    , optparse-applicative, typed-process, inline-r
    , R, stdenv }:
    mkDerivation {
      pname = "hsr";
      version = "1.0.0";
      src = "";
      isLibrary = true;
      isExecutable = false;
      libraryHaskellDepends = [
        base
        protolude
        inline-r
        typed-process
        optparse-applicative
        data-default
      ];
      librarySystemDepends = [
        pkgs.pandoc
        R
        pkgs.rPackages.docopt
        pkgs.rPackages.pracma
        pkgs.rPackages.tidyr
        pkgs.rPackages.readr
        pkgs.rPackages.magrittr
        pkgs.rPackages.formatR
        pkgs.rPackages.purrr
        pkgs.rPackages.wrapr
        pkgs.rPackages.knitr
        pkgs.rPackages.cowplot
        pkgs.rPackages.plotly
        pkgs.rPackages.lubridate
        pkgs.rPackages.dplyr
        pkgs.rPackages.ggplot2
        pkgs.rPackages.fpp2
        pkgs.rPackages.prospectr
        pkgs.rPackages.gridExtra
        pkgs.rPackages.zoo
        pkgs.rPackages.xtable
        pkgs.rPackages.ggthemes
        pkgs.rPackages.data_table
      ];
      license = stdenv.lib.licenses.bsd3;
    };
    hanal = argolib.getHackEnv (pkgs//argopkgs)
      haskellPackages (haskellPackages.callPackage f {});

Valentin Reis's avatar
Valentin Reis committed
333 334
    report = analysis.overrideAttrs (old:{
      buildPhase = ''
335 336
        echo "analysis/zymakefolder.R ${powerexpe}"
        analysis/zymakefolder.R ${powerexpe}
Valentin Reis's avatar
Valentin Reis committed
337 338 339 340 341 342
      '';
      installPhase = ''
        mkdir -p $out
        cp -r index* $out
      '';
    });
343

Valentin Reis's avatar
Valentin Reis committed
344 345 346 347 348 349 350 351 352 353 354 355 356 357 358
    dhrunSpecs = pkgs.stdenv.mkDerivation rec {
      name = "dhrunSpecs";
      src = ../dhrun ;
      installPhase = ''
        mkdir -p $out
        cp -r $src/* $out
        substituteInPlace $out/assets/simple-H2O.xml --replace \
          H2O.HF.wfs.xml $out/assets/H2O.HF.wfs.xml
        substituteInPlace $out/assets/simple-H2O.xml --replace \
          O.BFD.xml $out/assets/O.BFD.xml
        substituteInPlace $out/assets/simple-H2O.xml --replace \
          H.BFD.xml $out/assets/H.BFD.xml
        substituteInPlace $out/lib.dh --replace \
          "dataDir = \"./\"" "dataDir = \"$out/\""
        substituteInPlace $out/lib.dh --replace \
Valentin Reis's avatar
Valentin Reis committed
359 360
          "https://xgitlab.cels.anl.gov/argo/dhrun/raw/master/" "./"
        ln -s ${dhrun}/share/resources $out/resources
Valentin Reis's avatar
Valentin Reis committed
361 362 363 364
        '';
      unpackPhase = "true";
    };

Valentin Reis's avatar
Valentin Reis committed
365
    doDhrun = dhrscript: test.overrideAttrs (old:{
Valentin Reis's avatar
Valentin Reis committed
366
      buildPhase = ''
Valentin Reis's avatar
Valentin Reis committed
367 368 369 370 371
        cat ${dhrscript} |\
        dhall resolve |\
        dhall normalize |\
        dhall-to-yaml |\
        dhrun run "-"
Valentin Reis's avatar
Valentin Reis committed
372
      '';
Valentin Reis's avatar
Valentin Reis committed
373 374 375 376 377
      buildInputs = old.buildInputs ++ [
        dhall-json
        pkgs.yq
        dhall
      ];
Valentin Reis's avatar
Valentin Reis committed
378
      DHRUNSPECS = "${dhrunSpecs}";
Valentin Reis's avatar
Valentin Reis committed
379
      DHRUNSHARE = "${dhrun}/share";
Valentin Reis's avatar
Valentin Reis committed
380
      installPhase = ''
Valentin Reis's avatar
Valentin Reis committed
381
        mkdir -p $out
Valentin Reis's avatar
Valentin Reis committed
382 383 384 385
        cp _output/* $out/
      '';
    });

Valentin Reis's avatar
Valentin Reis committed
386 387 388 389
    doDhrunApp = app :
    let
      dhrunscript = pkgs.writeText "test.dhall" ''
        let all = ${dhrunSpecs}/all-tests.dh
390
        in all.${app} True < NoCap = {=} | Cap : Text >
Valentin Reis's avatar
Valentin Reis committed
391 392 393 394 395 396 397
      '';
    in
    doDhrun dhrunscript;

    doDhrunKey = key:
    let
      dhrunscript = pkgs.writeText "test.dhall" ''
Valentin Reis's avatar
Valentin Reis committed
398
        let all = ${dhrunSpecs}/all-tests.dh in all.${key}
Valentin Reis's avatar
Valentin Reis committed
399 400 401 402
      '';
    in
    doDhrun dhrunscript;

403
    testGeneric     = doDhrunKey genericTestName;
Valentin Reis's avatar
Valentin Reis committed
404
    testHello       = doDhrunKey "hello";
405
    testRun         = doDhrunKey "run";
Valentin Reis's avatar
Valentin Reis committed
406 407 408 409
    testListen      = doDhrunKey "listen";
    testPerfwrapper = doDhrunKey "performance";
    testPower       = doDhrunKey "power";

Valentin Reis's avatar
Valentin Reis committed
410 411 412 413 414 415
    testSTREAM      = addBI (doDhrunApp "stream") stream;
    testAMG         = addBI (doDhrunApp "amg") amg;
    testOpenMC      = (addBI (doDhrunApp "openmc") openmc).overrideAttrs
      (old:{ OPENMC_CROSS_SECTIONS = "${openmc-data}/cross_sections.xml"; });
    testLAMMPS      = addBI (doDhrunApp "lammps") lammps;
    testQMCPack     = addBI (doDhrunApp "qmcpack") qmcpack ;
Valentin Reis's avatar
Valentin Reis committed
416

417
    testAll         = (doDhrunKey "hello").overrideAttrs
Valentin Reis's avatar
Valentin Reis committed
418 419 420 421 422 423 424 425 426
    (old:{
      buildInputs = old.buildInputs ++ [
        lammps
        qmcpack
        openmc
        stream
        amg
        openmc-data
      ];
Valentin Reis's avatar
Valentin Reis committed
427 428
      OPENMC_CROSS_SECTIONS = "${openmc-data}/cross_sections.xml";
      });
Valentin Reis's avatar
Valentin Reis committed
429 430 431 432 433 434 435 436 437 438 439 440 441 442 443

    powerexpe = testAll.overrideAttrs (old:{
      name = "power-experiments";
      buildInputs = old.buildInputs ++ [
        #debug tools
        pkgs.which
        #experiment tools
        zymake
      ];
      buildPhase = "zymake ${experiments-src}/experiments/zymakefile";
      installPhase = ''
        mkdir -p $out
        cp -r o/* $out
      '';
    });
444 445

    bld = bleeding;
446
  };
Valentin Reis's avatar
Valentin Reis committed
447 448


449
# re-exporting a full nixpkgs-like package set
450
in pkgs // argopkgs