Commit 08a2534a authored by Valentin Reis's avatar Valentin Reis
Browse files

dhrun workflow refactor

parent 3bd455f1
let lib = ./lib.dh
let lib = ./lib.dh
in let message = "HelloWorld"
let message = "HelloWorld"
in let cn = "testContainer"
let cn = "testContainer"
in let basicManifest = lib.manifestDir ++ "basic.json"
let basicManifest = lib.manifestDir ++ "basic.json"
in let argIfTest =
let argIfTest =
λ(isTest : Bool)
→ { isTest =
isTest
, timeout =
if isTest
then [ 600 ] : Optional Natural
else [] : Optional Natural
if isTest then [ 600 ] : Optional Natural else [] : Optional Natural
}
in let apps =
let Powercap = < Cap : Text | NoCap : {} >
let apps =
{ qmcpack =
λ(isTest : Bool)
→ lib.progressAppTest
→ λ(powercap : Powercap)
→ lib.extendApp
( lib.progressAppTest
( lib.emptyProgressAppTestArg
"mpiexec"
[ "-n"
......@@ -30,11 +30,19 @@ in let apps =
, lib.qmcPackDir ++ "/simple-H2O.xml"
]
⫽ argIfTest isTest
⫽ { cmdavoids = [ "Error", "error", "ERROR" ] }
⫽ { cmdavoids =
[ "Error", "error", "ERROR" ]
, powercap =
powercap
}
)
)
isTest
, lammps =
λ(isTest : Bool)
→ lib.progressAppTest
→ λ(powercap : Powercap)
→ lib.extendApp
( lib.progressAppTest
( lib.emptyProgressAppTestArg
"mpiexec"
[ "-n"
......@@ -46,11 +54,19 @@ in let apps =
, lib.lammpsDir ++ "/modified.lj"
]
⫽ argIfTest isTest
⫽ { cmdavoids = [ "Error", "error", "ERROR" ] }
⫽ { cmdavoids =
[ "Error", "error", "ERROR" ]
, powercap =
powercap
}
)
)
isTest
, openmc =
λ(isTest : Bool)
→ lib.progressAppTest
→ λ(powercap : Powercap)
→ lib.extendApp
( lib.progressAppTest
( lib.emptyProgressAppTestArg
"mpiexec"
[ "-n", if isTest then "2" else "24", "openmc" ]
......@@ -59,14 +75,20 @@ in let apps =
[ "OPENMC_CROSS_SECTIONS" ]
, cmdavoids =
[ "Error", "error", "ERROR" ]
, powercap =
powercap
}
)
⫽ { pre =
[ "cp --no-preserve=mode -r " ++ lib.openmcDir ++ "/* ." ]
}
)
isTest
, amg =
λ(isTest : Bool)
→ lib.progressAppTest
→ λ(powercap : Powercap)
→ lib.extendApp
( lib.progressAppTest
( lib.emptyProgressAppTestArg
"mpiexec"
[ "-n"
......@@ -88,11 +110,17 @@ in let apps =
[ { varname = "OMP_NUM_THREADS", value = "1" } ]
, cmdavoids =
[ "Error", "error", "ERROR" ]
, powercap =
powercap
}
)
)
isTest
, stream =
λ(isTest : Bool)
→ lib.progressAppTest
→ λ(powercap : Powercap)
→ lib.extendApp
( lib.progressAppTest
( lib.emptyProgressAppTestArg "stream_c" ([] : List Text)
⫽ argIfTest isTest
⫽ { vars =
......@@ -105,11 +133,17 @@ in let apps =
]
, cmdavoids =
[ "Error", "error", "ERROR" ]
, powercap =
powercap
}
)
)
isTest
}
in { hello =
in { apps =
apps
, hello =
lib.makeBase
( lib.emptyMakeBaseArgs "echo" basicManifest cn
⫽ { args =
......@@ -174,23 +208,13 @@ in { hello =
}
)
, stream =
apps.stream True
apps.stream True < NoCap = {=} | Cap : Text >
, openmc =
apps.openmc True
apps.openmc True < NoCap = {=} | Cap : Text >
, lammps =
apps.lammps True
apps.lammps True < NoCap = {=} | Cap : Text >
, qmcpack =
apps.qmcpack True
apps.qmcpack True < NoCap = {=} | Cap : Text >
, amg =
apps.amg True
, streamRun =
apps.stream False
, openmcRun =
apps.openmc False
, lammpsRun =
apps.lammps False
, qmcpackRun =
apps.qmcpack False
, amgRun =
apps.amg False
apps.amg True < NoCap = {=} | Cap : Text >
}
let values =
let values =
https://xgitlab.cels.anl.gov/argo/dhrun/raw/master/resources/package.dh
in let types =
let types =
https://xgitlab.cels.anl.gov/argo/dhrun/raw/master/resources/types.dh
in let dataDir = env:DHRUNSPECS as Text ++ "/"
let dataDir = "./"
in let manifestDir = dataDir ++ "manifests/"
let manifestDir = dataDir ++ "manifests/"
in let assetDir = dataDir ++ "assets/"
let assetDir = dataDir ++ "assets/"
in let openmcDir = assetDir ++ "openmc/"
let openmcDir = assetDir ++ "openmc/"
in let lammpsDir = assetDir
let lammpsDir = assetDir
in let qmcPackDir = assetDir
let qmcPackDir = assetDir
in let basicCommand =
let basicCommand =
λ ( args
: { name :
Text
......@@ -54,7 +54,9 @@ in let basicCommand =
[] : Optional Natural
}
in let MakeBaseArgs =
let Powercap = < Cap : Text | NoCap : {} >
let MakeBaseArgs =
{ manifest :
Text
, cmd :
......@@ -71,13 +73,15 @@ in let MakeBaseArgs =
List Text
, vars :
List types.EnvVar
, passvars :
List Text
, timeout :
Optional Natural
, passvars :
List Text
, powercap :
Powercap
}
in let emptyMakeBaseArgs =
let emptyMakeBaseArgs =
λ(cmd : Text)
→ λ(manifest : Text)
→ λ(containerName : Text)
......@@ -97,13 +101,28 @@ in let emptyMakeBaseArgs =
[] : List Text
, vars =
values.emptyVars
, timeout =
[] : Optional Natural
, passvars =
[] : List Text
, powercap =
< NoCap = {=} | Cap : Text >
}
: MakeBaseArgs
let addVerbose = λ(v : Bool) → if v then [ "--verbose" ] else [] : List Text
in let addVerbose = λ(v : Bool) → if v then [ "--verbose" ] else [] : List Text
let addPowercap =
λ(p : Powercap)
→ merge
{ Cap =
λ(x : Text) → [ "--powercap", x ]
, NoCap =
λ(x : {}) → [] : List Text
}
p
in let makeBase =
let makeBase =
λ(args : MakeBaseArgs)
→ { cmds =
[ basicCommand
......@@ -117,7 +136,9 @@ in let makeBase =
[ "Traceback" ] : List Text
}
⫽ { args =
[ "--nrm_log", "./nrm_log" ] # addVerbose args.verbose
[ "--nrm_log", "./nrm_log" ]
# addVerbose args.verbose
# addPowercap args.powercap
, vars =
[ { varname =
"ARGO_NODEOS_CONFIG"
......@@ -166,14 +187,14 @@ in let makeBase =
}
: types.Config
in let makeExtended =
let makeExtended =
λ(args : MakeBaseArgs)
→ λ(cmds : List types.Cmd)
→ let b = makeBase args in b ⫽ { cmds = b.cmds # cmds }
in let OptionalSleep = < Cmd : Text | Sleep : {} >
let OptionalSleep = < Cmd : Text | Sleep : {} >
in let FilterTestArgs =
let FilterTestArgs =
{ manifestname :
Text
, filter :
......@@ -192,9 +213,11 @@ in let FilterTestArgs =
Bool
, timeout :
Optional Natural
, powercap :
Powercap
}
in let emptyFilterTestArgs =
let emptyFilterTestArgs =
λ(filter : Text)
→ λ(cmd : Text)
→ { cmd =
......@@ -215,21 +238,19 @@ in let emptyFilterTestArgs =
False
, timeout =
[] : Optional Natural
, powercap =
< NoCap = {=} | Cap : Text >
}
: FilterTestArgs
in let appTest =
let appTest =
λ(args : FilterTestArgs)
→ let cn = "testContainer"
in makeExtended
{ manifest =
manifestDir
++ ( if args.isTest
then args.manifestname
else "parallel.json"
)
++ (if args.isTest then args.manifestname else "parallel.json")
, containerName =
cn
, cmd =
......@@ -248,6 +269,8 @@ in let appTest =
args.passvars
, timeout =
args.timeout
, powercap =
args.powercap
}
[ let b =
values.emptyCmd
......@@ -277,7 +300,7 @@ in let appTest =
}
]
in let ProgressAppTestArg =
let ProgressAppTestArg =
{ cmd :
Text
, args :
......@@ -292,9 +315,11 @@ in let ProgressAppTestArg =
Bool
, timeout :
Optional Natural
, powercap :
Powercap
}
in let emptyProgressAppTestArg =
let emptyProgressAppTestArg =
λ(cmd : Text)
→ λ(args : List Text)
→ { cmd =
......@@ -311,10 +336,12 @@ in let emptyProgressAppTestArg =
False
, timeout =
[] : Optional Natural
, powercap =
< NoCap = {=} | Cap : Text >
}
: ProgressAppTestArg
in let progressAppTest =
let progressAppTest =
λ(args : ProgressAppTestArg)
→ appTest
( emptyFilterTestArgs "progress" args.cmd
......@@ -329,12 +356,40 @@ in let progressAppTest =
, cmdavoids =
args.cmdavoids
, isTest =
True
args.isTest
, timeout =
args.timeout
, powercap =
args.powercap
}
)
let mkListen =
λ(cn : Text)
→ λ(filter : Text)
→ let b =
values.emptyCmd
{ name =
"nrm"
, outfile =
filter ++ ".out"
, errfile =
filter ++ ".err"
}
in b ⫽ { args = [ "listen", "-u", cn, "--filter", filter ] }
let powerexpeCmds =
[ mkListen "testContainer" "power"
, mkListen "testContainer" "performance"
, mkListen "testContainer" "control"
] : List types.Cmd
let extendApp =
λ(c : types.Config)
→ λ(isTest : Bool)
→ if isTest then c else c ⫽ { cmds = c.cmds # powerexpeCmds }
in { values =
values
, emptyMakeBaseArgs =
......@@ -359,4 +414,8 @@ in { values =
manifestDir
, emptyProgressAppTestArg =
emptyProgressAppTestArg
, powerexpeCmds =
powerexpeCmds
, extendApp =
extendApp
}
\ No newline at end of file
......@@ -2,13 +2,13 @@
"fetch": {
"args": {
"fetchSubmodules": false,
"rev": "cb0e20d6db96fe09a501076c7a2c265359982814",
"sha256": "0d9pkbax0phh392j6pzkn365wbsgd0h1cmm58rwq8zf9lb0pgkg2",
"rev": "6d7ed964296202023f72f18b964dc18d74a34f42",
"sha256": "1rbylp6hfwpqzi1dgrqyr6micynyswnw7wsnz5pmvgr2zxawd149",
"url": "https://github.com/nixos/nixpkgs-channels.git"
},
"fn": "fetchgit"
},
"rev": "refs/heads/nixos-18.03",
"rev": "refs/heads/nixos-19.03",
"type": "fetchgit",
"url": "https://github.com/nixos/nixpkgs-channels.git"
}
\ No newline at end of file
......@@ -5,7 +5,7 @@
pkgs ? import (hostPkgs.nix-update-source.fetch ./pkgs.json).src {}
, # versioned nixpkgs-older version for zmcat
pkgs-old ? import (hostPkgs.nix-update-source.fetch ./pkgs-old.json).src {}
bleeding ? import (hostPkgs.nix-update-source.fetch ./bleeding.json).src {}
, # fetcher alias for the remaining arguments
fetched ? s: (pkgs.nix-update-source.fetch s).src
......@@ -123,24 +123,29 @@ let
inline-r = dontCheck super.inline-r;
argonix = (self.callCabal2nix "argonix" argonix-src ) {};
repoquality = (self.callCabal2nix "repoquality" repoquality-src ) {};
dhrun = (self.callCabal2nix "dhrun" dhrun-src ) {};
powerplots = (self.callCabal2nix "powerplots" powerplots-src ) {};
};
};
bleedingHaskellPackages = bleeding.haskellPackages.override {
overrides = self: super: with bleeding.haskell.lib;
rec {
dhrun = (self.callCabal2nix "dhrun" dhrun-src ) {};
};
};
# re-export from hpkgs and add non-haskell build rule specialization
inherit (haskellPackages) repoquality;
inherit (bleedingHaskellPackages) dhall dhall-json;
dhrun = haskellPackages.dhrun.overrideAttrs (old:{
dhrun = bleedingHaskellPackages.dhrun.overrideAttrs (old:{
installPhase = old.installPhase + ''
mkdir -p $out/share/
cp -r resources $out/share/
'';
});
dhrun-tests = dhrun.overrideAttrs (_:{doCheck=true;});
argonix = haskellPackages.argonix.overrideAttrs (old:{
installPhase = old.installPhase + ''
mkdir -p $out/share/
......@@ -179,10 +184,6 @@ let
};
});
# archived for sake of the simplicity of this file
zmcat = pkgs-old.haskellPackages.callPackage ./zmcat {};
zmqat = pkgs-old.haskellPackages.zmqat;
# ci/experimental environments
test = pkgs.stdenv.mkDerivation rec {
name = "testEnv";
......@@ -219,34 +220,6 @@ let
#OPENMC_PWD = "${argotk}/share/openmc";
};
testApplications = test.overrideAttrs (old:{
name = "testEnv-with-applications";
buildInputs = old.buildInputs ++ [
lammps
qmcpack
openmc
stream
amg
openmc-data
];
});
powerexpe = testApplications.overrideAttrs (old:{
name = "power-experiments";
buildInputs = old.buildInputs ++ [
#debug tools
pkgs.which
zmcat
#experiment tools
zymake
];
buildPhase = "zymake ${experiments-src}/experiments/zymakefile";
installPhase = ''
mkdir -p $out
cp -r o/* $out
'';
});
analysis = pkgs.stdenv.mkDerivation rec {
name = "analysis";
src = experiments-src;
......@@ -370,16 +343,34 @@ let
substituteInPlace $out/lib.dh --replace \
"dataDir = \"./\"" "dataDir = \"$out/\""
substituteInPlace $out/lib.dh --replace \
"https://xgitlab.cels.anl.gov/argo/dhrun/raw/master/" "${dhrun}/share/"
"https://xgitlab.cels.anl.gov/argo/dhrun/raw/master/" "./"
ln -s ${dhrun}/share/resources $out/resources
'';
unpackPhase = "true";
};
doDhrun = drv: command: drv.overrideAttrs (old:{
doDhrun = drv: app:
let
dhrscript = pkgs.writeText "test.dhall" ''
let all = ${dhrunSpecs}/all-tests.dh
in all.apps.${app} True < NoCap = {=} | Cap : Text >
'';
in
drv.overrideAttrs (old:{
buildPhase = ''
dhrun run $DHRUNSPECS/tests/${command}.dh
cat ${dhrscript} |\
dhall resolve |\
dhall normalize |\
dhall-to-yaml |\
dhrun run "-"
'';
buildInputs = old.buildInputs ++ [
dhall-json
pkgs.yq
dhall
];
DHRUNSPECS = "${dhrunSpecs}";
DHRUNSHARE = "${dhrun}/share";
installPhase = ''
mkdir -p $out
cp _output/* $out/
......@@ -413,6 +404,23 @@ let
openmc-data
];
OPENMC_CROSS_SECTIONS = "${openmc-data}/cross_sections.xml";})) "tests";
powerexpe = testAll.overrideAttrs (old:{
name = "power-experiments";
buildInputs = old.buildInputs ++ [
#debug tools
pkgs.which
#experiment tools
zymake
];
buildPhase = "zymake ${experiments-src}/experiments/zymakefile";
installPhase = ''
mkdir -p $out
cp -r o/* $out
'';
});
};
# re-exporting a full nixpkgs-like package set
in pkgs // argopkgs
......@@ -2,8 +2,8 @@
"fetch": {
"args": {
"fetchSubmodules": false,
"rev": "b0dae2397afcdeefc7d64d672f361b4c3b17d8e6",
"sha256": "1h1cqzb7zs13q14b6rksrz096v0m8xkqwpl8nwpgax7wwwcnfsh8",
"rev": "3a4d24a7e360e7a012deeb01180c4a4e7ab3e491",
"sha256": "1kindv8nks9gh3s54fy6dz1gnv7bxllmzpsbymfyj1jflwz9fvdf",
"url": "https://xgitlab.cels.anl.gov/argo/dhrun.git"
},
"fn": "fetchgit"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment