Commit 08a2534a authored by Valentin Reis's avatar Valentin Reis
Browse files

dhrun workflow refactor

parent 3bd455f1
let lib = ./lib.dh let lib = ./lib.dh
in let message = "HelloWorld" let message = "HelloWorld"
in let cn = "testContainer" let cn = "testContainer"
in let basicManifest = lib.manifestDir ++ "basic.json" let basicManifest = lib.manifestDir ++ "basic.json"
in let argIfTest = let argIfTest =
λ(isTest : Bool) λ(isTest : Bool)
→ { isTest = → { isTest =
isTest isTest
, timeout = , timeout =
if isTest if isTest then [ 600 ] : Optional Natural else [] : Optional Natural
then [ 600 ] : Optional Natural
else [] : Optional Natural
} }
in let apps = let Powercap = < Cap : Text | NoCap : {} >
let apps =
{ qmcpack = { qmcpack =
λ(isTest : Bool) λ(isTest : Bool)
→ lib.progressAppTest → λ(powercap : Powercap)
→ lib.extendApp
( lib.progressAppTest
( lib.emptyProgressAppTestArg ( lib.emptyProgressAppTestArg
"mpiexec" "mpiexec"
[ "-n" [ "-n"
...@@ -30,11 +30,19 @@ in let apps = ...@@ -30,11 +30,19 @@ in let apps =
, lib.qmcPackDir ++ "/simple-H2O.xml" , lib.qmcPackDir ++ "/simple-H2O.xml"
] ]
⫽ argIfTest isTest ⫽ argIfTest isTest
⫽ { cmdavoids = [ "Error", "error", "ERROR" ] } ⫽ { cmdavoids =
[ "Error", "error", "ERROR" ]
, powercap =
powercap
}
) )
)
isTest
, lammps = , lammps =
λ(isTest : Bool) λ(isTest : Bool)
→ lib.progressAppTest → λ(powercap : Powercap)
→ lib.extendApp
( lib.progressAppTest
( lib.emptyProgressAppTestArg ( lib.emptyProgressAppTestArg
"mpiexec" "mpiexec"
[ "-n" [ "-n"
...@@ -46,11 +54,19 @@ in let apps = ...@@ -46,11 +54,19 @@ in let apps =
, lib.lammpsDir ++ "/modified.lj" , lib.lammpsDir ++ "/modified.lj"
] ]
⫽ argIfTest isTest ⫽ argIfTest isTest
⫽ { cmdavoids = [ "Error", "error", "ERROR" ] } ⫽ { cmdavoids =
[ "Error", "error", "ERROR" ]
, powercap =
powercap
}
) )
)
isTest
, openmc = , openmc =
λ(isTest : Bool) λ(isTest : Bool)
→ lib.progressAppTest → λ(powercap : Powercap)
→ lib.extendApp
( lib.progressAppTest
( lib.emptyProgressAppTestArg ( lib.emptyProgressAppTestArg
"mpiexec" "mpiexec"
[ "-n", if isTest then "2" else "24", "openmc" ] [ "-n", if isTest then "2" else "24", "openmc" ]
...@@ -59,14 +75,20 @@ in let apps = ...@@ -59,14 +75,20 @@ in let apps =
[ "OPENMC_CROSS_SECTIONS" ] [ "OPENMC_CROSS_SECTIONS" ]
, cmdavoids = , cmdavoids =
[ "Error", "error", "ERROR" ] [ "Error", "error", "ERROR" ]
, powercap =
powercap
} }
) )
⫽ { pre = ⫽ { pre =
[ "cp --no-preserve=mode -r " ++ lib.openmcDir ++ "/* ." ] [ "cp --no-preserve=mode -r " ++ lib.openmcDir ++ "/* ." ]
} }
)
isTest
, amg = , amg =
λ(isTest : Bool) λ(isTest : Bool)
→ lib.progressAppTest → λ(powercap : Powercap)
→ lib.extendApp
( lib.progressAppTest
( lib.emptyProgressAppTestArg ( lib.emptyProgressAppTestArg
"mpiexec" "mpiexec"
[ "-n" [ "-n"
...@@ -88,11 +110,17 @@ in let apps = ...@@ -88,11 +110,17 @@ in let apps =
[ { varname = "OMP_NUM_THREADS", value = "1" } ] [ { varname = "OMP_NUM_THREADS", value = "1" } ]
, cmdavoids = , cmdavoids =
[ "Error", "error", "ERROR" ] [ "Error", "error", "ERROR" ]
, powercap =
powercap
} }
) )
)
isTest
, stream = , stream =
λ(isTest : Bool) λ(isTest : Bool)
→ lib.progressAppTest → λ(powercap : Powercap)
→ lib.extendApp
( lib.progressAppTest
( lib.emptyProgressAppTestArg "stream_c" ([] : List Text) ( lib.emptyProgressAppTestArg "stream_c" ([] : List Text)
⫽ argIfTest isTest ⫽ argIfTest isTest
⫽ { vars = ⫽ { vars =
...@@ -105,11 +133,17 @@ in let apps = ...@@ -105,11 +133,17 @@ in let apps =
] ]
, cmdavoids = , cmdavoids =
[ "Error", "error", "ERROR" ] [ "Error", "error", "ERROR" ]
, powercap =
powercap
} }
) )
)
isTest
} }
in { hello = in { apps =
apps
, hello =
lib.makeBase lib.makeBase
( lib.emptyMakeBaseArgs "echo" basicManifest cn ( lib.emptyMakeBaseArgs "echo" basicManifest cn
⫽ { args = ⫽ { args =
...@@ -174,23 +208,13 @@ in { hello = ...@@ -174,23 +208,13 @@ in { hello =
} }
) )
, stream = , stream =
apps.stream True apps.stream True < NoCap = {=} | Cap : Text >
, openmc = , openmc =
apps.openmc True apps.openmc True < NoCap = {=} | Cap : Text >
, lammps = , lammps =
apps.lammps True apps.lammps True < NoCap = {=} | Cap : Text >
, qmcpack = , qmcpack =
apps.qmcpack True apps.qmcpack True < NoCap = {=} | Cap : Text >
, amg = , amg =
apps.amg True apps.amg True < NoCap = {=} | Cap : Text >
, streamRun =
apps.stream False
, openmcRun =
apps.openmc False
, lammpsRun =
apps.lammps False
, qmcpackRun =
apps.qmcpack False
, amgRun =
apps.amg False
} }
let values = let values =
https://xgitlab.cels.anl.gov/argo/dhrun/raw/master/resources/package.dh https://xgitlab.cels.anl.gov/argo/dhrun/raw/master/resources/package.dh
in let types = let types =
https://xgitlab.cels.anl.gov/argo/dhrun/raw/master/resources/types.dh https://xgitlab.cels.anl.gov/argo/dhrun/raw/master/resources/types.dh
in let dataDir = env:DHRUNSPECS as Text ++ "/" let dataDir = "./"
in let manifestDir = dataDir ++ "manifests/" let manifestDir = dataDir ++ "manifests/"
in let assetDir = dataDir ++ "assets/" let assetDir = dataDir ++ "assets/"
in let openmcDir = assetDir ++ "openmc/" let openmcDir = assetDir ++ "openmc/"
in let lammpsDir = assetDir let lammpsDir = assetDir
in let qmcPackDir = assetDir let qmcPackDir = assetDir
in let basicCommand = let basicCommand =
λ ( args λ ( args
: { name : : { name :
Text Text
...@@ -54,7 +54,9 @@ in let basicCommand = ...@@ -54,7 +54,9 @@ in let basicCommand =
[] : Optional Natural [] : Optional Natural
} }
in let MakeBaseArgs = let Powercap = < Cap : Text | NoCap : {} >
let MakeBaseArgs =
{ manifest : { manifest :
Text Text
, cmd : , cmd :
...@@ -71,13 +73,15 @@ in let MakeBaseArgs = ...@@ -71,13 +73,15 @@ in let MakeBaseArgs =
List Text List Text
, vars : , vars :
List types.EnvVar List types.EnvVar
, passvars :
List Text
, timeout : , timeout :
Optional Natural Optional Natural
, passvars :
List Text
, powercap :
Powercap
} }
in let emptyMakeBaseArgs = let emptyMakeBaseArgs =
λ(cmd : Text) λ(cmd : Text)
→ λ(manifest : Text) → λ(manifest : Text)
→ λ(containerName : Text) → λ(containerName : Text)
...@@ -97,13 +101,28 @@ in let emptyMakeBaseArgs = ...@@ -97,13 +101,28 @@ in let emptyMakeBaseArgs =
[] : List Text [] : List Text
, vars = , vars =
values.emptyVars values.emptyVars
, timeout =
[] : Optional Natural
, passvars = , passvars =
[] : List Text [] : List Text
, powercap =
< NoCap = {=} | Cap : Text >
} }
: MakeBaseArgs
let addVerbose = λ(v : Bool) → if v then [ "--verbose" ] else [] : List Text
in let addVerbose = λ(v : Bool) → if v then [ "--verbose" ] else [] : List Text let addPowercap =
λ(p : Powercap)
→ merge
{ Cap =
λ(x : Text) → [ "--powercap", x ]
, NoCap =
λ(x : {}) → [] : List Text
}
p
in let makeBase = let makeBase =
λ(args : MakeBaseArgs) λ(args : MakeBaseArgs)
→ { cmds = → { cmds =
[ basicCommand [ basicCommand
...@@ -117,7 +136,9 @@ in let makeBase = ...@@ -117,7 +136,9 @@ in let makeBase =
[ "Traceback" ] : List Text [ "Traceback" ] : List Text
} }
⫽ { args = ⫽ { args =
[ "--nrm_log", "./nrm_log" ] # addVerbose args.verbose [ "--nrm_log", "./nrm_log" ]
# addVerbose args.verbose
# addPowercap args.powercap
, vars = , vars =
[ { varname = [ { varname =
"ARGO_NODEOS_CONFIG" "ARGO_NODEOS_CONFIG"
...@@ -166,14 +187,14 @@ in let makeBase = ...@@ -166,14 +187,14 @@ in let makeBase =
} }
: types.Config : types.Config
in let makeExtended = let makeExtended =
λ(args : MakeBaseArgs) λ(args : MakeBaseArgs)
→ λ(cmds : List types.Cmd) → λ(cmds : List types.Cmd)
→ let b = makeBase args in b ⫽ { cmds = b.cmds # cmds } → let b = makeBase args in b ⫽ { cmds = b.cmds # cmds }
in let OptionalSleep = < Cmd : Text | Sleep : {} > let OptionalSleep = < Cmd : Text | Sleep : {} >
in let FilterTestArgs = let FilterTestArgs =
{ manifestname : { manifestname :
Text Text
, filter : , filter :
...@@ -192,9 +213,11 @@ in let FilterTestArgs = ...@@ -192,9 +213,11 @@ in let FilterTestArgs =
Bool Bool
, timeout : , timeout :
Optional Natural Optional Natural
, powercap :
Powercap
} }
in let emptyFilterTestArgs = let emptyFilterTestArgs =
λ(filter : Text) λ(filter : Text)
→ λ(cmd : Text) → λ(cmd : Text)
→ { cmd = → { cmd =
...@@ -215,21 +238,19 @@ in let emptyFilterTestArgs = ...@@ -215,21 +238,19 @@ in let emptyFilterTestArgs =
False False
, timeout = , timeout =
[] : Optional Natural [] : Optional Natural
, powercap =
< NoCap = {=} | Cap : Text >
} }
: FilterTestArgs
in let appTest = let appTest =
λ(args : FilterTestArgs) λ(args : FilterTestArgs)
→ let cn = "testContainer" → let cn = "testContainer"
in makeExtended in makeExtended
{ manifest = { manifest =
manifestDir manifestDir
++ ( if args.isTest ++ (if args.isTest then args.manifestname else "parallel.json")
then args.manifestname
else "parallel.json"
)
, containerName = , containerName =
cn cn
, cmd = , cmd =
...@@ -248,6 +269,8 @@ in let appTest = ...@@ -248,6 +269,8 @@ in let appTest =
args.passvars args.passvars
, timeout = , timeout =
args.timeout args.timeout
, powercap =
args.powercap
} }
[ let b = [ let b =
values.emptyCmd values.emptyCmd
...@@ -277,7 +300,7 @@ in let appTest = ...@@ -277,7 +300,7 @@ in let appTest =
} }
] ]
in let ProgressAppTestArg = let ProgressAppTestArg =
{ cmd : { cmd :
Text Text
, args : , args :
...@@ -292,9 +315,11 @@ in let ProgressAppTestArg = ...@@ -292,9 +315,11 @@ in let ProgressAppTestArg =
Bool Bool
, timeout : , timeout :
Optional Natural Optional Natural
, powercap :
Powercap
} }
in let emptyProgressAppTestArg = let emptyProgressAppTestArg =
λ(cmd : Text) λ(cmd : Text)
→ λ(args : List Text) → λ(args : List Text)
→ { cmd = → { cmd =
...@@ -311,10 +336,12 @@ in let emptyProgressAppTestArg = ...@@ -311,10 +336,12 @@ in let emptyProgressAppTestArg =
False False
, timeout = , timeout =
[] : Optional Natural [] : Optional Natural
, powercap =
< NoCap = {=} | Cap : Text >
} }
: ProgressAppTestArg : ProgressAppTestArg
in let progressAppTest = let progressAppTest =
λ(args : ProgressAppTestArg) λ(args : ProgressAppTestArg)
→ appTest → appTest
( emptyFilterTestArgs "progress" args.cmd ( emptyFilterTestArgs "progress" args.cmd
...@@ -329,12 +356,40 @@ in let progressAppTest = ...@@ -329,12 +356,40 @@ in let progressAppTest =
, cmdavoids = , cmdavoids =
args.cmdavoids args.cmdavoids
, isTest = , isTest =
True args.isTest
, timeout = , timeout =
args.timeout args.timeout
, powercap =
args.powercap
} }
) )
let mkListen =
λ(cn : Text)
→ λ(filter : Text)
→ let b =
values.emptyCmd
{ name =
"nrm"
, outfile =
filter ++ ".out"
, errfile =
filter ++ ".err"
}
in b ⫽ { args = [ "listen", "-u", cn, "--filter", filter ] }
let powerexpeCmds =
[ mkListen "testContainer" "power"
, mkListen "testContainer" "performance"
, mkListen "testContainer" "control"
] : List types.Cmd
let extendApp =
λ(c : types.Config)
→ λ(isTest : Bool)
→ if isTest then c else c ⫽ { cmds = c.cmds # powerexpeCmds }
in { values = in { values =
values values
, emptyMakeBaseArgs = , emptyMakeBaseArgs =
...@@ -359,4 +414,8 @@ in { values = ...@@ -359,4 +414,8 @@ in { values =
manifestDir manifestDir
, emptyProgressAppTestArg = , emptyProgressAppTestArg =
emptyProgressAppTestArg emptyProgressAppTestArg
, powerexpeCmds =
powerexpeCmds
, extendApp =
extendApp
} }
\ No newline at end of file
...@@ -2,13 +2,13 @@ ...@@ -2,13 +2,13 @@
"fetch": { "fetch": {
"args": { "args": {
"fetchSubmodules": false, "fetchSubmodules": false,
"rev": "cb0e20d6db96fe09a501076c7a2c265359982814", "rev": "6d7ed964296202023f72f18b964dc18d74a34f42",
"sha256": "0d9pkbax0phh392j6pzkn365wbsgd0h1cmm58rwq8zf9lb0pgkg2", "sha256": "1rbylp6hfwpqzi1dgrqyr6micynyswnw7wsnz5pmvgr2zxawd149",
"url": "https://github.com/nixos/nixpkgs-channels.git" "url": "https://github.com/nixos/nixpkgs-channels.git"
}, },
"fn": "fetchgit" "fn": "fetchgit"
}, },
"rev": "refs/heads/nixos-18.03", "rev": "refs/heads/nixos-19.03",
"type": "fetchgit", "type": "fetchgit",
"url": "https://github.com/nixos/nixpkgs-channels.git" "url": "https://github.com/nixos/nixpkgs-channels.git"
} }
\ No newline at end of file
...@@ -5,7 +5,7 @@ ...@@ -5,7 +5,7 @@
pkgs ? import (hostPkgs.nix-update-source.fetch ./pkgs.json).src {} pkgs ? import (hostPkgs.nix-update-source.fetch ./pkgs.json).src {}
, # versioned nixpkgs-older version for zmcat , # versioned nixpkgs-older version for zmcat
pkgs-old ? import (hostPkgs.nix-update-source.fetch ./pkgs-old.json).src {} bleeding ? import (hostPkgs.nix-update-source.fetch ./bleeding.json).src {}
, # fetcher alias for the remaining arguments , # fetcher alias for the remaining arguments
fetched ? s: (pkgs.nix-update-source.fetch s).src fetched ? s: (pkgs.nix-update-source.fetch s).src
...@@ -123,24 +123,29 @@ let ...@@ -123,24 +123,29 @@ let
inline-r = dontCheck super.inline-r; inline-r = dontCheck super.inline-r;
argonix = (self.callCabal2nix "argonix" argonix-src ) {}; argonix = (self.callCabal2nix "argonix" argonix-src ) {};
repoquality = (self.callCabal2nix "repoquality" repoquality-src ) {}; repoquality = (self.callCabal2nix "repoquality" repoquality-src ) {};
dhrun = (self.callCabal2nix "dhrun" dhrun-src ) {};
powerplots = (self.callCabal2nix "powerplots" powerplots-src ) {}; powerplots = (self.callCabal2nix "powerplots" powerplots-src ) {};
}; };
}; };
bleedingHaskellPackages = bleeding.haskellPackages.override {
overrides = self: super: with bleeding.haskell.lib;
rec {
dhrun = (self.callCabal2nix "dhrun" dhrun-src ) {};
};
};
# re-export from hpkgs and add non-haskell build rule specialization # re-export from hpkgs and add non-haskell build rule specialization
inherit (haskellPackages) repoquality; inherit (haskellPackages) repoquality;
inherit (bleedingHaskellPackages) dhall dhall-json;
dhrun = haskellPackages.dhrun.overrideAttrs (old:{ dhrun = bleedingHaskellPackages.dhrun.overrideAttrs (old:{
installPhase = old.installPhase + '' installPhase = old.installPhase + ''
mkdir -p $out/share/ mkdir -p $out/share/
cp -r resources $out/share/ cp -r resources $out/share/
''; '';
}); });
dhrun-tests = dhrun.overrideAttrs (_:{doCheck=true;});
argonix = haskellPackages.argonix.overrideAttrs (old:{ argonix = haskellPackages.argonix.overrideAttrs (old:{
installPhase = old.installPhase + '' installPhase = old.installPhase + ''
mkdir -p $out/share/ mkdir -p $out/share/
...@@ -179,10 +184,6 @@ let ...@@ -179,10 +184,6 @@ let
}; };
}); });
# archived for sake of the simplicity of this file
zmcat = pkgs-old.haskellPackages.callPackage ./zmcat {};