Commit 08a2534a authored by Valentin Reis's avatar Valentin Reis
Browse files

dhrun workflow refactor

parent 3bd455f1
let lib = ./lib.dh let lib = ./lib.dh
in let message = "HelloWorld" let message = "HelloWorld"
in let cn = "testContainer" let cn = "testContainer"
in let basicManifest = lib.manifestDir ++ "basic.json" let basicManifest = lib.manifestDir ++ "basic.json"
in let argIfTest = let argIfTest =
λ(isTest : Bool) λ(isTest : Bool)
→ { isTest = → { isTest =
isTest isTest
, timeout = , timeout =
if isTest if isTest then [ 600 ] : Optional Natural else [] : Optional Natural
}
then [ 600 ] : Optional Natural
else [] : Optional Natural
}
in let apps = let Powercap = < Cap : Text | NoCap : {} >
{ qmcpack =
λ(isTest : Bool) let apps =
→ lib.progressAppTest { qmcpack =
( lib.emptyProgressAppTestArg λ(isTest : Bool)
"mpiexec" → λ(powercap : Powercap)
[ "-n" → lib.extendApp
, if isTest then "2" else "24" ( lib.progressAppTest
, "qmcpack" ( lib.emptyProgressAppTestArg
, lib.qmcPackDir ++ "/simple-H2O.xml" "mpiexec"
] [ "-n"
⫽ argIfTest isTest , if isTest then "2" else "24"
⫽ { cmdavoids = [ "Error", "error", "ERROR" ] } , "qmcpack"
) , lib.qmcPackDir ++ "/simple-H2O.xml"
, lammps = ]
λ(isTest : Bool) ⫽ argIfTest isTest
→ lib.progressAppTest ⫽ { cmdavoids =
( lib.emptyProgressAppTestArg [ "Error", "error", "ERROR" ]
"mpiexec" , powercap =
[ "-n" powercap
, if isTest then "2" else "24"
, "-bind-to"
, "core"
, "lmp_mpi"
, "-i"
, lib.lammpsDir ++ "/modified.lj"
]
⫽ argIfTest isTest
⫽ { cmdavoids = [ "Error", "error", "ERROR" ] }
)
, openmc =
λ(isTest : Bool)
→ lib.progressAppTest
( lib.emptyProgressAppTestArg
"mpiexec"
[ "-n", if isTest then "2" else "24", "openmc" ]
⫽ argIfTest isTest
⫽ { passvars =
[ "OPENMC_CROSS_SECTIONS" ]
, cmdavoids =
[ "Error", "error", "ERROR" ]
}
)
⫽ { pre =
[ "cp --no-preserve=mode -r " ++ lib.openmcDir ++ "/* ." ]
} }
, amg = )
λ(isTest : Bool) )
→ lib.progressAppTest isTest
, lammps =
λ(isTest : Bool)
→ λ(powercap : Powercap)
→ lib.extendApp
( lib.progressAppTest
( lib.emptyProgressAppTestArg
"mpiexec"
[ "-n"
, if isTest then "2" else "24"
, "-bind-to"
, "core"
, "lmp_mpi"
, "-i"
, lib.lammpsDir ++ "/modified.lj"
]
⫽ argIfTest isTest
⫽ { cmdavoids =
[ "Error", "error", "ERROR" ]
, powercap =
powercap
}
)
)
isTest
, openmc =
λ(isTest : Bool)
→ λ(powercap : Powercap)
→ lib.extendApp
( lib.progressAppTest
( lib.emptyProgressAppTestArg ( lib.emptyProgressAppTestArg
"mpiexec" "mpiexec"
[ "-n" [ "-n", if isTest then "2" else "24", "openmc" ]
, if isTest then "2" else "24"
, "amg"
, "-problem"
, "2"
, "-n"
, "90"
, "90"
, "90"
, "-P"
, "2"
, if isTest then "1" else "12"
, "1"
]
⫽ argIfTest isTest
⫽ { vars =
[ { varname = "OMP_NUM_THREADS", value = "1" } ]
, cmdavoids =
[ "Error", "error", "ERROR" ]
}
)
, stream =
λ(isTest : Bool)
→ lib.progressAppTest
( lib.emptyProgressAppTestArg "stream_c" ([] : List Text)
⫽ argIfTest isTest ⫽ argIfTest isTest
⫽ { vars = ⫽ { passvars =
[ { varname = [ "OPENMC_CROSS_SECTIONS" ]
"OMP_NUM_THREADS"
, value =
if isTest then "2" else "24"
}
, { varname = "OMP_PLACES", value = "cores" }
]
, cmdavoids = , cmdavoids =
[ "Error", "error", "ERROR" ] [ "Error", "error", "ERROR" ]
, powercap =
powercap
} }
) )
} ⫽ { pre =
[ "cp --no-preserve=mode -r " ++ lib.openmcDir ++ "/* ." ]
}
)
isTest
, amg =
λ(isTest : Bool)
→ λ(powercap : Powercap)
→ lib.extendApp
( lib.progressAppTest
( lib.emptyProgressAppTestArg
"mpiexec"
[ "-n"
, if isTest then "2" else "24"
, "amg"
, "-problem"
, "2"
, "-n"
, "90"
, "90"
, "90"
, "-P"
, "2"
, if isTest then "1" else "12"
, "1"
]
⫽ argIfTest isTest
⫽ { vars =
[ { varname = "OMP_NUM_THREADS", value = "1" } ]
, cmdavoids =
[ "Error", "error", "ERROR" ]
, powercap =
powercap
}
)
)
isTest
, stream =
λ(isTest : Bool)
→ λ(powercap : Powercap)
→ lib.extendApp
( lib.progressAppTest
( lib.emptyProgressAppTestArg "stream_c" ([] : List Text)
⫽ argIfTest isTest
⫽ { vars =
[ { varname =
"OMP_NUM_THREADS"
, value =
if isTest then "2" else "24"
}
, { varname = "OMP_PLACES", value = "cores" }
]
, cmdavoids =
[ "Error", "error", "ERROR" ]
, powercap =
powercap
}
)
)
isTest
}
in { hello = in { apps =
apps
, hello =
lib.makeBase lib.makeBase
( lib.emptyMakeBaseArgs "echo" basicManifest cn ( lib.emptyMakeBaseArgs "echo" basicManifest cn
⫽ { args = ⫽ { args =
...@@ -174,23 +208,13 @@ in { hello = ...@@ -174,23 +208,13 @@ in { hello =
} }
) )
, stream = , stream =
apps.stream True apps.stream True < NoCap = {=} | Cap : Text >
, openmc = , openmc =
apps.openmc True apps.openmc True < NoCap = {=} | Cap : Text >
, lammps = , lammps =
apps.lammps True apps.lammps True < NoCap = {=} | Cap : Text >
, qmcpack = , qmcpack =
apps.qmcpack True apps.qmcpack True < NoCap = {=} | Cap : Text >
, amg = , amg =
apps.amg True apps.amg True < NoCap = {=} | Cap : Text >
, streamRun =
apps.stream False
, openmcRun =
apps.openmc False
, lammpsRun =
apps.lammps False
, qmcpackRun =
apps.qmcpack False
, amgRun =
apps.amg False
} }
let values = let values =
https://xgitlab.cels.anl.gov/argo/dhrun/raw/master/resources/package.dh https://xgitlab.cels.anl.gov/argo/dhrun/raw/master/resources/package.dh
in let types = let types =
https://xgitlab.cels.anl.gov/argo/dhrun/raw/master/resources/types.dh https://xgitlab.cels.anl.gov/argo/dhrun/raw/master/resources/types.dh
in let dataDir = env:DHRUNSPECS as Text ++ "/" let dataDir = "./"
in let manifestDir = dataDir ++ "manifests/" let manifestDir = dataDir ++ "manifests/"
in let assetDir = dataDir ++ "assets/" let assetDir = dataDir ++ "assets/"
in let openmcDir = assetDir ++ "openmc/" let openmcDir = assetDir ++ "openmc/"
in let lammpsDir = assetDir let lammpsDir = assetDir
in let qmcPackDir = assetDir let qmcPackDir = assetDir
in let basicCommand = let basicCommand =
λ ( args λ ( args
: { name : : { name :
Text Text
, outprefix : , outprefix :
Text Text
, outwants : , outwants :
List Text List Text
, outavoids : , outavoids :
List Text List Text
} }
) )
→ { name = → { name =
args.name args.name
, args = , args =
[] : List Text [] : List Text
, out = , out =
{ filename = { filename =
"${args.outprefix}.out" "${args.outprefix}.out"
, filecheck = , filecheck =
{ wants = args.outwants, avoids = args.outavoids } { wants = args.outwants, avoids = args.outavoids }
}
, err =
{ filename =
"${args.outprefix}.err"
, filecheck =
values.emptyCheck ⫽ { avoids = args.outavoids }
}
, postchecks =
[] : List types.FileCheck
, vars =
values.emptyVars
, passvars =
[ "PATH" ]
, timeout =
[] : Optional Natural
} }
, err =
{ filename =
"${args.outprefix}.err"
, filecheck =
values.emptyCheck ⫽ { avoids = args.outavoids }
}
, postchecks =
[] : List types.FileCheck
, vars =
values.emptyVars
, passvars =
[ "PATH" ]
, timeout =
[] : Optional Natural
}
let Powercap = < Cap : Text | NoCap : {} >
let MakeBaseArgs =
{ manifest :
Text
, cmd :
Text
, containerName :
Text
, args :
List Text
, verbose :
Bool
, cmdwants :
List Text
, cmdavoids :
List Text
, vars :
List types.EnvVar
, timeout :
Optional Natural
, passvars :
List Text
, powercap :
Powercap
}
in let MakeBaseArgs = let emptyMakeBaseArgs =
{ manifest : λ(cmd : Text)
Text → λ(manifest : Text)
, cmd : → λ(containerName : Text)
Text → { manifest =
, containerName : manifest
Text , cmd =
, args : cmd
List Text , containerName =
, verbose : containerName
Bool , args =
, cmdwants : [] : List Text
List Text , verbose =
, cmdavoids : False
List Text , cmdwants =
, vars : [] : List Text
List types.EnvVar , cmdavoids =
, passvars : [] : List Text
List Text , vars =
, timeout : values.emptyVars
Optional Natural , timeout =
[] : Optional Natural
, passvars =
[] : List Text
, powercap =
< NoCap = {=} | Cap : Text >
} }
: MakeBaseArgs
in let emptyMakeBaseArgs = let addVerbose = λ(v : Bool) → if v then [ "--verbose" ] else [] : List Text
λ(cmd : Text)
→ λ(manifest : Text)
→ λ(containerName : Text)
→ { manifest =
manifest
, cmd =
cmd
, containerName =
containerName
, args =
[] : List Text
, verbose =
False
, cmdwants =
[] : List Text
, cmdavoids =
[] : List Text
, vars =
values.emptyVars
, passvars =
[] : List Text
}
in let addVerbose = λ(v : Bool) → if v then [ "--verbose" ] else [] : List Text let addPowercap =
λ(p : Powercap)
→ merge
{ Cap =
λ(x : Text) → [ "--powercap", x ]
, NoCap =
λ(x : {}) → [] : List Text
}
p
in let makeBase = let makeBase =
λ(args : MakeBaseArgs) λ(args : MakeBaseArgs)
→ { cmds = → { cmds =
[ basicCommand [ basicCommand
{ name =
"nrmd"
, outprefix =
"nrmd"
, outwants =
[] : List Text
, outavoids =
[ "Traceback" ] : List Text
}
⫽ { args =
[ "--nrm_log", "./nrm_log" ]
# addVerbose args.verbose
# addPowercap args.powercap
, vars =
[ { varname =
"ARGO_NODEOS_CONFIG"
, value =
"/tmp/argo_nodeos_config"
}
]
}
, let b =
basicCommand
{ name = { name =
"nrmd" "nrm"
, outprefix = , outprefix =
"nrmd" "nrmrun"
, outwants = , outwants =
[] : List Text args.cmdwants
, outavoids = , outavoids =
[ "Traceback" ] : List Text args.cmdavoids
} }
in b
⫽ { args = ⫽ { args =
[ "--nrm_log", "./nrm_log" ] # addVerbose args.verbose [ "run"
, vars = , "-u"
[ { varname = , args.containerName
"ARGO_NODEOS_CONFIG" , args.manifest
, value = , args.cmd
"/tmp/argo_nodeos_config" ]
} # args.args
] , passvars =
b.passvars # args.passvars
, timeout =
args.timeout
} }
, let b = ]
basicCommand , verbose =
{ name = True
"nrm" , cleaning =
, outprefix = True
"nrmrun" , workdir =
, outwants = "./_output"
args.cmdwants , pre =
, outavoids = [] : List Text
args.cmdavoids , post =
} [] : List Text
}
in b : types.Config
⫽ { args =
[ "run" let makeExtended =
, "-u" λ(args : MakeBaseArgs)
, args.containerName → λ(cmds : List types.Cmd)
, args.manifest → let b = makeBase args in b ⫽ { cmds = b.cmds # cmds }
, args.cmd
]
# args.args
, passvars =
b.passvars # args.passvars
, timeout =
args.timeout
}
]
, verbose =
True
, cleaning =
True
, workdir =
"./_output"
, pre =
[] : List Text
, post =
[] : List Text
}
: types.Config
in let makeExtended = let OptionalSleep = < Cmd : Text | Sleep : {} >
λ(args : MakeBaseArgs)
→ λ(cmds : List types.Cmd)
→ let b = makeBase args in b ⫽ { cmds = b.cmds # cmds }
in let OptionalSleep = < Cmd : Text | Sleep : {} > let FilterTestArgs =
{ manifestname :
Text
, filter :
Text
, cmd :
Text
, args :
List Text
, vars :
List types.EnvVar
, passvars :
List Text
, cmdavoids :
List Text
, isTest :
Bool
, timeout :
Optional Natural
, powercap :
Powercap
}
in let FilterTestArgs = let emptyFilterTestArgs =
{ manifestname : λ(filter : Text)
Text → λ(cmd : Text)
, filter : → { cmd =
Text cmd
, cmd : , manifestname =
Text