Commit 698ca2f3 authored by Valentin Reis's avatar Valentin Reis
Browse files

Merge branch 'dhall-refact' into 'master'

Dhall refactor

See merge request !6
parents 1c44133c afb64947
Pipeline #7701 failed with stages
in 1 minute and 45 seconds
#!/usr/bin/env bash
echo "restarted runner."
ssh root@140.221.10.9 systemctl restart gitlab-runner2.service
echo "restarted runner."
ssh root@129.114.111.114 systemctl restart gitlab-runner2.service
echo "restarted runner."
ssh root@129.114.110.3 systemctl restart gitlab-runner2.service
echo "restarted runner."
ssh root@129.114.111.116 systemctl restart gitlab-runner2.service
echo "restarted runner."
ssh root@129.114.33.201 systemctl restart gitlab-runner2.service
echo "restarted runner."
ssh gitlab-runner@argo-phi1 sudo systemctl restart gitlab-runner.service
ssh gitlab-runner@argo-phi1.mcs sudo systemctl restart gitlab-runner.service
echo "restarted runner."
λ(dataDir : Text)
-> λ(manifestDir : Text)
→ let lib = ./lib.dh dataDir manifestDir
→ λ(manifestDir : Text)
→ λ(isTest : Bool)
→ let assetDir = dataDir ++ "test_assets/"
let types = ./dhrunlib/types.dh
let argotypes = ./argolib/types.dh
let argo = ./argolib/package.dh
let message = "HelloWorld"
let cn = "testContainer"
let basicManifest = manifestDir ++ "basic.yml"
let idCmd = let id = λ(a : Type) → λ(x : a) → x in id types.Cmd
let singularityManifest = manifestDir ++ "singularity.yml"
let argIfTest =
λ(isTest : Bool)
→ { isTest =
isTest
, timeout =
if isTest
then [ 600 ] : Optional Natural
else [] : Optional Natural
let dhrunTemplate =
{ cmds =
[] : List types.Cmd
, verbose =
True
, cleaning =
True
, workdir =
"./_output"
, pre =
[] : List Text
, post =
[] : List Text
}
: types.Config
let Powercap = < Cap : Text | NoCap : {} >
let makeExtended =
λ(daemonArg : argotypes.MkDaemon)
→ λ(daemonOverrides : types.Cmd → types.Cmd)
→ λ(runArg : argotypes.MkRun)
→ λ(runOverrides : types.Cmd → types.Cmd)
→ λ(extraCmds : List types.Cmd)
→ dhrunTemplate
⫽ { cmds =
[ daemonOverrides (argo.nrmd.mk daemonArg)
, runOverrides (argo.run.mk runArg)
]
# extraCmds
}
in { run-singularity =
lib.makeBase
( lib.emptyMakeBaseArg "echo" singularityManifest cn
⫽ { args =
[ message ]
, verbose =
True
, singularity =
True
, cmdwants =
[] : List Text
, cmdavoids =
[ "Traceback" ]
, timeout =
[ 600 ] : Optional Natural
let manifests =
{ basic =
manifestDir ++ "basic.yml"
, perf =
manifestDir ++ "perfwrap.yml"
, singularity =
manifestDir ++ "singularity.yml"
}
let ifTest =
λ(t : Type)
→ λ(default : t)
→ λ(value : t)
→ if isTest then value else default
let ifTestList = ifTest (List Text) ([] : List Text)
let withDaemon =
λ(daemonArg : argotypes.MkDaemon)
→ λ(runArg : argotypes.MkRun)
→ λ(runOverrides : types.Cmd → types.Cmd)
→ λ(extraCmds : List types.Cmd)
→ makeExtended
( daemonArg
⫽ { daemonavoids =
ifTestList [ "Traceback", "ValueError" ]
, powercap =
< NoCap = {=} | Cap : Text > : argotypes.Powercap
}
)
, run =
lib.makeBase
( lib.emptyMakeBaseArg "echo" basicManifest cn
⫽ { args =
[ message ]
, verbose =
True
, cmdwants =
[] : List Text
, cmdavoids =
[ "Traceback" ]
, timeout =
[ 600 ] : Optional Natural
idCmd
( runArg
⫽ { cmdavoids =
runArg.cmdavoids # ifTestList [ "Traceback", "ValueError" ]
}
)
, pwd =
lib.makeBase
( lib.emptyMakeBaseArg "pwd" basicManifest cn
⫽ { args =
[] : List Text
, verbose =
True
, cmdwants =
[ "toto" ] : List Text
, cmdavoids =
[ "Traceback" ]
, timeout =
[ 600 ] : Optional Natural
, runwd =
[ "toto" ] : Optional Text
}
( λ(c : types.Cmd)
→ runOverrides
( c
⫽ { timeout =
ifTest
(Optional Natural)
([] : Optional Natural)
([ 600 ] : Optional Natural)
}
)
)
extraCmds
let withDaemonNoExtras =
λ(runArg : argotypes.MkRun)
→ λ(runOverrides : types.Cmd → types.Cmd)
→ withDaemon argo.nrmd.defarg runArg runOverrides ([] : List types.Cmd)
let withDaemonExtraListen =
λ(filter : Text)
→ λ(doFilter : Bool)
→ λ(runArg : argotypes.MkRun)
→ λ(runOverrides : types.Cmd → types.Cmd)
→ withDaemon
argo.nrmd.defarg
runArg
runOverrides
[ argo.listen.mk
(argo.listen.defarg cn ⫽ { filter = filter, doFilter = doFilter })
]
let withDaemonExtraProgresstest = withDaemonExtraListen "progress" True
let mkrun =
λ(manifest : Text) → λ(name : Text) → argo.run.defarg name manifest cn
let basicRun = mkrun manifests.basic
let perfRun = mkrun manifests.perf
let singularityRun = mkrun manifests.singularity
in { list =
dhrunTemplate ⫽ { cmds = [ argo.nrmd.default, argo.list.default ] }
, run =
withDaemonNoExtras (basicRun "true") idCmd
, pwd =
let somedir = "toto"
in withDaemonNoExtras
(basicRun "pwd" ⫽ { cmdwants = [ somedir ] : List Text })
(λ(x : types.Cmd) → x ⫽ { otherwd = [ somedir ] : Optional Text })
, exitcode =
lib.makeBase
( lib.emptyMakeBaseArg "false" basicManifest cn
⫽ { args =
[] : List Text
, verbose =
True
, cmdwants =
[] : List Text
, cmdavoids =
[ "Traceback" ]
, timeout =
[ 600 ] : Optional Natural
, runExitcode =
[ +1 ] : Optional Integer
}
)
withDaemonNoExtras
(basicRun "false")
(λ(x : types.Cmd) → x ⫽ { exitcode = [ +1 ] : Optional Integer })
, hello =
lib.makeBase
( lib.emptyMakeBaseArg "echo" basicManifest cn
⫽ { args =
[ message ]
, verbose =
True
, cmdwants =
[ message ]
, cmdavoids =
[ "Traceback" ]
, timeout =
[ 600 ] : Optional Natural
}
withDaemonNoExtras
( basicRun "echo"
⫽ { args = [ message ], cmdwants = [ message ] : List Text }
)
idCmd
, listen =
lib.makeExtended
( lib.emptyMakeBaseArg "sleep" basicManifest cn
⫽ { args =
[ "1" ]
, verbose =
True
, cmdwants =
[] : List Text
, cmdavoids =
[ "Traceback" ]
, timeout =
[ 600 ] : Optional Natural
}
)
[ lib.values.emptyCmd
{ name =
"nrm"
, outfile =
"nrmlisten.out"
, errfile =
"nrmlisten.err"
}
⫽ { args = [ "listen", "-u", cn ] }
]
withDaemonExtraListen
","
False
(basicRun "sleep" ⫽ { args = [ "1" ] })
idCmd
, power =
lib.appTest
( lib.emptyFilterTestArgs "power" "sleep"
⫽ { timeout =
[ 600 ] : Optional Natural
, args =
[ "15" ]
, isTest =
True
}
)
withDaemonExtraListen
"power"
True
(basicRun "sleep" ⫽ { args = [ "15" ] })
idCmd
, performance =
lib.appTest
( lib.emptyFilterTestArgs "performance" "sleep"
⫽ { timeout =
[ 600 ] : Optional Natural
, args =
[ "15" ]
, isTest =
True
, manifestname =
"perfwrap.yml"
}
)
withDaemonExtraListen
"performance"
True
(perfRun "sleep" ⫽ { args = [ "15" ] })
idCmd
, run-singularity =
withDaemon
(argo.nrmd.defarg ⫽ { singularity = True })
(singularityRun "true")
idCmd
([] : List types.Cmd)
, qmcpack =
λ(isTest : Bool)
→ λ(powercap : Powercap)
→ lib.extendApp
( lib.progressAppTest
( lib.emptyProgressAppTestArg
"mpiexec"
[ "-n"
, if isTest then "2" else "24"
withDaemonExtraProgresstest
( basicRun "mpiexec"
⫽ { args =
[ if isTest then "2" else "24"
, "qmcpack"
, lib.qmcPackDir ++ "/simple-H2O.xml"
, assetDir ++ "/simple-H2O.xml"
]
⫽ argIfTest isTest
⫽ { cmdavoids =
[ "Error", "error", "ERROR" ]
, powercap =
powercap
}
)
)
isTest
, cmdavoids =
[ "Error", "error", "ERROR" ]
}
)
idCmd
, lammps =
λ(isTest : Bool)
→ λ(powercap : Powercap)
→ lib.extendApp
( lib.progressAppTest
( lib.emptyProgressAppTestArg
"mpiexec"
withDaemonExtraProgresstest
( basicRun "mpiexec"
⫽ { args =
[ "-n"
, if isTest then "2" else "24"
, "-bind-to"
, "core"
, "lmp_mpi"
, "-i"
, lib.lammpsDir ++ "/modified.lj"
, assetDir ++ "/modified.lj"
]
⫽ argIfTest isTest
⫽ { cmdavoids =
[ "Error", "error", "ERROR" ]
, powercap =
powercap
}
)
)
isTest
, cmdavoids =
[ "Error", "error", "ERROR" ]
}
)
idCmd
, openmc =
λ(isTest : Bool)
→ λ(powercap : Powercap)
→ lib.extendApp
( lib.progressAppTest
( lib.emptyProgressAppTestArg
"mpiexec"
withDaemonExtraProgresstest
( basicRun "mpiexec"
⫽ { args =
[ "-n", if isTest then "2" else "24", "openmc" ]
⫽ argIfTest isTest
⫽ { passvars =
[ "OPENMC_CROSS_SECTIONS" ]
, cmdavoids =
[ "Error" ]
, powercap =
powercap
}
)
⫽ { pre =
[ "cp --no-preserve=mode -r " ++ lib.openmcDir ++ "/* ." ]
, cmdavoids =
[ "Error", "error", "ERROR" ]
, passvars =
[ "OPENMC_CROSS_SECTIONS" ]
}
)
isTest
idCmd
⫽ { pre =
[ "cp --no-preserve=mode -r " ++ assetDir ++ "openmc/* ." ]
}
, amg =
λ(isTest : Bool)
→ λ(powercap : Powercap)
→ lib.extendApp
( lib.progressAppTest
( lib.emptyProgressAppTestArg
"mpiexec"
withDaemonExtraProgresstest
( basicRun "mpiexec"
⫽ { args =
[ "-n"
, if isTest then "2" else "24"
, "amg"
......@@ -242,38 +230,32 @@
, if isTest then "1" else "12"
, "1"
]
⫽ argIfTest isTest
⫽ { vars =
[ { varname = "OMP_NUM_THREADS", value = "1" } ]
, cmdavoids =
[ "Error", "error", "ERROR" ]
, powercap =
powercap
}
)
)
isTest
, cmdavoids =
[ "Error", "error", "ERROR" ]
}
)
( λ(x : types.Cmd)
→ x ⫽ { vars = [ { varname = "OMP_NUM_THREADS", value = "1" } ] }
)
, stream =
λ(isTest : Bool)
→ λ(powercap : Powercap)
→ lib.extendApp
( lib.progressAppTest
( lib.emptyProgressAppTestArg "stream_c" ([] : List Text)
⫽ argIfTest isTest
⫽ { vars =
[ { varname =
"OMP_NUM_THREADS"
, value =
if isTest then "2" else "24"
}
, { varname = "OMP_PLACES", value = "cores" }
]
, cmdavoids =
[ "Error", "error", "ERROR" ]
, powercap =
powercap
}
)
)
isTest
withDaemonExtraProgresstest
( basicRun "stream_c"
⫽ { args =
[] : List Text
, cmdavoids =
[ "Error", "error", "ERROR" ]
}
)
( λ(x : types.Cmd)
→ x
⫽ { vars =
[ { varname =
"OMP_NUM_THREADS"
, value =
if isTest then "2" else "24"
}
, { varname = "OMP_PLACES", value = "cores" }
]
}
)
}
let values = ../dhrunlib/package.dh
let types = ../dhrunlib/types.dh
in λ ( args
: { name :
Text
, outprefix :
Text
, outwants :
List Text
, outavoids :
List Text
}
)
→ { name =
args.name
, args =
[] : List Text
, out =
{ filename =
"${args.outprefix}.out"
, filecheck =
{ wants = args.outwants, avoids = args.outavoids }
}
, err =
{ filename =
"${args.outprefix}.err"
, filecheck =
values.emptyCheck ⫽ { avoids = args.outavoids }
}
, postchecks =
[] : List types.FileCheck
, vars =
values.emptyVars
, passvars =
[ "PATH" ]
, timeout =
[] : Optional Natural
, otherwd =
[] : Optional Text
, exitcode =
[] : Optional Integer
}
let types = ../dhrunlib/types.dh
let basicCommand = ./basicCommand.dh
let list =
basicCommand
{ name =
"nrm"
, outprefix =
"nrmlist"
, outavoids =
[] : List Text
, outwants =
[] : List Text
}
⫽ { args = [ "list" ] }
: types.Cmd
in { default = list : types.Cmd }
let values = ../dhrunlib/package.dh
let types = ../dhrunlib/types.dh
let argotypes = ./types.dh
let basicCommand = ./basicCommand.dh
let defaultMkListen =
λ(containerName : Text)
→ { attr =
"performance"
, doFilter =
False
, filter =
","
, containerName =
containerName
}
: argotypes.MkListen
let listen =
λ(a : argotypes.MkListen)
→ basicCommand
{ name =
"nrm"
, outprefix =
"nrmlisten"
, outavoids =
[] : List Text
, outwants =
if a.doFilter then [ a.filter ] else [] : List Text
}
⫽ { args = [ "listen", "-u", a.containerName, "--filter", a.filter ] }
: types.Cmd
let default =
λ(containerName : Text)
→ listen (defaultMkListen containerName) : types.Cmd
in { defarg =
defaultMkListen : Text → argotypes.MkListen
, mk =
listen : argotypes.MkListen → types.Cmd
, default =
default : Text → types.Cmd
}
let values = ../dhrunlib/package.dh
let types = ../dhrunlib/types.dh
let argotypes = ./types.dh
let basicCommand = ./basicCommand.dh
let addVerbose = λ(v : Bool) → if v then [ "--verbose" ] else [] : List Text
let addSingularity =
λ(v : Bool)
→ if v then [ "--container-runtime", "singularity" ] else [] : List Text
let addPowercap =
λ(p : argotypes.Powercap)
→ merge
{ Cap =
λ(x : Text) → [ "--powercap", x ]
, NoCap =
λ(x : {}) → [] : List Text
}
p
let defaultMkDaemon =
{ verbose =
True
, powercap =
< NoCap = {=} | Cap : Text >
, singularity =
False
, daemonavoids =