Commit 2ec229e8 authored by Valentin Reis's avatar Valentin Reis
Browse files

added data retreiver.

parent a1715bf6
Pipeline #5125 passed with stage
in 1 minute and 29 seconds
......@@ -5,11 +5,12 @@ license-file: LICENSE
author: Valentin Reis
maintainer: fre@freux.fr
build-type: Simple
extra-source-files: ChangeLog.md
cabal-version: >=1.10
executable argonix
main-is: argonix.hs
Main-Is: Argonix.hs
hs-source-dirs: src
Other-Modules: Nixexpr
build-depends:
base,
turtle,
......@@ -19,7 +20,6 @@ executable argonix
text,
neat-interpolation,
system-filepath
hs-source-dirs: src
default-language: Haskell2010
ghc-options:
-threaded
......
{-# language RecordWildCards #-}
{-# LANGUAGE QuasiQuotes #-}
{-# language ApplicativeDo #-}
{-# language OverloadedStrings #-}
module Main where
import Data.Default
import Nixexpr
import Turtle hiding ( header
, text
, arguments
......@@ -16,77 +16,18 @@ import Prelude hiding ( FilePath )
import Data.Text ( pack
, unpack
, Text
)
import NeatInterpolation
import System.Posix.Process
nixExpression :: Text
nixExpression =
[text|
let
hostPkgs = import <nixpkgs> {};
argopkgs-src = builtins.fetchGit {
url = https://xgitlab.cels.anl.gov/argo/argopkgs.git;
rev = "";
};
in
{
argopkgs ? import argopkgs-src {},
argotk-src ? argopkgs.argotk.src,
nrm-src ? argopkgs.nodelevel.nrm.src,
containers-src ? argopkgs.nodelevel.containers.src,
libnrm-src ? argopkgs.nodelevel.libnrm.src,
amg-src ? argopkgs.applications.nrm.amg.src,
lammps-src ? argopkgs.applications.nrm.lammps.src,
qmcpack-src ? argopkgs.applications.nrm.qmcpack.src,
stream-src ? argopkgs.applications.nrm.stream.src,
dgemm_randomwalk-src ? argopkgs.applications.nrm.dgemm_randomwalk.src,
}:
rec {
argotk = argopkgs.argotk.overrideAttrs (old: {src = argotk-src;} );
nrm = argopkgs.nodelevel.nrm.overrideAttrs (old: {src = nrm-src;} );
libnrm = argopkgs.nodelevel.libnrm.overrideAttrs (old: {src = libnrm-src;} );
containers = argopkgs.nodelevel.containers.overrideAttrs (old: {src = containers-src;} );
amg = (argopkgs.applications.nrm.amg.overrideAttrs (old: {src = amg-src;} )).override {libnrm = libnrm;};
qmcpack = (argopkgs.applications.nrm.qmcpack.overrideAttrs (old: {src = qmcpack-src;} )).override {libnrm = libnrm;};
stream-test = (argopkgs.applications.nrm.stream.overrideAttrs (old: {src = stream-src;} )).override {libnrm = libnrm; iterationCount = "20";};
stream = (argopkgs.applications.nrm.stream.overrideAttrs (old: {src = stream-src;} )).override {libnrm = libnrm; iterationCount = "20000";};
lammps = (argopkgs.applications.nrm.lammps.overrideAttrs (old: {src = lammps-src;} )).override {libnrm = libnrm;};
test = argopkgs.stdenv.mkDerivation rec {
name = "testEnv";
installPhase = "mkdir $$out";
unpackPhase = "true";
buildInputs = [
argopkgs.coreutils
argopkgs.utillinux
argopkgs.mpich2
argotk
containers
stream-test
nrm
];
shellHook = ''
source $${argotk}/share/argotk.sh
echo "Installed bash completion."
'';
MANIFESTS = "$${argopkgs.argonix}/share/manifests";
};
expe = test.overrideAttrs (old:{
buildInputs = old.buildInputs ++ [amg lammps qmcpack stream];
}
);
}
|]
data StackArgs = StackArgs
{ verbosity :: Verbosity
, enableApps :: Bool
, remoteBuild :: Bool
, targetMachine :: Maybe String
, run :: Maybe String
, remoteBuild :: Bool
, targetMachine :: Maybe Text
, retreive :: Maybe Text
, retreiveAs :: Maybe Text
, run :: Maybe Text
, overrides :: [(String, FilePath)]
} deriving (Show)
data Verbosity = Verbose | Normal deriving (Show)
......@@ -98,6 +39,8 @@ instance Default StackArgs where
remoteBuild = False,
run = Nothing,
targetMachine = Nothing,
retreive = Nothing,
retreiveAs = Nothing,
overrides = []
}
......@@ -110,11 +53,23 @@ executorParser = do
enableApps <- flag
False
True
(long "provision_apps" <> short 'a' <> help "Build and provision applications.")
(long "provision_apps" <> short 'a' <> help
"Build and provision applications."
)
remoteBuild <- flag
False
True
(long "remote_build" <> short 'r' <> help "Build on the remote rather than locally.")
(long "remote_build" <> short 'r' <> help
"Build on the remote rather than locally."
)
retreive <- optional $ strOption
(long "retreive" <> metavar "RELATIVE_PATH" <> help
"File/Folder to retreive from the remote machine."
)
retreiveAs <- optional $ strOption
(long "retreiveAs" <> metavar "LOCAL_PATH" <> help
"File/Folder to save retreived data as (locally)."
)
targetMachine <- optional $ strOption
(long "target_machine" <> metavar "USER@HOST" <> help
"Target machine. defaults to localhost via forking."
......@@ -156,54 +111,85 @@ main = sh $ do
a@StackArgs {..} <- liftIO $ execParser opts
let arguments = argumentsBuilder a
case targetMachine of
Nothing -> liftIO $ executeFile "nix-shell" True (["-E", unpack nixExpression] ++ arguments) Nothing
Nothing -> liftIO $ executeFile
"nix-shell"
True
( ["-E", unpack nixExpression]
++ arguments
++ (case run of
Just cmd -> ["--run", unpack ("\"" <> cmd <> "\"")]
Nothing -> []
)
)
Nothing
Just host -> do
export "NIX_SSHOPTS" "source .profile; source .bash_profile;"
printf s "argonix: creating a derivation in the local store:\n"
drv <- single $ inproc
"nix-instantiate"
([ "--quiet", "-E", nixExpression] ++ map pack arguments)
(["--quiet", "-E", nixExpression] ++ map pack arguments)
empty
printf (s % "\n") $ lineToText drv
if remoteBuild then do
printf s "argonix: copying the derivation's closure (without build output).\n"
proc "nix-copy-closure"
["--to", pack host, lineToText drv]
empty
else do
printf s "argonix: building the derivation's output:\n"
proc "nix-store" ["--realize", lineToText drv, "--quiet"] empty
printf s "argonix: copying the derivation's closure (with build output).\n"
proc "nix-copy-closure"
["--include-outputs", "--to", pack host, lineToText drv]
empty
liftIO $ executeFile
"ssh"
True
["-t", host, unpack ("source .profile; source .bash_profile; nix-shell "<> lineToText drv)]
Nothing
if remoteBuild
then do
printf
s
"argonix: copying the derivation's closure (without build output).\n"
proc "nix-copy-closure" ["--to", host, lineToText drv] empty
else do
printf s "argonix: building the derivation's output:\n"
proc "nix-store" ["--realize", lineToText drv, "--quiet"] empty
printf
s
"argonix: copying the derivation's closure (with build output).\n"
proc "nix-copy-closure"
["--include-outputs", "--to", host, lineToText drv]
empty
case run of
Just cmd -> do
_ <- proc
"ssh"
[ "-t"
, host
, "source .profile; source .bash_profile; nix-shell "
<> lineToText drv
<> " --run "
<> "\"" <> cmd <> "\""
]
empty
mapM_
(\r -> proc "scp"
["-r", host <> ":" <> r, fromMaybe "." retreiveAs]
empty
)
retreive
Nothing -> liftIO $ executeFile
"ssh"
True
[ "-t"
, unpack host
, unpack
( "source .profile; source .bash_profile; nix-shell "
<> lineToText drv
)
]
Nothing
where
argumentsBuilder :: StackArgs -> [String]
argumentsBuilder StackArgs {..} =
["-A", if enableApps then "expe" else "test"]
++ concat
[ ["--arg", longform <> "-src", encodeString p]
| (longform, p) <- overrides
]
++ (case run of
Just cmd -> ["--run", cmd]
Nothing -> []
)
["-A", if enableApps then "expe" else "test"] ++ concat
[ ["--arg", longform <> "-src", encodeString p]
| (longform, p) <- overrides
]
opts = info
(executorParser <**> helper)
( fullDesc
<> progDesc
"Argo environment provisioning/deployment/execution."
<> progDesc "Argo environment provisioning/deployment/execution."
<> header "argonix"
)
{-# LANGUAGE QuasiQuotes #-}
module Nixexpr where
import Data.Text ( Text)
import NeatInterpolation
nixExpression :: Text
nixExpression =
[text|
let
hostPkgs = import <nixpkgs> {};
argopkgs-src = builtins.fetchGit {
url = https://xgitlab.cels.anl.gov/argo/argopkgs.git;
rev = "";
};
in
{
argopkgs ? import argopkgs-src {},
argotk-src ? argopkgs.argotk.src,
nrm-src ? argopkgs.nodelevel.nrm.src,
containers-src ? argopkgs.nodelevel.containers.src,
libnrm-src ? argopkgs.nodelevel.libnrm.src,
amg-src ? argopkgs.applications.nrm.amg.src,
lammps-src ? argopkgs.applications.nrm.lammps.src,
qmcpack-src ? argopkgs.applications.nrm.qmcpack.src,
stream-src ? argopkgs.applications.nrm.stream.src,
dgemm_randomwalk-src ? argopkgs.applications.nrm.dgemm_randomwalk.src,
}:
rec {
argotk = argopkgs.argotk.overrideAttrs (old: {src = argotk-src;} );
nrm = argopkgs.nodelevel.nrm.overrideAttrs (old: {src = nrm-src;} );
libnrm = argopkgs.nodelevel.libnrm.overrideAttrs (old: {src = libnrm-src;} );
containers = argopkgs.nodelevel.containers.overrideAttrs (old: {src = containers-src;} );
amg = (argopkgs.applications.nrm.amg.overrideAttrs (old: {src = amg-src;} )).override {libnrm = libnrm;};
qmcpack = (argopkgs.applications.nrm.qmcpack.overrideAttrs (old: {src = qmcpack-src;} )).override {libnrm = libnrm;};
stream-test = (argopkgs.applications.nrm.stream.overrideAttrs (old: {src = stream-src;} )).override {libnrm = libnrm; iterationCount = "20";};
stream = (argopkgs.applications.nrm.stream.overrideAttrs (old: {src = stream-src;} )).override {libnrm = libnrm; iterationCount = "20000";};
lammps = (argopkgs.applications.nrm.lammps.overrideAttrs (old: {src = lammps-src;} )).override {libnrm = libnrm;};
test = argopkgs.stdenv.mkDerivation rec {
name = "testEnv";
installPhase = "mkdir $$out";
unpackPhase = "true";
buildInputs = [
argopkgs.coreutils
argopkgs.utillinux
argopkgs.mpich2
argotk
containers
stream-test
nrm
];
shellHook = ''
source $${argotk}/share/argotk.sh
echo "Installed bash completion."
'';
MANIFESTS = "$${argopkgs.argonix}/share/manifests";
};
expe = test.overrideAttrs (old:{
buildInputs = old.buildInputs ++ [amg lammps qmcpack stream];
}
);
}
|]
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment