Merge branch 'nrm-gen'

dhrun/all-tests.dh

@@ -6,6 +6,8 @@ let cn = "testContainer"
 
 let basicManifest = lib.manifestDir ++ "basic.json"
 
+let singularityManifest = lib.manifestDir ++ "singularity.json"
+
 let argIfTest =
         λ(isTest : Bool)
       → { isTest =
@@ -16,7 +18,24 @@ let argIfTest =
 
 let Powercap = < Cap : Text | NoCap : {} >
 
-in  { run =
+in  { run-singularity =
+        lib.makeBase
+        (   lib.emptyMakeBaseArg "echo" singularityManifest cn
+          ⫽ { args =
+                [ message ]
+            , verbose =
+                True
+            , singularity =
+                True
+            , cmdwants =
+                [] : List Text
+            , cmdavoids =
+                [ "Traceback" ]
+            , timeout =
+                [ 600 ] : Optional Natural
+            }
+        )
+    , run =
         lib.makeBase
         (   lib.emptyMakeBaseArg "echo" basicManifest cn
           ⫽ { args =

dhrun/lib.dh

@@ -67,6 +67,8 @@ let MakeBaseArg =
           List Text
       , verbose :
           Bool
+      , singularity :
+          Bool
       , cmdwants :
           List Text
       , cmdavoids :
@@ -96,6 +98,8 @@ let emptyMakeBaseArg =
           , args =
               [] : List Text
           , verbose =
+              True
+          , singularity =
               False
           , cmdwants =
               [] : List Text
@@ -116,6 +120,10 @@ let emptyMakeBaseArg =
 
 let addVerbose = λ(v : Bool) → if v then [ "--verbose" ] else [] : List Text
 
+let addSingularity =
+        λ(v : Bool)
+      → if v then [ "--container-runtime", "singularity" ] else [] : List Text
+
 let addPowercap =
         λ(p : Powercap)
       → merge
@@ -142,6 +150,7 @@ let makeBase =
                 ⫽ { args =
                         [ "--nrm_log", "./nrm_log" ]
                       # addVerbose args.verbose
+                      # addSingularity args.singularity
                       # addPowercap args.powercap
                   , vars =
                       [ { varname =
@@ -269,6 +278,8 @@ let appTest =
                 args.args
             , verbose =
                 True
+            , singularity =
+                False 
             , cmdwants =
                 [] : List Text
             , cmdavoids =

dhrun/manifests/singularity.json

+{
+	"acKind": "ImageManifest",
+	"acVersion": "0.6.0",
+	"name": "test",
+	"app": {
+		"isolators": [
+			{
+				"name": "argo/scheduler",
+				"value": {
+					"policy": "SCHED_OTHER",
+					"priority": "0"
+				}
+			},
+			{
+				"name": "argo/container",
+				"value": {
+					"cpus": "1",
+					"mems": "1"
+				}
+			},
+			{
+				"name": "argo/perfwrapper",
+				"value": {
+					 "enabled": "0"
+				}
+			},
+			{
+				"name": "argo/power",
+				"value": {
+					"enabled": "1",
+					"profile": "1",
+					"policy": "NONE",
+					"damper": "1e9",
+					"slowdown": "1.1"
+				}
+			},
+			{
+				"name": "argo/monitoring",
+				"value": {
+					 "enabled": "1",
+					 "ratelimit": "10000000"
+				}
+			},
+			{
+				"name": "argo/hwbind",
+				"value": {
+					 "enabled": "0"
+				}
+			}
+		]
+	},
+	"image": {
+		"path": "docker://ubuntu",
+		"type": "docker"
+	}
+}

gitlab-ci/applications.yml

@@ -28,52 +28,52 @@
 buildAMG:
   extends: .build
   script:
-    - "argonix build amg -g --argopkgs $ARGOPKGS $EXTRA --verbose"
+    - "argonix build amg --argopkgs $ARGOPKGS $EXTRA --verbose"
 
 buildQMCPack:
   extends: .build
   script:
-    - "argonix build qmcpack -g --argopkgs $ARGOPKGS $EXTRA --verbose"
+    - "argonix build qmcpack --argopkgs $ARGOPKGS $EXTRA --verbose"
 
 buildOpenMC:
   extends: .build
   script:
-    - "argonix build openmc -g --argopkgs $ARGOPKGS $EXTRA --verbose"
+    - "argonix build openmc --argopkgs $ARGOPKGS $EXTRA --verbose"
 
 buildSTREAM:
   extends: .build
   script:
-    - "argonix build stream -g --argopkgs $ARGOPKGS $EXTRA --verbose"
+    - "argonix build stream --argopkgs $ARGOPKGS $EXTRA --verbose"
 
 buildLAMMPS:
   extends: .build
   script:
-    - "argonix build lammps -g --argopkgs $ARGOPKGS $EXTRA --verbose"
+    - "argonix build lammps --argopkgs $ARGOPKGS $EXTRA --verbose"
 
 testAMG:
   extends: .applications
   script:
-    - "argonix build testAMG -g --argopkgs $ARGOPKGS $EXTRA --verbose"
+    - "argonix build testAMG --argopkgs $ARGOPKGS $EXTRA --verbose"
 
 testSTREAM:
   extends: .applications
   script:
-    - "argonix build testSTREAM -g --argopkgs $ARGOPKGS $EXTRA --verbose"
+    - "argonix build testSTREAM --argopkgs $ARGOPKGS $EXTRA --verbose"
 
 testOpenMC:
   extends: .applications
   script:
-    - "argonix build testOpenMC -g --argopkgs $ARGOPKGS $EXTRA --verbose"
+    - "argonix build testOpenMC --argopkgs $ARGOPKGS $EXTRA --verbose"
 
 testQMCPack:
   extends: .applications
   script:
-    - "argonix build testQMCPack -g --argopkgs $ARGOPKGS $EXTRA --verbose"
+    - "argonix build testQMCPack --argopkgs $ARGOPKGS $EXTRA --verbose"
 
 testLAMMPS:
   extends: .applications
   script:
-    - "argonix build testLAMMPS -g --argopkgs $ARGOPKGS $EXTRA --verbose"
+    - "argonix build testLAMMPS --argopkgs $ARGOPKGS $EXTRA --verbose"
 
 #testQMCPack:
   #extends: .applications

gitlab-ci/integration.yml

@@ -18,27 +18,27 @@
 testPwd:
   extends: .test
   script:
-    - "argonix build testPwd --argopkgs $ARGOPKGS $EXTRA --verbose"
+    - "argonix test pwd --argopkgs $ARGOPKGS $EXTRA --verbose"
 
 testRun:
   extends: .test
   script:
-    - "argonix build testRun --argopkgs $ARGOPKGS $EXTRA --verbose"
+    - "argonix test run --argopkgs $ARGOPKGS $EXTRA --verbose"
 
 testHello:
   extends: .test
   script:
-    - "argonix build testHello --argopkgs $ARGOPKGS $EXTRA --verbose"
+    - "argonix test hello --argopkgs $ARGOPKGS $EXTRA --verbose"
 
 testListen:
   extends: .test
   script:
-    - "argonix build testListen --argopkgs $ARGOPKGS $EXTRA --verbose"
+    - "argonix test listen --argopkgs $ARGOPKGS $EXTRA --verbose"
 
 testPerfwrapper:
   extends: .test
   script:
-    - "argonix build testPerfwrapper --argopkgs $ARGOPKGS $EXTRA --verbose"
+    - "argonix test performance --argopkgs $ARGOPKGS $EXTRA --verbose"
   tags:
     - performance
     - integration
@@ -46,6 +46,6 @@ testPerfwrapper:
 testPower:
   extends: .test
   script:
-    - "argonix build testPower --argopkgs $ARGOPKGS $EXTRA --verbose"
+    - "argonix test power --argopkgs $ARGOPKGS $EXTRA --verbose"
   tags:
     - knl

pkgs/argonix/pin.json

@@ -2,8 +2,8 @@
   "fetch": {
     "args": {
       "fetchSubmodules": false,
-      "rev": "bc7b9efd7f55c4e8f4257ce0813d935fa729326e",
-      "sha256": "0x6xgp5gjn2kp7n1j8v9f5f04fc8wl3dv5b3x40a37x4bvph5npr",
+      "rev": "0ea6332d6101f3b72a9d5c737a268973a0bd84ef",
+      "sha256": "0f5j1kl5gi8ll3s1mszw650hhsvww5vl9n220my1jvqqi3qrx9ah",
       "url": "https://xgitlab.cels.anl.gov/argo/argonix.git"
     },
     "fn": "fetchgit"

pkgs/default.nix

@@ -50,8 +50,11 @@ let
     (old:{buildInputs=old.buildInputs ++ [ bi ];});
 
   argopkgs = rec {
+
     # library of nix expressions
     argolib       = import ../lib/utils.nix;
+
+    flatcc        = callPackage ./flatcc      {};
     # Argo components (source overridabel via argument)
     aml           = callPackage ./aml        {
       src=aml-src;
@@ -61,7 +64,11 @@ let
     };
     numabench     = callPackage ./numabench  {src=numabench-src; hwloc = hwloc2;};
     excit         = callPackage ./excit      {src=excit-src;};
-    nrm           = callPackage ./nrm        {src=nrm-src;};
+    nrm           = (callPackage ./nrm        {src=nrm-src;}).overrideAttrs (o:{
+      installPhase=o.installPhase+''
+        cp -r nrm/schemas $out/lib/python2.7/site-packages/nrm/
+      '';
+    });
     containers    = callPackage ./containers {src=containers-src;};
     libnrm        = callPackage ./libnrm     {src=libnrm-src;};
     libnrm-pinned = callPackage ./libnrm     {src=fetched libnrm/pin.json;};
@@ -70,6 +77,13 @@ let
       buildInputs = oldAttrs.buildInputs ++ [ pkgs.mpich2 ];
     });
 
+    #jsonschema3 = pkgs.pythonPackages.jsonschema.overrideAttrs(o:{
+      #src=pkgs.fetchurl{
+        #url="https://files.pythonhosted.org/packages/1f/7f/a020327823b9c405ee6f85ab3053ff171e10801b19cfe55c78bb0b3810e7/jsonschema-3.0.1.tar.gz";
+        #sha256 = "03g20i1xfg4qdlk4475pl4pp7y0h37g1fbgs5qhy678q9xb822hc";
+      #};
+    #});
+
     nrm-pytest    = nrm.overrideAttrs (old:{checkPhase="py.test --deselect=test/test_messaging.py";});
 
     aml-check = aml.overrideAttrs (old:{
@@ -165,6 +179,7 @@ let
         monad-mock     = doJailbreak super.monad-mock;
         generic-pretty = doJailbreak super.generic-pretty;
         jsonschema-gen = doJailbreak super.jsonschema-gen;
+        aeson-t = doJailbreak super.aeson-t;
 
         #minor cabal and version fix
         panhandle    = (self.callCabal2nix "panhandle" (builtins.fetchGit {
@@ -268,6 +283,7 @@ let
         # argo components
         containers
         nrm
+        libnrm
       ];
 
       #required to be able to build the drv.
@@ -455,12 +471,9 @@ let
     doDhrun dhrunscript;
 
     testGeneric     = doDhrunKey genericTestName;
-    testHello       = doDhrunKey "hello";
-    testRun         = doDhrunKey "run";
-    testPwd         = doDhrunKey "pwd";
-    testListen      = doDhrunKey "listen";
-    testPerfwrapper = doDhrunKey "performance";
-    testPower       = doDhrunKey "power";
+    testSingularity = (doDhrunKey "run-singularity").overrideAttrs (o:{
+      buildInputs=o.buildInputs++[ pkgs.singularity ];
+    });
 
     testSTREAM      = addBI (doDhrunApp "stream") stream;
     testAMG         = addBI (doDhrunApp "amg") amg;
@@ -500,6 +513,5 @@ let
     bld = bleeding;
   };
 
-
 # re-exporting a full nixpkgs-like package set
 in pkgs // argopkgs

pkgs/flatcc/default.nix

+{ stdenv
+, fetchFromGitHub
+, cmake
+}:
+
+stdenv.mkDerivation rec {
+  name = "flatcc";
+  version = "0.5.3";
+
+  src = fetchFromGitHub {
+    owner = "dvidelabs";
+    repo = "flatcc";
+    rev = "v${version}";
+    sha256 = "06wnwvnkhw1rk0y3nncjmcyjy3bgpw8i9xqd5gpbhbrm38718cjk";
+  };
+
+  nativeBuildInputs = [ cmake ];
+
+  cmakeFlags = [
+    "-DFLATCC_INSTALL=on"
+    "-DCMAKE_BUILD_TYPE=Release"
+  ];
+
+  meta = {
+    description = "FlatBuffers Compiler and Library in C for C ";
+    homepage = https://github.com/dvidelabs/flatcc;
+    license = [ stdenv.lib.licenses.asl20 ];
+  };
+}

pkgs/libnrm/pin.json

@@ -2,13 +2,13 @@
   "fetch": {
     "args": {
       "fetchSubmodules": false,
-      "rev": "9b52e9fb9209a0a19ba21a830e6a2c0cbddf3052",
-      "sha256": "1canjww3igzd8d1wxpjfvzjalc0iacqljxl0nbpc5j0s6clj0q45",
+      "rev": "70720b12e5e3d25436738ebc05963c847b671134",
+      "sha256": "0xq8s47saza5ml9pmkf5sb6ia0ilbnly2r5v6m770hzw50b4q7cf",
       "url": "https://xgitlab.cels.anl.gov/argo/libnrm.git"
     },
     "fn": "fetchgit"
   },
-  "rev": "refs/heads/master",
+  "rev": "refs/heads/nrm-gen",
   "type": "fetchgit",
   "url": "https://xgitlab.cels.anl.gov/argo/libnrm.git"
 }
\ No newline at end of file

pkgs/nrm/default.nix

@@ -11,6 +11,7 @@ pythonPackages.buildPythonPackage {
     linuxPackages.perf
     pythonPackages.docopt
     pythonPackages.scipy
+    pythonPackages.warlock
     containers
   ];
   checkInputs = [ pythonPackages.pytest ];

pkgs/nrm/pin.json

@@ -2,13 +2,13 @@
   "fetch": {
     "args": {
       "fetchSubmodules": false,
-      "rev": "753c0048f999255ca42dce10e18e71c109af7ed7",
-      "sha256": "01wngv506nx43h6jjrbzg7axwpnfn7920nn9dajm7das0mbfv9bx",
+      "rev": "cd1d86b30e13aabeacc782075f0dafe957fc7aed",
+      "sha256": "1vhx340mkvag11qfiwcm6dkqs4qf5laqfi1zmh0pr1f6pyqn1l62",
       "url": "https://xgitlab.cels.anl.gov/argo/nrm.git"
     },
     "fn": "fetchgit"
   },
-  "rev": "refs/heads/master",
+  "rev": "refs/heads/nrm-gen",
   "type": "fetchgit",
   "url": "https://xgitlab.cels.anl.gov/argo/nrm.git"
 }
\ No newline at end of file