Commit 3e0207f6 authored by Valentin Reis's avatar Valentin Reis

Merge branch 'refactor' into 'master'

Refactoring in preparation for AML CI improvements.

See merge request !8
parents ca8111e7 4578d7bd
Pipeline #10161 passed with stages
in 1 minute and 40 seconds
......@@ -7,14 +7,13 @@ stages:
- test
- applications
- quality
- style
include:
- /gitlab-ci/argonix.yml
- /gitlab-ci/ci.yml
- /gitlab-ci/components.yml
- /gitlab-ci/dhrun.yml
- /gitlab-ci/argonix-build.yml
- /gitlab-ci/excit.yml
- /gitlab-ci/integration.yml
- /gitlab-ci/repoquality.yml
- /gitlab-ci/applications.yml
#- /gitlab-ci/numabench.yml
- /gitlab-ci/dhrun.yml
No preview for this file type
......@@ -35,7 +35,7 @@ let
require = argomodules;
environment.argo.known-hosts.enable = true;
environment.argo.provider-tacc.enable = true;
environment.argo.provider-openspace.enable = true;
environment.argo.root-access.enable = true;
environment.argo.ssh-config.enable = true;
......@@ -51,6 +51,17 @@ let
nix.nrBuildUsers = 30;
nix.trustedUsers = [ "root" "fre" ];
#nix.binaryCaches = [
#"https://cache.nixos.org"
##"http://129.114.24.212/serve"
#];
#nix.binaryCachePublicKeys = [
#"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
#"example-nix-cache-1:HSwzbJmGDidTrax3Lvx1vMSvto04VN2O5cjfXAG9uz0="
#];
services.ntp.enable = false;
services.openssh.allowSFTP = false;
......
#!/usr/bin/env bash
nixops deploy -s argo.nixops --include gitlab-runner-1 gitlab-runner-2 gitlab-runner-3 gitlab-runner-4 gitlab-runner-5
nixops deploy -s argo.nixops --include gitlab-runner-1 gitlab-runner-2 gitlab-runner-3 gitlab-runner-4
#!/usr/bin/env bash
ssh-keygen -R 129.114.111.64
ssh-keygen -R 129.114.111.114
ssh-keygen -R 129.114.110.214
ssh-keygen -R 129.114.110.3
ssh-keygen -R 129.114.33.201
ssh-keygen -R 129.114.111.83
ssh-keygen -R 129.114.111.116
#!/usr/bin/env bash
echo "booted runner."
ssh root@140.221.10.9 reboot
echo "booted runner."
ssh root@129.114.111.114 reboot
echo "booted runner."
ssh root@129.114.110.3 reboot
echo "booted runner."
ssh root@129.114.111.116 reboot
echo "booted runner."
ssh root@129.114.33.201 reboot
echo "booted runner."
#ssh gitlab-runner@argo-phi1 sudo systemctl restart gitlab-runner.service
---
#ARGUMENTS:
#The authoritative packaging source
#ARGOPKGS
#Extra argonix arguments
#EXTRA
.build:
stage: build
tags:
- integration
artifacts:
paths:
- result/*
- nixtmpdir/
expire_in: 1 week
when: always
.test:
stage: test
tags:
- integration
artifacts:
paths:
- result/*
- nixtmpdir/
expire_in: 1 week
when: always
aml:
extends: .build
script:
- "argonix build aml --argopkgs $ARGOPKGS $EXTRA --verbose"
aml-check:
extends: .test
script:
- "argonix build aml-check --argopkgs $ARGOPKGS $EXTRA --verbose"
aml-check-knl:
extends: .test
tags:
- knl
script:
- "argonix build aml-check-knl --argopkgs $ARGOPKGS $EXTRA --verbose"
readthedocs:
stage: docs
when: on_success
only:
- master
- /v[0-9]+\.[0-9]+\.x/
tags:
- integration
script:
- nix run nixpkgs.curl -c curl -X POST -d "branch=$CI_COMMIT_REF_NAME" -d "token=$READTHEDOCS_TOKEN" https://readthedocs.org/api/v2/webhook/argo-aml/83161/
dist:
stage: release
when: on_success
only:
- tags
tags:
- integration
script:
- nix-build "$ARGOPKGS" -A aml-dist --arg aml-src ./.
- nix-shell "$ARGOPKGS" -A aml-dist --arg aml-src ./. --run "./release.sh CREATE $CI_JOB_ID $CI_PROJECT_ID $RELEASE_TOKEN $CI_COMMIT_REF_NAME"
artifacts:
when: on_success
paths:
- result/*.tar.gz
- CHECKSUM
expire_in: 1000y
#Importing this file adds repository quality check.
#
# ARGUMENTS
# ARGOPKGS : The authoritative packaging source
# EXTRA_CHECKPATCH : extra checkpatch cli args
#
checkpatch:
stage: quality
script:
- CACHE=$(mktemp -d --suffix=nixcache /tmp/deletable-nix-cache-XXXX)
- XDG_CACHE_HOME=$CACHE nix run -f "$ARGOPKGS" checkpatch --command checkpatch.pl
- rm -rf $CACHE
- nix run -f "$ARGOPKGS" checkpatch --command checkpatch.pl $EXTRA_CHECKPATCH
tags:
- integration
---
ci-environments:
stage: build
script:
- nix-build -A ci
......@@ -3,17 +3,6 @@
#ARGOPKGS
#Extra argonix arguments
#EXTRA
#
.standalonetest:
stage: test
tags:
- integration
artifacts:
paths:
- result/*
- nixtmpdir/
expire_in: 1 week
when: always
.build:
stage: build
......@@ -31,11 +20,6 @@ nrm:
script:
- "argonix build nrm --argopkgs $ARGOPKGS $EXTRA --verbose"
#nrm-pytest:
#extends: .standalonetest
#script:
#- "argonix build nrm-pytest --argopkgs $ARGOPKGS $EXTRA --verbose"
libnrm:
extends: .build
script:
......
#Importing this file adds repository quality check.
#
# ARGUMENTS
# ARGOPKGS : The authoritative packaging source
#
repoquality:
stage: quality
script:
- CACHE=$(mktemp -d --suffix=nixcache /tmp/deletable-nix-cache-XXXX)
- XDG_CACHE_HOME=$CACHE nix run -f "$ARGOPKGS" repoquality --command repoquality
- rm -rf $CACHE
- nix run -f "$ARGOPKGS" repoquality --command repoquality
tags:
- integration
......@@ -2,13 +2,13 @@
"fetch": {
"args": {
"fetchSubmodules": false,
"rev": "4e8c08e5874c70914cee61a73bc771b7fdc3cc20",
"sha256": "18qpv829lqjsm07cazgzfsdccg0rhl55nj8pck25d2haywffgghg",
"rev": "bfc976af0c0404d2321cb64f0d32d1d70395d5d8",
"sha256": "168iw33gd7an2fb0nxwaq8bwf0f95hibl80jbg2icc136sk6czvs",
"url": "https://xgitlab.cels.anl.gov/argo/aml.git"
},
"fn": "fetchgit"
},
"rev": "refs/heads/replicaset",
"rev": "refs/heads/master",
"type": "fetchgit",
"url": "https://xgitlab.cels.anl.gov/argo/aml.git"
}
\ No newline at end of file
......@@ -97,57 +97,14 @@ let
libnrm-pmpi = callPackage ./libnrm/mpi.nix { src = libnrm-src; };
libnrm-pinned = callPackage ./libnrm { src = fetched libnrm/pin.json; };
#jsonschema3 = pkgs.pythonPackages.jsonschema.overrideAttrs(o:{
#src=pkgs.fetchurl{
#url="https://files.pythonhosted.org/packages/1f/7f/a020327823b9c405ee6f85ab3053ff171e10801b19cfe55c78bb0b3810e7/jsonschema-3.0.1.tar.gz";
#sha256 = "03g20i1xfg4qdlk4475pl4pp7y0h37g1fbgs5qhy678q9xb822hc";
#};
#});
#nrm-pytest = nrm.overrideAttrs (old:{buildInputs = old.buildInputs++[nrm]; doCheck=true;checkPhase="py.test --deselect=test/test_messaging.py";});
aml-check = aml.overrideAttrs (old: {
doCheck = true;
checkPhase = "make check";
});
mkl = callPackage ./mkl {
compiler_path = "/opt/intel/compilers_and_libraries/linux";
};
icc = callPackage ./icc {
compiler_path = "/opt/intel/compilers_and_libraries/linux";
};
aml-check-knl = aml-check.overrideAttrs (old: {
makeFlags = [ "-j64" "V=1" ];
configureFlags = [ "--enable-benchmarks" ];
buildInputs = old.buildInputs ++ [ pkgs.man-db icc mkl ];
CC = "${icc}/bin/icc";
preConfigure = ''
export CFLAGS="-mkl -xhost"
export CC="${icc}/bin/icc"
find . -name '*.o'
find . -name '*.o' -delete
'';
});
aml-dist = aml.overrideAttrs (old: {
nativeBuildInputs = old.nativeBuildInputs ++ [
pkgs.doxygen
pkgs.curl
pkgs.bash
pkgs.graphviz
pkgs.coreutils
(pkgs.python.withPackages
(ps: [ ps.sphinx_rtd_theme ps.sphinx ps.breathe ps.recommonmark ]))
];
buildPhase = "make dist";
installPhase = ''
mkdir -p $out
cp *.tar.gz $out/
'';
});
numabench-check = numabench.overrideAttrs (old: {
buildPhase = "make check";
installPhase = ''
......@@ -549,6 +506,39 @@ let
'';
};
#legacy shell target (used by some aml branches):
aml-dist = ci.aml.doc;
ci = let
ciShell = x:
(pkgs.mkShell x).overrideAttrs (o: {
installPhase = "touch $out";
phases = [ "installPhase" ];
});
in {
aml-format = ciShell {
name = "aml-format";
buildInputs = [
(pkgs.clang-tools.override
(o: { llvmPackages = pkgs.llvmPackages_7; }))
];
};
aml-doc = ciShell {
name = "aml-doc";
inputsFrom = [ aml ];
buildInputs = [
pkgs.doxygen
pkgs.curl
pkgs.bash
pkgs.graphviz
pkgs.coreutils
(pkgs.python.withPackages
(ps: [ ps.sphinx_rtd_theme ps.sphinx ps.breathe ps.recommonmark ]))
];
};
};
};
# re-exporting a full nixpkgs-like package set
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment