Skip to content

GitLab

Commit 0364a3ff authored by Valentin Reis's avatar Valentin Reis
Browse files

Merge branch 'power-app-expe'

parents b6bb2595 c13d2217
Pipeline #5551 passed with stages
in 1 minute and 6 seconds
Hide whitespace changes
Inline Side-by-side
Showing with 269 additions and 383 deletions
deployments/argo.nixops
View file @ 0364a3ff
No preview for this file type
deployments/chameleon.nix deleted 100644 → 0
View file @ b6bb2595
{
config,
lib ? pkgs.lib,
pkgs ? import ../pin.nix {jsonpath=../nixpkgs-18.03.json;},
...
}:
let
keys = [ (pkgs.lib.readFile keys/id_rsa_vrg.pub) ];
argopkgs = import ../pkgs {};
hydraSrc = builtins.fetchTarball https://github.com/nixos/hydra/archive/master.tar.gz;
argomodules = import ../modules/module-list.nix;
in
{
imports = [ <nixpkgs/nixos/maintainers/scripts/openstack/nova-image.nix> ];
#time.timeZone = "America/Chicago";
#require=argomodules;
#environment.argo.known-hosts.enable=true;
#environment.variables.TERM = "xterm";
#i18n.defaultLocale = "en_US.UTF-8";
#nix.trustedUsers=["root" "fre" ];
#environment.systemPackages = [ pkgs.git ];
#services.openssh.enable = true;
#users.extraUsers.root.openssh.authorizedKeys.keys = [ (pkgs.lib.readFile ./keys/id_buildfarm.pub)];
#users.extraUsers.fre.openssh.authorizedKeys.keys = [ (pkgs.lib.readFile ./keys/id_buildfarm.pub)];
}
deployments/ci.nix
View file @ 0364a3ff
......@@ -14,41 +14,23 @@ let
imports = [ ./gitlab-runner.nix];
time.timeZone = "America/Chicago";
deployment.keys."id_buildfarm" = {
destDir = "/run";
keyFile = ./id_buildfarm.secret;
user = "fre";
group = "users";
permissions = "600";
};
deployment.keys."gitlab.cfg" = {
destDir = "/run";
keyFile = ./gitlab.cfg.secret;
user = "fre";
group = "users";
permissions = "600";
};
time.timeZone = "America/Chicago";
deployment.keys."id_buildfarm" = {
destDir = "/run";
keyFile = ./keys/id_buildfarm.secret;
user = "fre";
group = "users";
permissions = "600";
};
systemd.services.tunnel-hydra= {
path = [pkgs.autossh];
enable= true;
description = "ssh tunnel to hydra";
after = [];
wantedBy = [ "multi-user.target" ];
environment.AUTOSSH_GATETIME="0";
environment.AUTOSSH_POLL="30";
serviceConfig = {
User = "fre";
Restart = "on-success";
Type = "simple";
ExecStart = ''
${pkgs.autossh}/bin/autossh -M 0 -N -o "ServerAliveInterval 30" -o "ServerAliveCountMax 3" -T -R 2210:localhost:22 fre@argo.freux.fr -i /run/id_buildfarm
'';
};
};
deployment.keys."gitlab.cfg" = {
destDir = "/run";
keyFile = ./keys/gitlab.cfg.secret;
user = "fre";
group = "users";
permissions = "600";
};
require=argomodules;
environment.argo.known-hosts.enable=true;
......@@ -66,7 +48,7 @@ let
services.ntp.enable = false;
services.openssh.allowSFTP = false;
environment.systemPackages = [ pkgs.git ];
environment.systemPackages = [ pkgs.unar pkgs.wget pkgs.git ];
virtualisation.docker.enable = true;
......@@ -93,8 +75,9 @@ in
gitlab-runner-3 = mkChameleonRunner "129.114.110.3" "chameleon-129.114.110.3";
gitlab-runner-2 = mkChameleonRunner "129.114.111.116" "chameleon-129.114.111.116";
gitlab-runner-1 = mkChameleonRunner "129.114.33.201" "chameleon-129.114.33.201";
gitlab-runner-0 = mkChameleonRunner "129.114.111.64" "chameleon-129.114.111.64";
slave-desktop =
gitlab-runner-physical =
{ ... }:
{
deployment.targetEnv = "none";
......@@ -106,7 +89,7 @@ in
deployment.keys."id_buildfarm" = {
destDir = "/run";
keyFile = ./id_buildfarm.secret;
keyFile = ./keys/id_buildfarm.secret;
user = "fre";
group = "users";
permissions = "600";
......@@ -114,7 +97,7 @@ in
deployment.keys."gitlab.cfg" = {
destDir = "/run";
keyFile = ./gitlab.cfg.secret;
keyFile = ./keys/gitlab.cfg.secret;
user = "fre";
group = "users";
permissions = "600";
......@@ -153,7 +136,7 @@ in
services.ntp.enable = false;
services.openssh.allowSFTP = false;
environment.systemPackages = [ pkgs.git ];
environment.systemPackages = [ pkgs.git pkgs.unar pkgs.wget ];
virtualisation.docker.enable = true;
......@@ -161,27 +144,6 @@ in
services.gitlab-runner2.registrationConfigFile = "/run/gitlab.cfg";
services.gitlab-runner2.packages = [pkgs.bash pkgs.docker-machine pkgs.shadow pkgs.git];
#services.gitlab-runner = {
#enable = true;
#packages = [ pkgs.bash pkgs.docker-machine pkgs.shadow];
#configFile = ./gitlab-ci.toml;
#configOptions = {
#concurrent = 2;
#runners = [ { builds_dir = "";
#docker = { cache_dir = "";
#disable_cache = true;
#host = "";
#image = "nixos/nix:2.1.3";
#privileged = true;
#};
#executor = "docker";
#name = "docker-nix-2.1.3";
#token = pkgs.lib.removeSuffix "\n" (builtins.readFile ./gitlab_token.secret);
#url = "https://xgitlab.cels.anl.gov/";
#}];
#};
#};
nix.gc = {
automatic = true;
dates = "05:15";
......@@ -191,220 +153,4 @@ in
users.extraUsers.root.openssh.authorizedKeys.keys = [ (pkgs.lib.readFile ./keys/id_buildfarm.pub)];
users.extraUsers.fre.openssh.authorizedKeys.keys = [ (pkgs.lib.readFile ./keys/id_buildfarm.pub)];
};
}
#hydra-tacc =
#{ config, ... }:
#{
#deployment.targetEnv = "none";
#deployment.targetHost = "argo.freux.fr";
#deployment.keys."id_buildfarm" = {
#destDir = "/run";
#keyFile = ./id_buildfarm.secret;
#user = "hydra-queue-runner";
#group = "hydra";
#permissions = "600";
#};
#time.timeZone = "America/Chicago";
#require=argomodules;
#environment.argo.known-hosts.enable=true;
#environment.argo.provider-tacc.enable=true;
#environment.argo.root-access.enable=true;
#environment.argo.ssh-config.enable=true;
#environment.variables.TERM = "xterm";
#imports = [ "${hydraSrc}/hydra-module.nix" ];
#i18n.defaultLocale = "en_US.UTF-8";
#services.ntp.enable = false;
#services.openssh.allowSFTP = false;
#assertions = pkgs.lib.singleton {
#assertion = pkgs.system == "x86_64-linux";
#message = "unsupported system ${pkgs.system}";
#};
#nix = {
#sshServe= { inherit keys; enable=true;};
#package = pkgs.nixUnstable; trustedUsers = [ "hydra" ]; binaryCaches = [ "https://cache.nixos.org" ];
#useChroot = true;
#nrBuildUsers = 30;
#distributedBuilds = true;
#buildMachines = [
#{
#hostName = "localhost";
#maxJobs = 40;
#speedFactor = 1;
##sshKey = "/run/id_buildfarm";
#sshUser = "fre";
#systems = ["builtin" "x86_64-linux" "i686-linux"];
#supportedFeatures = [ "nixos-test" "benchmark" ];
#}
##{
##hostName = "slave-desktop-tunnel";
##maxJobs = 40;
##speedFactor = 1;
##sshKey = "/run/id_buildfarm";
##sshUser = "fre";
##systems = ["builtin" "x86_64-linux" "i686-linux"];
##supportedFeatures = [ "nixos-test" "benchmark" "icc" ];
##}
#];
#extraOptions = "auto-optimise-store = true";
#};
##programs.ssh.extraConfig = ''
##Host slave-desktop-tunnel
##HostName localhost
##Port 2210
##User frex
##IdentityFile /run/id_buildfarm
##'';
#networking = {
#firewall = {
#allowedTCPPorts=[ 2210 80 443 8081];
#allowedUDPPorts=[ 2210 80 443 8081];
#};
#};
#services.nginx = {
#enable = true;
#user = "hydra-queue-runner";
#group= "hydra";
#virtualHosts = {
#"argo.freux.fr" = {
#basicAuth = { argo = "${builtins.readFile ./auth_argo.secret}"; };
#enableACME = true;
#forceSSL = true;
#locations."/store".root="/nix";
#locations."/store".extraConfig="autoindex on;";
#locations."/cache".root="/var/lib/hydra";
#locations."/cache".extraConfig="autoindex on;";
#locations."/"= {
#proxyPass="http://127.0.0.1:6080/";
#extraConfig = ''
#proxy_redirect http://127.0.0.1:6080 https://argo.freux.fr;
#proxy_set_header Host $host;
#proxy_set_header X-Real-IP $remote_addr;
#proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
#proxy_set_header X-Forwarded-Proto $scheme;
#proxy_set_header X-Request-Base /;
#'';
#};
#locations."/hydra"= {
#proxyPass="http://127.0.0.1:8080/";
#extraConfig = ''
#proxy_redirect http://127.0.0.1:8080 https://argo.freux.fr/hydra;
#proxy_set_header Host $host;
#proxy_set_header X-Real-IP $remote_addr;
#proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
#proxy_set_header X-Forwarded-Proto $scheme;
#proxy_set_header X-Request-Base /hydra;
#'';
#};
#};
#};
#};
#services.hydra = {
#useSubstitutes = true;
#enable = true;
#hydraURL = "https://argo.freux.fr/hydra";
#listenHost = "127.0.0.1";
#notificationSender = "hydra@example.org";
#port = 8080;
#extraConfig = ''
#store_uri = file:///var/lib/hydra/cache?secret-key=/etc/nix/argo.freux.fr/secret
#using_frontend_proxy 1
#base_uri argo.freux.fr/hydra
#binary_cache_public_uri argo.freux.fr/cache
#max_output_size = 4294967296
#secret-key=/etc/nix/argo.freux.fr/secret
#'';
#buildMachinesFiles = [ "/etc/nix/machines" ];
#};
#environment.systemPackages = [ pkgs.nix-serve ];
#services.postgresql = {
#package = pkgs.postgresql94;
#dataDir = "/var/db/postgresql-${config.services.postgresql.package.psqlSchema}";
#};
#systemd.services.hydra-manual-setup = let
#hydraEnv =
#{ HYDRA_DBI = config.services.hydra.dbi;
#HYDRA_CONFIG = "/var/lib/hydra/hydra.conf";
#HYDRA_DATA = "/var/lib/hydra";
#};
#in {
#description = "Create Admin User for Hydra";
#serviceConfig.Type = "oneshot";
#serviceConfig.RemainAfterExit = true;
#wantedBy = [ "multi-user.target" ];
#requires = [ "hydra-init.service" ];
#after = [ "hydra-init.service" ];
#environment = { NIX_REMOTE = "daemon";
#SSL_CERT_FILE = "/etc/ssl/certs/ca-certificates.crt"; # Remove in 16.03
#PGPASSFILE = "/var/lib/hydra/pgpass";
#NIX_REMOTE_SYSTEMS = pkgs.lib.concatStringsSep ":" config.services.hydra.buildMachinesFiles;
#} // pkgs.lib.optionalAttrs (config.services.hydra.smtpHost != null) {
#EMAIL_SENDER_TRANSPORT = "SMTP";
#EMAIL_SENDER_TRANSPORT_host = config.services.hydrasmtpHost;
#} // hydraEnv // config.services.hydra.extraEnv;
#script = ''
#if [ ! -e ~hydra/.setup-is-complete ]; then
## create admin user
#/run/current-system/sw/bin/hydra-create-user fre --full-name 'Valentin Reis' --email-address 'fre@freux.fr' --password foobar --role admin
#/run/current-system/sw/bin/hydra-create-user swann --full-name 'Swann Perarnau' --email-address 'swann@anl.gov' --password swannswann --role admin
## create signing keys
#/run/current-system/sw/bin/install -d -m 551 /etc/nix/argo.freux.fr
#/run/current-system/sw/bin/nix-store --generate-binary-cache-key argo.freux.fr /etc/nix/argo.freux.fr/secret /etc/nix/argo.freux.fr/public
#/run/current-system/sw/bin/chown -R hydra:hydra /etc/nix/argo.freux.fr
#/run/current-system/sw/bin/chmod 440 /etc/nix/argo.freux.fr/secret
#/run/current-system/sw/bin/chmod 444 /etc/nix/argo.freux.fr/public
##store
#/run/current-system/sw/bin/install -d -m 776 /var/lib/hydra/cache
#/run/current-system/sw/bin/chown -R hydra-queue-runner:hydra /var/lib/hydra/cache
## done
#touch ~hydra/.setup-is-complete
#fi
#'';
#};
#services.hound={
#enable = true;
#listen = "localhost:6080";
#config = ''
#{
#"max-concurrent-indexers" : 2,
#"dbpath" : "${config.services.hound.home}/data",
#"repos" : {
#"argopkgs": { "url" : "https://xgitlab.cels.anl.gov/argo/argopkgs.git" },
#"nauts": { "url" : "https://xgitlab.cels.anl.gov/argo/nauts.git" },
#"nrm": { "url" : "https://xgitlab.cels.anl.gov/argo/nrm.git" },
#"infrastructure": { "url" : "https://xgitlab.cels.anl.gov/argo/infrastructure.git" },
#"cuttr": { "url" : "https://xgitlab.cels.anl.gov/argo/cuttr.git" },
#"aml": { "url" : "https://xgitlab.cels.anl.gov/argo/aml.git" },
#"yggdrasil-integration": { "url" : "https://xgitlab.cels.anl.gov/argo/yggdrasil-integration.git" },
#"yggdrasil": { "url" : "https://xgitlab.cels.anl.gov/argo/yggdrasil.git" },
#"libnrm": { "url" : "https://xgitlab.cels.anl.gov/argo/libnrm.git" },
#"progress-benchmarks": { "url" : "https://xgitlab.cels.anl.gov/argo/progress-benchmarks.git" },
#"umap": { "url" : "https://xgitlab.cels.anl.gov/argo/umap.git" },
#"power-bandit": { "url" : "https://xgitlab.cels.anl.gov/argo/power-bandit.git" },
#"kernel": { "url" : "https://xgitlab.cels.anl.gov/argo/kernel.git" },
#"util-linux": { "url" : "https://xgitlab.cels.anl.gov/argo/util-linux.git" },
#"libmsr": { "url" : "https://github.com/LLNL/libmsr.git" }
#}
#}
#'';
#};
#users.extraUsers.root.openssh.authorizedKeys.keys = [ (pkgs.lib.readFile ./keys/id_buildfarm.pub)];
#users.extraUsers.fre.openssh.authorizedKeys.keys = [ (pkgs.lib.readFile ./keys/id_buildfarm.pub)];
#};
deployments/ci2.nix deleted 100644 → 0
View file @ b6bb2595
{
pkgs ? import ../pin.nix {jsonpath=../nixpkgs-18.03.json;}
, ...
}:
{
time.timeZone = "America/Chicago";
environment.variables.TERM = "xterm";
i18n.defaultLocale = "en_US.UTF-8";
services.ntp.enable = false;
services.openssh.allowSFTP = false;
services.openssh.enable = true;
users.extraUsers.root.openssh.authorizedKeys.keys = [ (pkgs.lib.readFile ./keys/id_buildfarm.pub)];
users.extraUsers.fre.openssh.authorizedKeys.keys = [ (pkgs.lib.readFile ./keys/id_buildfarm.pub)];
}
deployments/gitlab-runner.nix
View file @ 0364a3ff
......@@ -87,7 +87,8 @@ in
--non-interactive=true \
--name ${cfg.name} \
--executor "shell" \
--tag-list "integration" \
--maximum-timeout 1800 \
--tag-list "integration,notknl"\
'';
ExecStart = ''${cfg.package.bin}/bin/gitlab-runner run \
--working-directory ${cfg.workDir} \
......
deployments/providers/openspace.nix deleted 100644 → 0
View file @ b6bb2595
{ config, lib, pkgs, ... }:
with lib;
{
}
gitlab-ci/applications.yml 0 → 100644
View file @ 0364a3ff
# ARGUMENTS
# ARGOPKGS : The authoritative packaging source
# EXTRA : Extra argonix arguments
.applications:
stage: applications
tags:
- integration
- notknl
artifacts:
paths:
- result/*
expire_in: 1 week
when: always
.build:
stage: build
tags:
- integration
buildAMG:
extends: .build
script:
- "argonix build amg -g --argopkgs $ARGOPKGS $EXTRA --verbose"
buildQMCPack:
extends: .build
script:
- "argonix build qmcpack -g --argopkgs $ARGOPKGS $EXTRA --verbose"
tags:
- integration
- notknl
buildOpenMC:
extends: .build
script:
- "argonix build openmc -g --argopkgs $ARGOPKGS $EXTRA --verbose"
buildSTREAM:
extends: .build
script:
- "argonix build stream -g --argopkgs $ARGOPKGS $EXTRA --verbose"
buildLAMMPS:
extends: .build
script:
- "argonix build lammps -g --argopkgs $ARGOPKGS $EXTRA --verbose"
testAMG:
extends: .applications
script:
- "argonix build testAMG -g --argopkgs $ARGOPKGS $EXTRA --verbose"
testSTREAM:
extends: .applications
script:
- "argonix build testSTREAM -g --argopkgs $ARGOPKGS $EXTRA --verbose"
testOpenMC:
extends: .applications
script:
- "argonix build testOpenMC -g --argopkgs $ARGOPKGS $EXTRA --verbose"
testLAMMPS:
extends: .applications
script:
- "argonix build testLAMMPS -g --argopkgs $ARGOPKGS $EXTRA --verbose"
testQMCPack:
extends: .applications
script:
- "argonix build testQMCPack -g --argopkgs $ARGOPKGS $EXTRA --verbose"
gitlab-ci/components.yml
View file @ 0364a3ff
......@@ -28,5 +28,3 @@ argotk:
extends: .build
script:
- "argonix build argotk --argopkgs $ARGOPKGS $EXTRA --verbose"
tags:
- knl
gitlab-ci/integration.yml
View file @ 0364a3ff
......@@ -8,6 +8,11 @@
stage: test
tags:
- integration
artifacts:
paths:
- result/*
expire_in: 1 week
when: always
testHello:
extends: .test
......
modules/ssh-config/ssh_config
View file @ 0364a3ff
Host argo-phi2
HostName localhost
Port 2210
User freux
IdentityFile /run/id_buildfarm
IdentityFile /etc/ssh/ssh_host_rsa_key
pkgs/argonix/pin.json
View file @ 0364a3ff
......@@ -2,8 +2,8 @@
"fetch": {
"args": {
"fetchSubmodules": false,
"rev": "578c83c5f257844444e5823cb8e020087ae6e8d9",
"sha256": "0b1w29jsgp9spgxj6f5nlsshxrh5l6wvi75w0mg9p4fqbzy13ab9",
"rev": "0ade5584af60fcd85746afa6397d4826aa73d12b",
"sha256": "1l5n3fy6mndp09dhh1vsc0and4iszcp05qp5r5ilnc2f89wg3486",
"url": "https://xgitlab.cels.anl.gov/argo/argonix.git"
},
"fn": "fetchgit"
......
pkgs/argotk/pin.json
View file @ 0364a3ff
......@@ -2,8 +2,8 @@
"fetch": {
"args": {
"fetchSubmodules": false,
"rev": "734524132b03385bd2e5a08fdf3291ca768146ed",
"sha256": "0qjndgp54w2wxncnl56cc6gkbkw5xr97zi4mpir7cdl4k518rm3j",
"rev": "570e7b925329387344da544dce282c3779d0af09",
"sha256": "0rjmk70jxwqjppcccn1n4lpv2dv9jmb0426z16asqj639wcykx1k",
"url": "https://xgitlab.cels.anl.gov/argo/argotk.git"
},
"fn": "fetchgit"
......
pkgs/default.nix
View file @ 0364a3ff
......@@ -4,6 +4,9 @@
, # versioned nixpkgs
pkgs ? import (hostPkgs.nix-update-source.fetch ./pkgs.json).src {}
, # versioned nixpkgs-older version for zmcat
pkgs-old ? import (hostPkgs.nix-update-source.fetch ./pkgs-old.json).src {}
, # fetcher alias for the remaining arguments
fetched ? s: (pkgs.nix-update-source.fetch s).src
......@@ -20,36 +23,52 @@
, stream-src ? fetched stream/pin.json
, lammps-src ? fetched lammps/pin.json
, #source overrides :tooling
, #source overrides: tooling
argotk-src ? builtins.fetchGit {inherit (fetched argotk/pin.json) url rev;}
, repoquality-src ? builtins.fetchGit {inherit (fetched repoquality/pin.json) url rev;}
, argonix-src ? builtins.fetchGit {inherit (fetched argonix/pin.json) url rev;}
, #used for hardware experiments.
sudoPath ? null
, #whether to rebuild apps when libnrm changes.
rebuildApps ? true
}:
let
callPackage = pkgs.lib.callPackageWith (pkgs // argopkgs);
argopkgs = rec {
# library of nix expressions
argolib = import ../lib/utils.nix;
argolib = import ../lib/utils.nix;
# Argo components (source overridabel via argument)
aml = callPackage ./aml {src = aml-src;};
nrm = callPackage ./nrm {src = nrm-src;};
containers = callPackage ./containers {src = containers-src;};
libnrm = callPackage ./libnrm {src = libnrm-src;};
libnrm-pmpi = libnrm.overrideAttrs (oldAttrs: rec {
aml = callPackage ./aml {src=aml-src;};
nrm = callPackage ./nrm {src=nrm-src;};
containers = callPackage ./containers {src=containers-src;};
libnrm = callPackage ./libnrm {src=libnrm-src;};
libnrm-pinned = callPackage ./libnrm {src=fetched libnrm/pin.json;};
libnrm-pmpi = libnrm.overrideAttrs (oldAttrs: rec {
configureFlags = "--enable-pmpi";
buildInputs = oldAttrs.buildInputs ++ [ pkgs.mpich2 ];
});
# applications (source overridable via argument)
amg = callPackage ./amg { nrmSupport=true; src = amg-src; };
qmcpack = callPackage ./qmcpack { nrmSupport=true; src = qmcpack-src; };
openmc = callPackage ./openmc { nrmSupport=true; src = openmc-src; };
stream = callPackage ./stream { nrmSupport=true; src = stream-src; };
lammps = callPackage ./lammps { nrmSupport=true; src = lammps-src; };
# applications with replaceDependency for libnrm.
repLibnrm = path: src: extra:
let
drv = callPackage path {
nrmSupport = true;
src = src;
libnrm = libnrm-pinned;
};
in if !rebuildApps then drv else pkgs.replaceDependency {
drv = drv;
oldDependency = libnrm-pinned;
newDependency = libnrm;
};
amg = repLibnrm ./amg amg-src {};
qmcpack = repLibnrm ./qmcpack qmcpack-src {};
openmc = repLibnrm ./openmc openmc-src {};
lammps = repLibnrm ./lammps lammps-src {};
stream = repLibnrm ./stream stream-src {iterationCount="20000";};