ci.nix 7.17 KB
Newer Older
Valentin Reis's avatar
Valentin Reis committed
1
{ pkgs ? import ../pin.nix { jsonpath = ../nixpkgs-unstable.json; } }:
Valentin Reis's avatar
Valentin Reis committed
2
let
Valentin Reis's avatar
Valentin Reis committed
3 4 5 6 7 8 9
  keys = [
    (pkgs.lib.readFile keys/id_rsa_swann.pub)
    (pkgs.lib.readFile keys/id_rsa_vrg.pub)
  ];
  argopkgs = import ../pkgs { };
  hydraSrc = builtins.fetchTarball
    "https://github.com/nixos/hydra/archive/master.tar.gz";
Valentin Reis's avatar
Valentin Reis committed
10
  argomodules = import ../modules/module-list.nix;
Valentin Reis's avatar
Valentin Reis committed
11 12 13 14
  mkChameleonRunner = ip:
    { ... }: {
      deployment.targetEnv = "none";
      deployment.targetHost = ip;
15

Valentin Reis's avatar
Valentin Reis committed
16
      imports = [ ./gitlab-runner.nix ];
17

Valentin Reis's avatar
Valentin Reis committed
18
      time.timeZone = "America/Chicago";
Valentin Reis's avatar
Valentin Reis committed
19

Valentin Reis's avatar
Valentin Reis committed
20 21 22 23 24
      deployment.keys."id_buildfarm" = {
        destDir = "/run";
        keyFile = ./keys/id_buildfarm.secret;
        user = "fre";
        group = "users";
Valentin Reis's avatar
Valentin Reis committed
25
        permissions = "666";
Valentin Reis's avatar
Valentin Reis committed
26
      };
27

Valentin Reis's avatar
Valentin Reis committed
28 29 30 31 32 33 34
      deployment.keys."gitlab.cfg" = {
        destDir = "/run";
        keyFile = ./keys/gitlab.cfg.secret;
        user = "fre";
        group = "users";
        permissions = "600";
      };
35

Valentin Reis's avatar
Valentin Reis committed
36 37 38 39 40
      require = argomodules;
      environment.argo.known-hosts.enable = true;
      environment.argo.provider-tacc.enable = true;
      environment.argo.root-access.enable = true;
      environment.argo.ssh-config.enable = true;
41

Valentin Reis's avatar
Valentin Reis committed
42 43 44 45
      environment.argo.singularity = {
        enable = true;
        package = argopkgs.singularity;
      };
Valentin Reis's avatar
Valentin Reis committed
46

Valentin Reis's avatar
Valentin Reis committed
47
      environment.variables.TERM = "xterm";
48

Valentin Reis's avatar
Valentin Reis committed
49 50 51 52
      i18n.defaultLocale = "en_US.UTF-8";
      nix.useSandbox = true;
      nix.nrBuildUsers = 30;
      nix.trustedUsers = [ "root" "fre" ];
53

Valentin Reis's avatar
Valentin Reis committed
54 55
      services.ntp.enable = false;
      services.openssh.allowSFTP = false;
56

Valentin Reis's avatar
Valentin Reis committed
57
      environment.systemPackages = [ pkgs.unar pkgs.wget pkgs.git ];
58

Valentin Reis's avatar
Valentin Reis committed
59
      virtualisation.docker.enable = true;
60

Valentin Reis's avatar
Valentin Reis committed
61 62 63 64 65
      services.gitlabrunner.enable = true;
      services.gitlabrunner.name = "chameleon-runner-" + ip;
      services.gitlabrunner.registrationConfigFile = "/run/gitlab.cfg";
      services.gitlabrunner.packages =
        [ pkgs.bash pkgs.docker-machine pkgs.shadow pkgs.git ];
66

Valentin Reis's avatar
Valentin Reis committed
67
      #nix.gc = {
68 69 70
      #automatic = false;
      #dates = "05:15";
      #options = ''--max-freed "$((32 * 1024**3 - 1024 * $(df -P -k /nix/store | tail -n 1 | ${pkgs.gawk}/bin/awk '{ print $4 }')))"'';
Valentin Reis's avatar
Valentin Reis committed
71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89
      #};
      services.openssh.enable = true;
      users.extraUsers.root.openssh.authorizedKeys.keys =
        [ (pkgs.lib.readFile ./keys/id_buildfarm.pub) ] ++ keys;
      users.extraUsers.fre.openssh.authorizedKeys.keys =
        [ (pkgs.lib.readFile ./keys/id_buildfarm.pub) ] ++ keys;
    };
in {
  network.description = "argo-ci";
  network.enableRollback = false;

  gitlab-runner-1 = mkChameleonRunner "129.114.24.216";
  gitlab-runner-2 = mkChameleonRunner "129.114.24.194";
  gitlab-runner-3 = mkChameleonRunner "129.114.24.215";
  gitlab-runner-4 = mkChameleonRunner "129.114.24.218";

  nix-store = { config, services, networking, ... }:
    let ip = "129.114.24.212";
    in {
90
      deployment.targetEnv = "none";
91
      deployment.targetHost = ip;
92

Valentin Reis's avatar
Valentin Reis committed
93 94
      time.timeZone = "America/Chicago";

Valentin Reis's avatar
Valentin Reis committed
95 96
      i18n.defaultLocale = "en_US.UTF-8";

Valentin Reis's avatar
Valentin Reis committed
97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112
      require = argomodules;
      environment.argo.known-hosts.enable = true;
      environment.argo.provider-openspace.enable = true;
      environment.argo.root-access.enable = true;

      #services.openssh.enable = true;
      users.extraUsers.root.openssh.authorizedKeys.keys =
        [ (pkgs.lib.readFile ./keys/id_buildfarm.pub) ] ++ keys;
      users.extraUsers.fre.openssh.authorizedKeys.keys =
        [ (pkgs.lib.readFile ./keys/id_buildfarm.pub) ] ++ keys;

      deployment.keys."nix-cache-key.sec" = {
        destDir = "/run";
        keyFile = ./keys/nix-cache-key.sec.secret;
        user = "nix-serve";
        group = "nogroup";
Valentin Reis's avatar
Valentin Reis committed
113 114 115 116 117 118 119 120 121
        permissions = "666";
      };

      deployment.keys."id_buildfarm" = {
        destDir = "/run";
        keyFile = ./keys/id_buildfarm.secret;
        user = "fre";
        group = "users";
        permissions = "666";
Valentin Reis's avatar
Valentin Reis committed
122 123 124
      };

      services.nix-serve = {
Valentin Reis's avatar
Valentin Reis committed
125
        enable = true;
Valentin Reis's avatar
Valentin Reis committed
126
        secretKeyFile = "/run/nix-cache-key.sec";
Valentin Reis's avatar
Valentin Reis committed
127 128
      };

Valentin Reis's avatar
Valentin Reis committed
129 130
      networking.firewall.allowedTCPPorts = [ 5000 80 3000 ];
      networking.firewall.allowedUDPPorts = [ 5000 80 3000 ];
131

Valentin Reis's avatar
Valentin Reis committed
132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154
      services.nginx = {
        enable = true;
        virtualHosts."_" = {
          locations."/" = {
            proxyPass =
              "http://localhost:${toString config.services.hydra.port}";
            extraConfig = ''
              proxy_set_header  Host              $host;
              proxy_set_header  X-Real-IP         $remote_addr;
              proxy_set_header  X-Forwarded-For   $proxy_add_x_forwarded_for;
            '';
          };
          locations."/serve/" = {
            proxyPass =
              "http://localhost:${toString config.services.nix-serve.port}/";
            extraConfig = ''
              proxy_set_header  Host              $host;
              proxy_set_header  X-Real-IP         $remote_addr;
              proxy_set_header  X-Forwarded-For   $proxy_add_x_forwarded_for;
            '';
          };
        };
      };
155

Valentin Reis's avatar
Valentin Reis committed
156 157 158 159 160
      services.postgresql = {
        enable = true;
        dataDir =
          "/var/db/postgresql-${config.services.postgresql.package.psqlSchema}";
      };
161

Valentin Reis's avatar
Valentin Reis committed
162 163 164
      programs.ssh.extraConfig = ''
        StrictHostKeyChecking no
      '';
165

Valentin Reis's avatar
Valentin Reis committed
166 167 168 169 170 171 172
      services.hydra = {
        enable = true;
        hydraURL = "https://${ip}/hydra";
        notificationSender = "fre@freux.fr";
        useSubstitutes = true;
        smtpHost = "localhost";
        extraConfig = ''
Valentin Reis's avatar
Valentin Reis committed
173
          secret-key=/run/nix-cache-key.sec
Valentin Reis's avatar
Valentin Reis committed
174 175 176
          store_uri =  file:///var/lib/hydra/cache?secret-key=/run/nix-cache-key.sec
          using_frontend_proxy 1
        '';
Valentin Reis's avatar
Valentin Reis committed
177
        buildMachinesFiles = [ "/etc/nix/machines" ];
Valentin Reis's avatar
Valentin Reis committed
178
      };
179

Valentin Reis's avatar
Valentin Reis committed
180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202
      systemd.services.hydra-manual-setup = {
        description = "Initial setup for Hydra";
        serviceConfig.Type = "oneshot";
        serviceConfig.RemainAfterExit = true;
        wantedBy = [ "multi-user.target" ];
        requires = [ "hydra-init.service" ];
        after = [ "hydra-init.service" ];
        environment =
          builtins.removeAttrs (config.systemd.services.hydra-init.environment)
          [ "PATH" ];
        script = ''
          if [ ! -e ~hydra/.setup-is-complete ]; then
            # create signing keys
            /run/current-system/sw/bin/install -d -m 551 /etc/nix/${ip}
            /run/current-system/sw/bin/chown -R hydra:hydra /etc/nix/${ip}
            # create cache
            /run/current-system/sw/bin/install -d -m 755 /var/lib/hydra/cache
            /run/current-system/sw/bin/chown -R hydra-queue-runner:hydra /var/lib/hydra/cache
            # done
            touch ~hydra/.setup-is-complete
          fi
        '';
      };
203

Valentin Reis's avatar
Valentin Reis committed
204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221
      nix = {
        useSandbox = true;
        extraOptions = "auto-optimise-store = true";
        trustedUsers = [ "hydra" "fre" "root" ];
        binaryCaches = [ "https://cache.nixos.org" ];
        buildMachines = [{
          hostName = "localhost";
          systems = [ "x86_64-linux" "i686-linux" ];
          speedFactor = 1;
          maxJobs = 6;
          supportedFeatures = [ ];
          sshKey = "/run/id_buildfarm";
          sshUser = "fre";
        }];
        nrBuildUsers = 30;
        distributedBuilds = true;
      };

222
      environment.systemPackages = [ pkgs.unar pkgs.wget pkgs.git pkgs.vim pkgs.htop];
Valentin Reis's avatar
Valentin Reis committed
223

Valentin Reis's avatar
Valentin Reis committed
224
    };
225

Valentin Reis's avatar
Valentin Reis committed
226
}