diff options
Diffstat (limited to 'nixpkgs/nixos/modules/services/continuous-integration/gitlab-runner.nix')
| -rw-r--r-- | nixpkgs/nixos/modules/services/continuous-integration/gitlab-runner.nix | 540 |
1 files changed, 437 insertions, 103 deletions
diff --git a/nixpkgs/nixos/modules/services/continuous-integration/gitlab-runner.nix b/nixpkgs/nixos/modules/services/continuous-integration/gitlab-runner.nix index bd4cf6a37ba..eacfed85ddf 100644 --- a/nixpkgs/nixos/modules/services/continuous-integration/gitlab-runner.nix +++ b/nixpkgs/nixos/modules/services/continuous-integration/gitlab-runner.nix @@ -1,160 +1,494 @@ { config, lib, pkgs, ... }: - with lib; - let cfg = config.services.gitlab-runner; - configFile = - if (cfg.configFile == null) then - (pkgs.runCommand "config.toml" { - buildInputs = [ pkgs.remarshal ]; - preferLocalBuild = true; - } '' - remarshal -if json -of toml \ - < ${pkgs.writeText "config.json" (builtins.toJSON cfg.configOptions)} \ - > $out - '') - else - cfg.configFile; hasDocker = config.virtualisation.docker.enable; + hashedServices = with builtins; (mapAttrs' (name: service: nameValuePair + "${name}_${config.networking.hostName}_${ + substring 0 12 + (hashString "md5" (unsafeDiscardStringContext (toJSON service)))}" + service) + cfg.services); + configPath = "$HOME/.gitlab-runner/config.toml"; + configureScript = pkgs.writeShellScriptBin "gitlab-runner-configure" ( + if (cfg.configFile != null) then '' + mkdir -p $(dirname ${configPath}) + cp ${cfg.configFile} ${configPath} + # make config file readable by service + chown -R --reference=$HOME $(dirname ${configPath}) + '' else '' + export CONFIG_FILE=${configPath} + + mkdir -p $(dirname ${configPath}) + + # remove no longer existing services + gitlab-runner verify --delete + + # current and desired state + NEEDED_SERVICES=$(echo ${concatStringsSep " " (attrNames hashedServices)} | tr " " "\n") + REGISTERED_SERVICES=$(gitlab-runner list 2>&1 | grep 'Executor' | awk '{ print $1 }') + + # difference between current and desired state + NEW_SERVICES=$(grep -vxF -f <(echo "$REGISTERED_SERVICES") <(echo "$NEEDED_SERVICES") || true) + OLD_SERVICES=$(grep -vxF -f <(echo "$NEEDED_SERVICES") <(echo "$REGISTERED_SERVICES") || true) + + # register new services + ${concatStringsSep "\n" (mapAttrsToList (name: service: '' + if echo "$NEW_SERVICES" | grep -xq ${name}; then + bash -c ${escapeShellArg (concatStringsSep " \\\n " ([ + "set -a && source ${service.registrationConfigFile} &&" + "gitlab-runner register" + "--non-interactive" + "--name ${name}" + "--executor ${service.executor}" + "--limit ${toString service.limit}" + "--request-concurrency ${toString service.requestConcurrency}" + "--maximum-timeout ${toString service.maximumTimeout}" + ] ++ service.registrationFlags + ++ optional (service.buildsDir != null) + "--builds-dir ${service.buildsDir}" + ++ optional (service.preCloneScript != null) + "--pre-clone-script ${service.preCloneScript}" + ++ optional (service.preBuildScript != null) + "--pre-build-script ${service.preBuildScript}" + ++ optional (service.postBuildScript != null) + "--post-build-script ${service.postBuildScript}" + ++ optional (service.tagList != [ ]) + "--tag-list ${concatStringsSep "," service.tagList}" + ++ optional service.runUntagged + "--run-untagged" + ++ optional service.protected + "--access-level ref_protected" + ++ optional service.debugTraceDisabled + "--debug-trace-disabled" + ++ map (e: "--env ${escapeShellArg e}") (mapAttrsToList (name: value: "${name}=${value}") service.environmentVariables) + ++ optionals (service.executor == "docker") ( + assert ( + assertMsg (service.dockerImage != null) + "dockerImage option is required for docker executor (${name})"); + [ "--docker-image ${service.dockerImage}" ] + ++ optional service.dockerDisableCache + "--docker-disable-cache" + ++ optional service.dockerPrivileged + "--docker-privileged" + ++ map (v: "--docker-volumes ${escapeShellArg v}") service.dockerVolumes + ++ map (v: "--docker-extra-hosts ${escapeShellArg v}") service.dockerExtraHosts + ++ map (v: "--docker-allowed-images ${escapeShellArg v}") service.dockerAllowedImages + ++ map (v: "--docker-allowed-services ${escapeShellArg v}") service.dockerAllowedServices + ) + ))} && sleep 1 + fi + '') hashedServices)} + + # unregister old services + for NAME in $(echo "$OLD_SERVICES") + do + [ ! -z "$NAME" ] && gitlab-runner unregister \ + --name "$NAME" && sleep 1 + done + + # update global options + remarshal --if toml --of json ${configPath} \ + | jq -cM '.check_interval = ${toString cfg.checkInterval} | + .concurrent = ${toString cfg.concurrent}' \ + | remarshal --if json --of toml \ + | sponge ${configPath} + + # make config file readable by service + chown -R --reference=$HOME $(dirname ${configPath}) + ''); + startScript = pkgs.writeShellScriptBin "gitlab-runner-start" '' + export CONFIG_FILE=${configPath} + exec gitlab-runner run --working-directory $HOME + ''; in { options.services.gitlab-runner = { enable = mkEnableOption "Gitlab Runner"; - configFile = mkOption { + type = types.nullOr types.path; default = null; description = '' Configuration file for gitlab-runner. - Use this option in favor of configOptions to avoid placing CI tokens in the nix store. - <option>configFile</option> takes precedence over <option>configOptions</option>. + <option>configFile</option> takes precedence over <option>services</option>. + <option>checkInterval</option> and <option>concurrent</option> will be ignored too. - Warning: Not using <option>configFile</option> will potentially result in secrets - leaking into the WORLD-READABLE nix store. + This option is deprecated, please use <option>services</option> instead. + You can use <option>registrationConfigFile</option> and + <option>registrationFlags</option> + for settings not covered by this module. ''; - type = types.nullOr types.path; }; - - configOptions = mkOption { + checkInterval = mkOption { + type = types.int; + default = 0; + example = literalExample "with lib; (length (attrNames config.services.gitlab-runner.services)) * 3"; description = '' - Configuration for gitlab-runner - <option>configFile</option> will take precedence over this option. - - Warning: all Configuration, especially CI token, will be stored in a - WORLD-READABLE file in the Nix Store. - - If you want to protect your CI token use <option>configFile</option> instead. + Defines the interval length, in seconds, between new jobs check. + The default value is 3; + if set to 0 or lower, the default value will be used. + See <link xlink:href="https://docs.gitlab.com/runner/configuration/advanced-configuration.html#how-check_interval-works">runner documentation</link> for more information. + ''; + }; + concurrent = mkOption { + type = types.int; + default = 1; + example = literalExample "config.nix.maxJobs"; + description = '' + Limits how many jobs globally can be run concurrently. + The most upper limit of jobs using all defined runners. + 0 does not mean unlimited. ''; - type = types.attrs; - example = { - concurrent = 2; - runners = [{ - name = "docker-nix-1.11"; - url = "https://CI/"; - token = "TOKEN"; - executor = "docker"; - builds_dir = ""; - docker = { - host = ""; - image = "nixos/nix:1.11"; - privileged = true; - disable_cache = true; - cache_dir = ""; - }; - }]; - }; }; - gracefulTermination = mkOption { - default = false; type = types.bool; + default = false; description = '' - Finish all remaining jobs before stopping, restarting or reconfiguring. - If not set gitlab-runner will stop immediatly without waiting for jobs to finish, - which will lead to failed builds. + Finish all remaining jobs before stopping. + If not set gitlab-runner will stop immediatly without waiting + for jobs to finish, which will lead to failed builds. ''; }; - gracefulTimeout = mkOption { - default = "infinity"; type = types.str; + default = "infinity"; example = "5min 20s"; - description = ''Time to wait until a graceful shutdown is turned into a forceful one.''; - }; - - workDir = mkOption { - default = "/var/lib/gitlab-runner"; - type = types.path; - description = "The working directory used"; + description = '' + Time to wait until a graceful shutdown is turned into a forceful one. + ''; }; - package = mkOption { - description = "Gitlab Runner package to use"; + type = types.package; default = pkgs.gitlab-runner; defaultText = "pkgs.gitlab-runner"; - type = types.package; example = literalExample "pkgs.gitlab-runner_1_11"; + description = "Gitlab Runner package to use."; }; - - packages = mkOption { - default = [ pkgs.bash pkgs.docker-machine ]; - defaultText = "[ pkgs.bash pkgs.docker-machine ]"; + extraPackages = mkOption { type = types.listOf types.package; + default = [ ]; description = '' - Packages to add to PATH for the gitlab-runner process. + Extra packages to add to PATH for the gitlab-runner process. ''; }; + services = mkOption { + description = "GitLab Runner services."; + default = { }; + example = literalExample '' + { + # runner for building in docker via host's nix-daemon + # nix store will be readable in runner, might be insecure + nix = { + # File should contain at least these two variables: + # `CI_SERVER_URL` + # `REGISTRATION_TOKEN` + registrationConfigFile = "/run/secrets/gitlab-runner-registration"; + dockerImage = "alpine"; + dockerVolumes = [ + "/nix/store:/nix/store:ro" + "/nix/var/nix/db:/nix/var/nix/db:ro" + "/nix/var/nix/daemon-socket:/nix/var/nix/daemon-socket:ro" + ]; + dockerDisableCache = true; + preBuildScript = pkgs.writeScript "setup-container" ''' + mkdir -p -m 0755 /nix/var/log/nix/drvs + mkdir -p -m 0755 /nix/var/nix/gcroots + mkdir -p -m 0755 /nix/var/nix/profiles + mkdir -p -m 0755 /nix/var/nix/temproots + mkdir -p -m 0755 /nix/var/nix/userpool + mkdir -p -m 1777 /nix/var/nix/gcroots/per-user + mkdir -p -m 1777 /nix/var/nix/profiles/per-user + mkdir -p -m 0755 /nix/var/nix/profiles/per-user/root + mkdir -p -m 0700 "$HOME/.nix-defexpr" - }; + . ''${pkgs.nix}/etc/profile.d/nix.sh + ''${pkgs.nix}/bin/nix-env -i ''${concatStringsSep " " (with pkgs; [ nix cacert git openssh ])} + + ''${pkgs.nix}/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable + ''${pkgs.nix}/bin/nix-channel --update nixpkgs + '''; + environmentVariables = { + ENV = "/etc/profile"; + USER = "root"; + NIX_REMOTE = "daemon"; + PATH = "/nix/var/nix/profiles/default/bin:/nix/var/nix/profiles/default/sbin:/bin:/sbin:/usr/bin:/usr/sbin"; + NIX_SSL_CERT_FILE = "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"; + }; + tagList = [ "nix" ]; + }; + # runner for building docker images + docker-images = { + # File should contain at least these two variables: + # `CI_SERVER_URL` + # `REGISTRATION_TOKEN` + registrationConfigFile = "/run/secrets/gitlab-runner-registration"; + dockerImage = "docker:stable"; + dockerVolumes = [ + "/var/run/docker.sock:/var/run/docker.sock" + ]; + tagList = [ "docker-images" ]; + }; + # runner for executing stuff on host system (very insecure!) + # make sure to add required packages (including git!) + # to `environment.systemPackages` + shell = { + # File should contain at least these two variables: + # `CI_SERVER_URL` + # `REGISTRATION_TOKEN` + registrationConfigFile = "/run/secrets/gitlab-runner-registration"; + executor = "shell"; + tagList = [ "shell" ]; + }; + # runner for everything else + default = { + # File should contain at least these two variables: + # `CI_SERVER_URL` + # `REGISTRATION_TOKEN` + registrationConfigFile = "/run/secrets/gitlab-runner-registration"; + dockerImage = "debian:stable"; + }; + } + ''; + type = types.attrsOf (types.submodule { + options = { + registrationConfigFile = mkOption { + type = types.path; + description = '' + Absolute path to a file with environment variables + used for gitlab-runner registration. + A list of all supported environment variables can be found in + <literal>gitlab-runner register --help</literal>. + + Ones that you probably want to set is + + <literal>CI_SERVER_URL=<CI server URL></literal> + + <literal>REGISTRATION_TOKEN=<registration secret></literal> + ''; + }; + registrationFlags = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "--docker-helper-image my/gitlab-runner-helper" ]; + description = '' + Extra command-line flags passed to + <literal>gitlab-runner register</literal>. + Execute <literal>gitlab-runner register --help</literal> + for a list of supported flags. + ''; + }; + environmentVariables = mkOption { + type = types.attrsOf types.str; + default = { }; + example = { NAME = "value"; }; + description = '' + Custom environment variables injected to build environment. + For secrets you can use <option>registrationConfigFile</option> + with <literal>RUNNER_ENV</literal> variable set. + ''; + }; + executor = mkOption { + type = types.str; + default = "docker"; + description = '' + Select executor, eg. shell, docker, etc. + See <link xlink:href="https://docs.gitlab.com/runner/executors/README.html">runner documentation</link> for more information. + ''; + }; + buildsDir = mkOption { + type = types.nullOr types.path; + default = null; + example = "/var/lib/gitlab-runner/builds"; + description = '' + Absolute path to a directory where builds will be stored + in context of selected executor (Locally, Docker, SSH). + ''; + }; + dockerImage = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + Docker image to be used. + ''; + }; + dockerVolumes = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "/var/run/docker.sock:/var/run/docker.sock" ]; + description = '' + Bind-mount a volume and create it + if it doesn't exist prior to mounting. + ''; + }; + dockerDisableCache = mkOption { + type = types.bool; + default = false; + description = '' + Disable all container caching. + ''; + }; + dockerPrivileged = mkOption { + type = types.bool; + default = false; + description = '' + Give extended privileges to container. + ''; + }; + dockerExtraHosts = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "other-host:127.0.0.1" ]; + description = '' + Add a custom host-to-IP mapping. + ''; + }; + dockerAllowedImages = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "ruby:*" "python:*" "php:*" "my.registry.tld:5000/*:*" ]; + description = '' + Whitelist allowed images. + ''; + }; + dockerAllowedServices = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "postgres:9" "redis:*" "mysql:*" ]; + description = '' + Whitelist allowed services. + ''; + }; + preCloneScript = mkOption { + type = types.nullOr types.path; + default = null; + description = '' + Runner-specific command script executed before code is pulled. + ''; + }; + preBuildScript = mkOption { + type = types.nullOr types.path; + default = null; + description = '' + Runner-specific command script executed after code is pulled, + just before build executes. + ''; + }; + postBuildScript = mkOption { + type = types.nullOr types.path; + default = null; + description = '' + Runner-specific command script executed after code is pulled + and just after build executes. + ''; + }; + tagList = mkOption { + type = types.listOf types.str; + default = [ ]; + description = '' + Tag list. + ''; + }; + runUntagged = mkOption { + type = types.bool; + default = false; + description = '' + Register to run untagged builds; defaults to + <literal>true</literal> when <option>tagList</option> is empty. + ''; + }; + limit = mkOption { + type = types.int; + default = 0; + description = '' + Limit how many jobs can be handled concurrently by this service. + 0 (default) simply means don't limit. + ''; + }; + requestConcurrency = mkOption { + type = types.int; + default = 0; + description = '' + Limit number of concurrent requests for new jobs from GitLab. + ''; + }; + maximumTimeout = mkOption { + type = types.int; + default = 0; + description = '' + What is the maximum timeout (in seconds) that will be set for + job when using this Runner. 0 (default) simply means don't limit. + ''; + }; + protected = mkOption { + type = types.bool; + default = false; + description = '' + When set to true Runner will only run on pipelines + triggered on protected branches. + ''; + }; + debugTraceDisabled = mkOption { + type = types.bool; + default = false; + description = '' + When set to true Runner will disable the possibility of + using the <literal>CI_DEBUG_TRACE</literal> feature. + ''; + }; + }; + }); + }; + }; config = mkIf cfg.enable { + warnings = optional (cfg.configFile != null) "services.gitlab-runner.`configFile` is deprecated, please use services.gitlab-runner.`services`."; + environment.systemPackages = [ cfg.package ]; systemd.services.gitlab-runner = { - path = cfg.packages; - environment = config.networking.proxy.envVars // { - # Gitlab runner will not start if the HOME variable is not set - HOME = cfg.workDir; - }; description = "Gitlab Runner"; + documentation = [ "https://docs.gitlab.com/runner/" ]; after = [ "network.target" ] ++ optional hasDocker "docker.service"; requires = optional hasDocker "docker.service"; wantedBy = [ "multi-user.target" ]; + environment = config.networking.proxy.envVars // { + HOME = "/var/lib/gitlab-runner"; + }; + path = with pkgs; [ + bash + gawk + jq + moreutils + remarshal + utillinux + cfg.package + ] ++ cfg.extraPackages; reloadIfChanged = true; - restartTriggers = [ - config.environment.etc."gitlab-runner/config.toml".source - ]; serviceConfig = { + # Set `DynamicUser` under `systemd.services.gitlab-runner.serviceConfig` + # to `lib.mkForce false` in your configuration to run this service as root. + # You can also set `User` and `Group` options to run this service as desired user. + # Make sure to restart service or changes won't apply. + DynamicUser = true; StateDirectory = "gitlab-runner"; - ExecReload= "${pkgs.coreutils}/bin/kill -HUP $MAINPID"; - ExecStart = ''${cfg.package.bin}/bin/gitlab-runner run \ - --working-directory ${cfg.workDir} \ - --config /etc/gitlab-runner/config.toml \ - --service gitlab-runner \ - --user gitlab-runner \ - ''; - - } // optionalAttrs (cfg.gracefulTermination) { + SupplementaryGroups = optional hasDocker "docker"; + ExecStartPre = "!${configureScript}/bin/gitlab-runner-configure"; + ExecStart = "${startScript}/bin/gitlab-runner-start"; + ExecReload = "!${configureScript}/bin/gitlab-runner-configure"; + } // optionalAttrs (cfg.gracefulTermination) { TimeoutStopSec = "${cfg.gracefulTimeout}"; KillSignal = "SIGQUIT"; KillMode = "process"; }; }; - - # Make the gitlab-runner command availabe so users can query the runner - environment.systemPackages = [ cfg.package ]; - - # Make sure the config can be reloaded on change - environment.etc."gitlab-runner/config.toml".source = configFile; - - users.users.gitlab-runner = { - group = "gitlab-runner"; - extraGroups = optional hasDocker "docker"; - uid = config.ids.uids.gitlab-runner; - home = cfg.workDir; - createHome = true; - }; - - users.groups.gitlab-runner.gid = config.ids.gids.gitlab-runner; + # Enable docker if `docker` executor is used in any service + virtualisation.docker.enable = mkIf ( + any (s: s.executor == "docker") (attrValues cfg.services) + ) (mkDefault true); }; + imports = [ + (mkRenamedOptionModule [ "services" "gitlab-runner" "packages" ] [ "services" "gitlab-runner" "extraPackages" ] ) + (mkRemovedOptionModule [ "services" "gitlab-runner" "configOptions" ] "Use services.gitlab-runner.services option instead" ) + (mkRemovedOptionModule [ "services" "gitlab-runner" "workDir" ] "You should move contents of workDir (if any) to /var/lib/gitlab-runner" ) + ]; } |