buildbot: add Python 3 support

5 files changed, 236 insertions, 135 deletions

diff --git a/nixos/doc/manual/release-notes/rl-1903.xml b/nixos/doc/manual/release-notes/rl-1903.xml
index 9cb5b93f27c..c4847b60e27 100644
--- a/nixos/doc/manual/release-notes/rl-1903.xml
+++ b/nixos/doc/manual/release-notes/rl-1903.xml
@@ -113,6 +113,15 @@
        (i.e. <literal>users.users.yourusername.extraGroups = ["video"];</literal>).
      </para>
    </listitem>
+   <listitem>
+    <para>
+     Buildbot now supports Python 3 and its packages have been moved to
+     <literal>pythonPackages</literal>. The options
+     <option>services.buildbot-master.package</option> and
+     <option>services.buildbot-worker.package</option> can be used to select
+     the Python 2 or 3 version of the package.
+    </para>
+   </listitem>
   </itemizedlist>
  </section>
 
diff --git a/nixos/modules/services/continuous-integration/buildbot/master.nix b/nixos/modules/services/continuous-integration/buildbot/master.nix
index 8d767de37f0..0f07e6133bb 100644
--- a/nixos/modules/services/continuous-integration/buildbot/master.nix
+++ b/nixos/modules/services/continuous-integration/buildbot/master.nix
@@ -6,8 +6,12 @@ with lib;
 
 let
   cfg = config.services.buildbot-master;
+
+  python = cfg.package.pythonModule;
+
   escapeStr = s: escape ["'"] s;
-  masterCfg = if cfg.masterCfg == null then pkgs.writeText "master.cfg" ''
+
+  defaultMasterCfg = pkgs.writeText "master.cfg" ''
     from buildbot.plugins import *
     factory = util.BuildFactory()
     c = BuildmasterConfig = dict(
@@ -27,8 +31,28 @@ let
       factory.addStep(step)
 
     ${cfg.extraConfig}
-  ''
-  else cfg.masterCfg;
+  '';
+
+  tacFile = pkgs.writeText "buildbot-master.tac" ''
+    import os
+
+    from twisted.application import service
+    from buildbot.master import BuildMaster
+
+    basedir = '${cfg.buildbotDir}'
+
+    configfile = '${cfg.masterCfg}'
+
+    # Default umask for server
+    umask = None
+
+    # note: this line is matched against to check that this is a buildmaster
+    # directory; do not edit it.
+    application = service.Application('buildmaster')
+
+    m = BuildMaster(basedir, configfile, umask)
+    m.setServiceParent(application)
+  '';
 
 in {
   options = {
@@ -66,9 +90,9 @@ in {
       };
 
       masterCfg = mkOption {
-        type = types.nullOr types.path;
+        type = types.path;
         description = "Optionally pass master.cfg path. Other options in this configuration will be ignored.";
-        default = null;
+        default = defaultMasterCfg;
         example = "/etc/nixos/buildbot/master.cfg";
       };
 
@@ -175,18 +199,25 @@ in {
 
       package = mkOption {
         type = types.package;
-        default = pkgs.buildbot-full;
-        defaultText = "pkgs.buildbot-full";
+        default = pkgs.pythonPackages.buildbot-full;
+        defaultText = "pkgs.pythonPackages.buildbot-full";
         description = "Package to use for buildbot.";
-        example = literalExample "pkgs.buildbot-full";
+        example = literalExample "pkgs.python3Packages.buildbot-full";
       };
 
       packages = mkOption {
-        default = with pkgs; [ python27Packages.twisted git ];
+        default = [ pkgs.git ];
         example = literalExample "[ pkgs.git ]";
         type = types.listOf types.package;
         description = "Packages to add to PATH for the buildbot process.";
       };
+
+      pythonPackages = mkOption {
+        default = pythonPackages: with pythonPackages; [ ];
+        defaultText = "pythonPackages: with pythonPackages; [ ]";
+        description = "Packages to add the to the PYTHONPATH of the buildbot process.";
+        example = literalExample "pythonPackages: with pythonPackages; [ requests ]";
+      };
     };
   };
 
@@ -210,14 +241,15 @@ in {
       description = "Buildbot Continuous Integration Server.";
       after = [ "network-online.target" ];
       wantedBy = [ "multi-user.target" ];
-      path = cfg.packages;
+      path = cfg.packages ++ cfg.pythonPackages python.pkgs;
+      environment.PYTHONPATH = "${python.withPackages (self: cfg.pythonPackages self ++ [ cfg.package ])}/${python.sitePackages}";
 
       preStart = ''
-        env > envvars
-        mkdir -vp ${cfg.buildbotDir}
-        ln -sfv ${masterCfg} ${cfg.buildbotDir}/master.cfg
-        rm -fv $cfg.buildbotDir}/buildbot.tac
-        ${cfg.package}/bin/buildbot create-master ${cfg.buildbotDir}
+        mkdir -vp "${cfg.buildbotDir}"
+        # Link the tac file so buildbot command line tools recognize the directory
+        ln -sf "${tacFile}" "${cfg.buildbotDir}/buildbot.tac"
+        ${cfg.package}/bin/buildbot create-master --db "${cfg.dbUrl}" "${cfg.buildbotDir}"
+        rm -f buildbot.tac.new master.cfg.sample
       '';
 
       serviceConfig = {
@@ -225,12 +257,11 @@ in {
         User = cfg.user;
         Group = cfg.group;
         WorkingDirectory = cfg.home;
-        ExecStart = "${cfg.package}/bin/buildbot start --nodaemon ${cfg.buildbotDir}";
+        # NOTE: call twistd directly with stdout logging for systemd
+        ExecStart = "${python.pkgs.twisted}/bin/twistd -o --nodaemon --pidfile= --logfile - --python ${tacFile}";
       };
-
     };
   };
 
   meta.maintainers = with lib.maintainers; [ nand0p mic92 ];
-
 }
diff --git a/nixos/modules/services/continuous-integration/buildbot/worker.nix b/nixos/modules/services/continuous-integration/buildbot/worker.nix
index 67c541570b9..4130ec918a7 100644
--- a/nixos/modules/services/continuous-integration/buildbot/worker.nix
+++ b/nixos/modules/services/continuous-integration/buildbot/worker.nix
@@ -7,6 +7,40 @@ with lib;
 let
   cfg = config.services.buildbot-worker;
 
+  python = cfg.package.pythonModule;
+
+  tacFile = pkgs.writeText "aur-buildbot-worker.tac" ''
+    import os
+    from io import open
+
+    from buildbot_worker.bot import Worker
+    from twisted.application import service
+
+    basedir = '${cfg.buildbotDir}'
+
+    # note: this line is matched against to check that this is a worker
+    # directory; do not edit it.
+    application = service.Application('buildbot-worker')
+
+    master_url_split = '${cfg.masterUrl}'.split(':')
+    buildmaster_host = master_url_split[0]
+    port = int(master_url_split[1])
+    workername = '${cfg.workerUser}'
+
+    with open('${cfg.workerPassFile}', 'r', encoding='utf-8') as passwd_file:
+        passwd = passwd_file.read().strip('\r\n')
+    keepalive = 600
+    umask = None
+    maxdelay = 300
+    numcpus = None
+    allow_shutdown = None
+
+    s = Worker(buildmaster_host, port, workername, passwd, basedir,
+               keepalive, umask=umask, maxdelay=maxdelay,
+               numcpus=numcpus, allow_shutdown=allow_shutdown)
+    s.setServiceParent(application)
+  '';
+
 in {
   options = {
     services.buildbot-worker = {
@@ -59,6 +93,23 @@ in {
         description = "Specifies the Buildbot Worker password.";
       };
 
+      workerPassFile = mkOption {
+        type = types.path;
+        description = "File used to store the Buildbot Worker password";
+      };
+
+      hostMessage = mkOption {
+        default = null;
+        type = types.nullOr types.str;
+        description = "Description of this worker";
+      };
+
+      adminMessage = mkOption {
+        default = null;
+        type = types.nullOr types.str;
+        description = "Name of the administrator of this worker";
+      };
+
       masterUrl = mkOption {
         default = "localhost:9989";
         type = types.str;
@@ -67,23 +118,24 @@ in {
 
       package = mkOption {
         type = types.package;
-        default = pkgs.buildbot-worker;
-        defaultText = "pkgs.buildbot-worker";
+        default = pkgs.pythonPackages.buildbot-worker;
+        defaultText = "pkgs.pythonPackages.buildbot-worker";
         description = "Package to use for buildbot worker.";
-        example = literalExample "pkgs.buildbot-worker";
+        example = literalExample "pkgs.python3Packages.buildbot-worker";
       };
 
       packages = mkOption {
-        default = with pkgs; [ python27Packages.twisted git ];
+        default = with pkgs; [ git ];
         example = literalExample "[ pkgs.git ]";
         type = types.listOf types.package;
         description = "Packages to add to PATH for the buildbot process.";
       };
-
     };
   };
 
   config = mkIf cfg.enable {
+    services.buildbot-worker.workerPassFile = mkDefault (pkgs.writeText "buildbot-worker-password" cfg.workerPass);
+
     users.groups = optional (cfg.group == "bbworker") {
       name = "bbworker";
     };
@@ -104,11 +156,16 @@ in {
       after = [ "network.target" "buildbot-master.service" ];
       wantedBy = [ "multi-user.target" ];
       path = cfg.packages;
+      environment.PYTHONPATH = "${python.withPackages (p: [ cfg.package ])}/${python.sitePackages}";
 
       preStart = ''
-        mkdir -vp ${cfg.buildbotDir}
-        rm -fv $cfg.buildbotDir}/buildbot.tac
-        ${cfg.package}/bin/buildbot-worker create-worker ${cfg.buildbotDir} ${cfg.masterUrl} ${cfg.workerUser} ${cfg.workerPass}
+        mkdir -vp "${cfg.buildbotDir}/info"
+        ${optionalString (cfg.hostMessage != null) ''
+          ln -sf "${pkgs.writeText "buildbot-worker-host" cfg.hostMessage}" "${cfg.buildbotDir}/info/host"
+        ''}
+        ${optionalString (cfg.adminMessage != null) ''
+          ln -sf "${pkgs.writeText "buildbot-worker-admin" cfg.adminMessage}" "${cfg.buildbotDir}/info/admin"
+        ''}
       '';
 
       serviceConfig = {
@@ -116,11 +173,9 @@ in {
         User = cfg.user;
         Group = cfg.group;
         WorkingDirectory = cfg.home;
-        Environment = "PYTHONPATH=${cfg.package}/lib/python2.7/site-packages:${pkgs.python27Packages.future}/lib/python2.7/site-packages";
 
         # NOTE: call twistd directly with stdout logging for systemd
-        #ExecStart = "${cfg.package}/bin/buildbot-worker start --nodaemon ${cfg.buildbotDir}";
-        ExecStart = "${pkgs.python27Packages.twisted}/bin/twistd -n -l - -y ${cfg.buildbotDir}/buildbot.tac";
+        ExecStart = "${python.pkgs.twisted}/bin/twistd --nodaemon --pidfile= --logfile - --python ${tacFile}";
       };
 
     };
diff --git a/nixos/release.nix b/nixos/release.nix
index 86383f502a7..5412080cca1 100644
--- a/nixos/release.nix
+++ b/nixos/release.nix
@@ -257,7 +257,7 @@ in rec {
   tests.boot = callSubTests tests/boot.nix {};
   tests.boot-stage1 = callTest tests/boot-stage1.nix {};
   tests.borgbackup = callTest tests/borgbackup.nix {};
-  tests.buildbot = callTest tests/buildbot.nix {};
+  tests.buildbot = callSubTests tests/buildbot.nix {};
   tests.cadvisor = callTestOnMatchingSystems ["x86_64-linux"] tests/cadvisor.nix {};
   tests.ceph = callTestOnMatchingSystems ["x86_64-linux"] tests/ceph.nix {};
   tests.certmgr = callSubTests tests/certmgr.nix {};
diff --git a/nixos/tests/buildbot.nix b/nixos/tests/buildbot.nix
index cf408dc7fec..399fd39005e 100644
--- a/nixos/tests/buildbot.nix
+++ b/nixos/tests/buildbot.nix
@@ -1,111 +1,117 @@
-# Test ensures buildbot master comes up correctly and workers can connect
-
-import ./make-test.nix ({ pkgs, ... } : {
-  name = "buildbot";
-
-  nodes = {
-    bbmaster = { pkgs, ... }: {
-      services.buildbot-master = {
-        enable = true;
-        package = pkgs.buildbot-full;
-
-        # NOTE: use fake repo due to no internet in hydra ci
-        factorySteps = [
-          "steps.Git(repourl='git://gitrepo/fakerepo.git', mode='incremental')"
-          "steps.ShellCommand(command=['bash', 'fakerepo.sh'])"
-        ];
-        changeSource = [
-          "changes.GitPoller('git://gitrepo/fakerepo.git', workdir='gitpoller-workdir', branch='master', pollinterval=300)"
-        ];
+{ system ? builtins.currentSystem }:
+
+with import ../lib/testing.nix { inherit system; };
+
+let
+  # Test ensures buildbot master comes up correctly and workers can connect
+  mkBuildbotTest = python: makeTest {
+    name = "buildbot";
+
+    nodes = {
+      bbmaster = { pkgs, ... }: {
+        services.buildbot-master = {
+          enable = true;
+          package = python.pkgs.buildbot-full;
+
+          # NOTE: use fake repo due to no internet in hydra ci
+          factorySteps = [
+            "steps.Git(repourl='git://gitrepo/fakerepo.git', mode='incremental')"
+            "steps.ShellCommand(command=['bash', 'fakerepo.sh'])"
+          ];
+          changeSource = [
+            "changes.GitPoller('git://gitrepo/fakerepo.git', workdir='gitpoller-workdir', branch='master', pollinterval=300)"
+          ];
+        };
+        networking.firewall.allowedTCPPorts = [ 8010 8011 9989 ];
+        environment.systemPackages = with pkgs; [ git python.pkgs.buildbot-full ];
       };
-      networking.firewall.allowedTCPPorts = [ 8010 8011 9989 ];
-      environment.systemPackages = with pkgs; [ git buildbot-full ];
-    };
 
-    bbworker = { pkgs, ... }: {
-      services.buildbot-worker = {
-        enable = true;
-        masterUrl = "bbmaster:9989";
+      bbworker = { pkgs, ... }: {
+        services.buildbot-worker = {
+          enable = true;
+          masterUrl = "bbmaster:9989";
+        };
+        environment.systemPackages = with pkgs; [ git python.pkgs.buildbot-worker ];
       };
-      environment.systemPackages = with pkgs; [ git buildbot-worker ];
-    };
 
-    gitrepo = { pkgs, ... }: {
-      services.openssh.enable = true;
-      networking.firewall.allowedTCPPorts = [ 22 9418 ];
-      environment.systemPackages = with pkgs; [ git ];
+      gitrepo = { pkgs, ... }: {
+        services.openssh.enable = true;
+        networking.firewall.allowedTCPPorts = [ 22 9418 ];
+        environment.systemPackages = with pkgs; [ git ];
+      };
     };
-  };
 
-  testScript = ''
-    #Start up and populate fake repo
-    $gitrepo->waitForUnit("multi-user.target");
-    print($gitrepo->execute(" \
-      git config --global user.name 'Nobody Fakeuser' && \
-      git config --global user.email 'nobody\@fakerepo.com' && \
-      rm -rvf /srv/repos/fakerepo.git /tmp/fakerepo && \
-      mkdir -pv /srv/repos/fakerepo ~/.ssh && \
-      ssh-keyscan -H gitrepo > ~/.ssh/known_hosts && \
-      cat ~/.ssh/known_hosts && \
-      cd /srv/repos/fakerepo && \
-      git init && \
-      echo -e '#!/bin/sh\necho fakerepo' > fakerepo.sh && \
-      cat fakerepo.sh && \
-      touch .git/git-daemon-export-ok && \
-      git add fakerepo.sh .git/git-daemon-export-ok && \
-      git commit -m fakerepo && \
-      git daemon --verbose --export-all --base-path=/srv/repos --reuseaddr & \
-    "));
-
-    # Test gitrepo
-    $bbmaster->waitForUnit("network-online.target");
-    #$bbmaster->execute("nc -z gitrepo 9418");
-    print($bbmaster->execute(" \
-      rm -rfv /tmp/fakerepo && \
-      git clone git://gitrepo/fakerepo /tmp/fakerepo && \
-      pwd && \
-      ls -la && \
-      ls -la /tmp/fakerepo \
-    "));
-
-    # Test start master and connect worker
-    $bbmaster->waitForUnit("buildbot-master.service");
-    $bbmaster->waitUntilSucceeds("curl -s --head http://bbmaster:8010") =~ /200 OK/;
-    $bbworker->waitForUnit("network-online.target");
-    $bbworker->execute("nc -z bbmaster 8010");
-    $bbworker->execute("nc -z bbmaster 9989");
-    $bbworker->waitForUnit("buildbot-worker.service");
-    print($bbworker->execute("ls -la /home/bbworker/worker"));
-
-
-    # Test stop buildbot master and worker
-    print($bbmaster->execute(" \
-      systemctl -l --no-pager status buildbot-master && \
-      systemctl stop buildbot-master \
-    "));
-    $bbworker->fail("nc -z bbmaster 8010");
-    $bbworker->fail("nc -z bbmaster 9989");
-    print($bbworker->execute(" \
-      systemctl -l --no-pager status buildbot-worker && \
-      systemctl stop buildbot-worker && \
-      ls -la /home/bbworker/worker \
-    "));
-
-
-    # Test buildbot daemon mode
-    # NOTE: daemon mode tests disabled due to broken PYTHONPATH child inheritence
-    #
-    #$bbmaster->execute("buildbot create-master /tmp");
-    #$bbmaster->execute("mv -fv /tmp/master.cfg.sample /tmp/master.cfg");
-    #$bbmaster->execute("sed -i 's/8010/8011/' /tmp/master.cfg");
-    #$bbmaster->execute("buildbot start /tmp");
-    #$bbworker->execute("nc -z bbmaster 8011");
-    #$bbworker->waitUntilSucceeds("curl -s --head http://bbmaster:8011") =~ /200 OK/;
-    #$bbmaster->execute("buildbot stop /tmp");
-    #$bbworker->fail("nc -z bbmaster 8011");
-
-  '';
-
-  meta.maintainers = with pkgs.stdenv.lib.maintainers; [ nand0p ];
-
-})
+    testScript = ''
+      #Start up and populate fake repo
+      $gitrepo->waitForUnit("multi-user.target");
+      print($gitrepo->execute(" \
+        git config --global user.name 'Nobody Fakeuser' && \
+        git config --global user.email 'nobody\@fakerepo.com' && \
+        rm -rvf /srv/repos/fakerepo.git /tmp/fakerepo && \
+        mkdir -pv /srv/repos/fakerepo ~/.ssh && \
+        ssh-keyscan -H gitrepo > ~/.ssh/known_hosts && \
+        cat ~/.ssh/known_hosts && \
+        cd /srv/repos/fakerepo && \
+        git init && \
+        echo -e '#!/bin/sh\necho fakerepo' > fakerepo.sh && \
+        cat fakerepo.sh && \
+        touch .git/git-daemon-export-ok && \
+        git add fakerepo.sh .git/git-daemon-export-ok && \
+        git commit -m fakerepo && \
+        git daemon --verbose --export-all --base-path=/srv/repos --reuseaddr & \
+      "));
+
+      # Test gitrepo
+      $bbmaster->waitForUnit("network-online.target");
+      #$bbmaster->execute("nc -z gitrepo 9418");
+      print($bbmaster->execute(" \
+        rm -rfv /tmp/fakerepo && \
+        git clone git://gitrepo/fakerepo /tmp/fakerepo && \
+        pwd && \
+        ls -la && \
+        ls -la /tmp/fakerepo \
+      "));
+
+      # Test start master and connect worker
+      $bbmaster->waitForUnit("buildbot-master.service");
+      $bbmaster->waitUntilSucceeds("curl -s --head http://bbmaster:8010") =~ /200 OK/;
+      $bbworker->waitForUnit("network-online.target");
+      $bbworker->execute("nc -z bbmaster 8010");
+      $bbworker->execute("nc -z bbmaster 9989");
+      $bbworker->waitForUnit("buildbot-worker.service");
+      print($bbworker->execute("ls -la /home/bbworker/worker"));
+
+
+      # Test stop buildbot master and worker
+      print($bbmaster->execute(" \
+        systemctl -l --no-pager status buildbot-master && \
+        systemctl stop buildbot-master \
+      "));
+      $bbworker->fail("nc -z bbmaster 8010");
+      $bbworker->fail("nc -z bbmaster 9989");
+      print($bbworker->execute(" \
+        systemctl -l --no-pager status buildbot-worker && \
+        systemctl stop buildbot-worker && \
+        ls -la /home/bbworker/worker \
+      "));
+
+
+      # Test buildbot daemon mode
+      $bbmaster->execute("buildbot create-master /tmp");
+      $bbmaster->execute("mv -fv /tmp/master.cfg.sample /tmp/master.cfg");
+      $bbmaster->execute("sed -i 's/8010/8011/' /tmp/master.cfg");
+      $bbmaster->execute("buildbot start /tmp");
+      $bbworker->execute("nc -z bbmaster 8011");
+      $bbworker->waitUntilSucceeds("curl -s --head http://bbmaster:8011") =~ /200 OK/;
+      $bbmaster->execute("buildbot stop /tmp");
+      $bbworker->fail("nc -z bbmaster 8011");
+
+    '';
+
+    meta.maintainers = with pkgs.stdenv.lib.maintainers; [ nand0p ];
+
+  };
+in {
+  python2 = mkBuildbotTest pkgs.python2;
+  python3 = mkBuildbotTest pkgs.python3;
+}