buildbot: add Python 3 support

13 files changed, 334 insertions, 232 deletions

diff --git a/nixos/doc/manual/release-notes/rl-1903.xml b/nixos/doc/manual/release-notes/rl-1903.xml
index 9cb5b93f27c..c4847b60e27 100644
--- a/nixos/doc/manual/release-notes/rl-1903.xml
+++ b/nixos/doc/manual/release-notes/rl-1903.xml
@@ -113,6 +113,15 @@
        (i.e. <literal>users.users.yourusername.extraGroups = ["video"];</literal>).
      </para>
    </listitem>
+   <listitem>
+    <para>
+     Buildbot now supports Python 3 and its packages have been moved to
+     <literal>pythonPackages</literal>. The options
+     <option>services.buildbot-master.package</option> and
+     <option>services.buildbot-worker.package</option> can be used to select
+     the Python 2 or 3 version of the package.
+    </para>
+   </listitem>
   </itemizedlist>
  </section>
 
diff --git a/nixos/modules/services/continuous-integration/buildbot/master.nix b/nixos/modules/services/continuous-integration/buildbot/master.nix
index 8d767de37f0..0f07e6133bb 100644
--- a/nixos/modules/services/continuous-integration/buildbot/master.nix
+++ b/nixos/modules/services/continuous-integration/buildbot/master.nix
@@ -6,8 +6,12 @@ with lib;
 
 let
   cfg = config.services.buildbot-master;
+
+  python = cfg.package.pythonModule;
+
   escapeStr = s: escape ["'"] s;
-  masterCfg = if cfg.masterCfg == null then pkgs.writeText "master.cfg" ''
+
+  defaultMasterCfg = pkgs.writeText "master.cfg" ''
     from buildbot.plugins import *
     factory = util.BuildFactory()
     c = BuildmasterConfig = dict(
@@ -27,8 +31,28 @@ let
       factory.addStep(step)
 
     ${cfg.extraConfig}
-  ''
-  else cfg.masterCfg;
+  '';
+
+  tacFile = pkgs.writeText "buildbot-master.tac" ''
+    import os
+
+    from twisted.application import service
+    from buildbot.master import BuildMaster
+
+    basedir = '${cfg.buildbotDir}'
+
+    configfile = '${cfg.masterCfg}'
+
+    # Default umask for server
+    umask = None
+
+    # note: this line is matched against to check that this is a buildmaster
+    # directory; do not edit it.
+    application = service.Application('buildmaster')
+
+    m = BuildMaster(basedir, configfile, umask)
+    m.setServiceParent(application)
+  '';
 
 in {
   options = {
@@ -66,9 +90,9 @@ in {
       };
 
       masterCfg = mkOption {
-        type = types.nullOr types.path;
+        type = types.path;
         description = "Optionally pass master.cfg path. Other options in this configuration will be ignored.";
-        default = null;
+        default = defaultMasterCfg;
         example = "/etc/nixos/buildbot/master.cfg";
       };
 
@@ -175,18 +199,25 @@ in {
 
       package = mkOption {
         type = types.package;
-        default = pkgs.buildbot-full;
-        defaultText = "pkgs.buildbot-full";
+        default = pkgs.pythonPackages.buildbot-full;
+        defaultText = "pkgs.pythonPackages.buildbot-full";
         description = "Package to use for buildbot.";
-        example = literalExample "pkgs.buildbot-full";
+        example = literalExample "pkgs.python3Packages.buildbot-full";
       };
 
       packages = mkOption {
-        default = with pkgs; [ python27Packages.twisted git ];
+        default = [ pkgs.git ];
         example = literalExample "[ pkgs.git ]";
         type = types.listOf types.package;
         description = "Packages to add to PATH for the buildbot process.";
       };
+
+      pythonPackages = mkOption {
+        default = pythonPackages: with pythonPackages; [ ];
+        defaultText = "pythonPackages: with pythonPackages; [ ]";
+        description = "Packages to add the to the PYTHONPATH of the buildbot process.";
+        example = literalExample "pythonPackages: with pythonPackages; [ requests ]";
+      };
     };
   };
 
@@ -210,14 +241,15 @@ in {
       description = "Buildbot Continuous Integration Server.";
       after = [ "network-online.target" ];
       wantedBy = [ "multi-user.target" ];
-      path = cfg.packages;
+      path = cfg.packages ++ cfg.pythonPackages python.pkgs;
+      environment.PYTHONPATH = "${python.withPackages (self: cfg.pythonPackages self ++ [ cfg.package ])}/${python.sitePackages}";
 
       preStart = ''
-        env > envvars
-        mkdir -vp ${cfg.buildbotDir}
-        ln -sfv ${masterCfg} ${cfg.buildbotDir}/master.cfg
-        rm -fv $cfg.buildbotDir}/buildbot.tac
-        ${cfg.package}/bin/buildbot create-master ${cfg.buildbotDir}
+        mkdir -vp "${cfg.buildbotDir}"
+        # Link the tac file so buildbot command line tools recognize the directory
+        ln -sf "${tacFile}" "${cfg.buildbotDir}/buildbot.tac"
+        ${cfg.package}/bin/buildbot create-master --db "${cfg.dbUrl}" "${cfg.buildbotDir}"
+        rm -f buildbot.tac.new master.cfg.sample
       '';
 
       serviceConfig = {
@@ -225,12 +257,11 @@ in {
         User = cfg.user;
         Group = cfg.group;
         WorkingDirectory = cfg.home;
-        ExecStart = "${cfg.package}/bin/buildbot start --nodaemon ${cfg.buildbotDir}";
+        # NOTE: call twistd directly with stdout logging for systemd
+        ExecStart = "${python.pkgs.twisted}/bin/twistd -o --nodaemon --pidfile= --logfile - --python ${tacFile}";
       };
-
     };
   };
 
   meta.maintainers = with lib.maintainers; [ nand0p mic92 ];
-
 }
diff --git a/nixos/modules/services/continuous-integration/buildbot/worker.nix b/nixos/modules/services/continuous-integration/buildbot/worker.nix
index 67c541570b9..4130ec918a7 100644
--- a/nixos/modules/services/continuous-integration/buildbot/worker.nix
+++ b/nixos/modules/services/continuous-integration/buildbot/worker.nix
@@ -7,6 +7,40 @@ with lib;
 let
   cfg = config.services.buildbot-worker;
 
+  python = cfg.package.pythonModule;
+
+  tacFile = pkgs.writeText "aur-buildbot-worker.tac" ''
+    import os
+    from io import open
+
+    from buildbot_worker.bot import Worker
+    from twisted.application import service
+
+    basedir = '${cfg.buildbotDir}'
+
+    # note: this line is matched against to check that this is a worker
+    # directory; do not edit it.
+    application = service.Application('buildbot-worker')
+
+    master_url_split = '${cfg.masterUrl}'.split(':')
+    buildmaster_host = master_url_split[0]
+    port = int(master_url_split[1])
+    workername = '${cfg.workerUser}'
+
+    with open('${cfg.workerPassFile}', 'r', encoding='utf-8') as passwd_file:
+        passwd = passwd_file.read().strip('\r\n')
+    keepalive = 600
+    umask = None
+    maxdelay = 300
+    numcpus = None
+    allow_shutdown = None
+
+    s = Worker(buildmaster_host, port, workername, passwd, basedir,
+               keepalive, umask=umask, maxdelay=maxdelay,
+               numcpus=numcpus, allow_shutdown=allow_shutdown)
+    s.setServiceParent(application)
+  '';
+
 in {
   options = {
     services.buildbot-worker = {
@@ -59,6 +93,23 @@ in {
         description = "Specifies the Buildbot Worker password.";
       };
 
+      workerPassFile = mkOption {
+        type = types.path;
+        description = "File used to store the Buildbot Worker password";
+      };
+
+      hostMessage = mkOption {
+        default = null;
+        type = types.nullOr types.str;
+        description = "Description of this worker";
+      };
+
+      adminMessage = mkOption {
+        default = null;
+        type = types.nullOr types.str;
+        description = "Name of the administrator of this worker";
+      };
+
       masterUrl = mkOption {
         default = "localhost:9989";
         type = types.str;
@@ -67,23 +118,24 @@ in {
 
       package = mkOption {
         type = types.package;
-        default = pkgs.buildbot-worker;
-        defaultText = "pkgs.buildbot-worker";
+        default = pkgs.pythonPackages.buildbot-worker;
+        defaultText = "pkgs.pythonPackages.buildbot-worker";
         description = "Package to use for buildbot worker.";
-        example = literalExample "pkgs.buildbot-worker";
+        example = literalExample "pkgs.python3Packages.buildbot-worker";
       };
 
       packages = mkOption {
-        default = with pkgs; [ python27Packages.twisted git ];
+        default = with pkgs; [ git ];
         example = literalExample "[ pkgs.git ]";
         type = types.listOf types.package;
         description = "Packages to add to PATH for the buildbot process.";
       };
-
     };
   };
 
   config = mkIf cfg.enable {
+    services.buildbot-worker.workerPassFile = mkDefault (pkgs.writeText "buildbot-worker-password" cfg.workerPass);
+
     users.groups = optional (cfg.group == "bbworker") {
       name = "bbworker";
     };
@@ -104,11 +156,16 @@ in {
       after = [ "network.target" "buildbot-master.service" ];
       wantedBy = [ "multi-user.target" ];
       path = cfg.packages;
+      environment.PYTHONPATH = "${python.withPackages (p: [ cfg.package ])}/${python.sitePackages}";
 
       preStart = ''
-        mkdir -vp ${cfg.buildbotDir}
-        rm -fv $cfg.buildbotDir}/buildbot.tac
-        ${cfg.package}/bin/buildbot-worker create-worker ${cfg.buildbotDir} ${cfg.masterUrl} ${cfg.workerUser} ${cfg.workerPass}
+        mkdir -vp "${cfg.buildbotDir}/info"
+        ${optionalString (cfg.hostMessage != null) ''
+          ln -sf "${pkgs.writeText "buildbot-worker-host" cfg.hostMessage}" "${cfg.buildbotDir}/info/host"
+        ''}
+        ${optionalString (cfg.adminMessage != null) ''
+          ln -sf "${pkgs.writeText "buildbot-worker-admin" cfg.adminMessage}" "${cfg.buildbotDir}/info/admin"
+        ''}
       '';
 
       serviceConfig = {
@@ -116,11 +173,9 @@ in {
         User = cfg.user;
         Group = cfg.group;
         WorkingDirectory = cfg.home;
-        Environment = "PYTHONPATH=${cfg.package}/lib/python2.7/site-packages:${pkgs.python27Packages.future}/lib/python2.7/site-packages";
 
         # NOTE: call twistd directly with stdout logging for systemd
-        #ExecStart = "${cfg.package}/bin/buildbot-worker start --nodaemon ${cfg.buildbotDir}";
-        ExecStart = "${pkgs.python27Packages.twisted}/bin/twistd -n -l - -y ${cfg.buildbotDir}/buildbot.tac";
+        ExecStart = "${python.pkgs.twisted}/bin/twistd --nodaemon --pidfile= --logfile - --python ${tacFile}";
       };
 
     };
diff --git a/nixos/release.nix b/nixos/release.nix
index 86383f502a7..5412080cca1 100644
--- a/nixos/release.nix
+++ b/nixos/release.nix
@@ -257,7 +257,7 @@ in rec {
   tests.boot = callSubTests tests/boot.nix {};
   tests.boot-stage1 = callTest tests/boot-stage1.nix {};
   tests.borgbackup = callTest tests/borgbackup.nix {};
-  tests.buildbot = callTest tests/buildbot.nix {};
+  tests.buildbot = callSubTests tests/buildbot.nix {};
   tests.cadvisor = callTestOnMatchingSystems ["x86_64-linux"] tests/cadvisor.nix {};
   tests.ceph = callTestOnMatchingSystems ["x86_64-linux"] tests/ceph.nix {};
   tests.certmgr = callSubTests tests/certmgr.nix {};
diff --git a/nixos/tests/buildbot.nix b/nixos/tests/buildbot.nix
index cf408dc7fec..399fd39005e 100644
--- a/nixos/tests/buildbot.nix
+++ b/nixos/tests/buildbot.nix
@@ -1,111 +1,117 @@
-# Test ensures buildbot master comes up correctly and workers can connect
-
-import ./make-test.nix ({ pkgs, ... } : {
-  name = "buildbot";
-
-  nodes = {
-    bbmaster = { pkgs, ... }: {
-      services.buildbot-master = {
-        enable = true;
-        package = pkgs.buildbot-full;
-
-        # NOTE: use fake repo due to no internet in hydra ci
-        factorySteps = [
-          "steps.Git(repourl='git://gitrepo/fakerepo.git', mode='incremental')"
-          "steps.ShellCommand(command=['bash', 'fakerepo.sh'])"
-        ];
-        changeSource = [
-          "changes.GitPoller('git://gitrepo/fakerepo.git', workdir='gitpoller-workdir', branch='master', pollinterval=300)"
-        ];
+{ system ? builtins.currentSystem }:
+
+with import ../lib/testing.nix { inherit system; };
+
+let
+  # Test ensures buildbot master comes up correctly and workers can connect
+  mkBuildbotTest = python: makeTest {
+    name = "buildbot";
+
+    nodes = {
+      bbmaster = { pkgs, ... }: {
+        services.buildbot-master = {
+          enable = true;
+          package = python.pkgs.buildbot-full;
+
+          # NOTE: use fake repo due to no internet in hydra ci
+          factorySteps = [
+            "steps.Git(repourl='git://gitrepo/fakerepo.git', mode='incremental')"
+            "steps.ShellCommand(command=['bash', 'fakerepo.sh'])"
+          ];
+          changeSource = [
+            "changes.GitPoller('git://gitrepo/fakerepo.git', workdir='gitpoller-workdir', branch='master', pollinterval=300)"
+          ];
+        };
+        networking.firewall.allowedTCPPorts = [ 8010 8011 9989 ];
+        environment.systemPackages = with pkgs; [ git python.pkgs.buildbot-full ];
       };
-      networking.firewall.allowedTCPPorts = [ 8010 8011 9989 ];
-      environment.systemPackages = with pkgs; [ git buildbot-full ];
-    };
 
-    bbworker = { pkgs, ... }: {
-      services.buildbot-worker = {
-        enable = true;
-        masterUrl = "bbmaster:9989";
+      bbworker = { pkgs, ... }: {
+        services.buildbot-worker = {
+          enable = true;
+          masterUrl = "bbmaster:9989";
+        };
+        environment.systemPackages = with pkgs; [ git python.pkgs.buildbot-worker ];
       };
-      environment.systemPackages = with pkgs; [ git buildbot-worker ];
-    };
 
-    gitrepo = { pkgs, ... }: {
-      services.openssh.enable = true;
-      networking.firewall.allowedTCPPorts = [ 22 9418 ];
-      environment.systemPackages = with pkgs; [ git ];
+      gitrepo = { pkgs, ... }: {
+        services.openssh.enable = true;
+        networking.firewall.allowedTCPPorts = [ 22 9418 ];
+        environment.systemPackages = with pkgs; [ git ];
+      };
     };
-  };
 
-  testScript = ''
-    #Start up and populate fake repo
-    $gitrepo->waitForUnit("multi-user.target");
-    print($gitrepo->execute(" \
-      git config --global user.name 'Nobody Fakeuser' && \
-      git config --global user.email 'nobody\@fakerepo.com' && \
-      rm -rvf /srv/repos/fakerepo.git /tmp/fakerepo && \
-      mkdir -pv /srv/repos/fakerepo ~/.ssh && \
-      ssh-keyscan -H gitrepo > ~/.ssh/known_hosts && \
-      cat ~/.ssh/known_hosts && \
-      cd /srv/repos/fakerepo && \
-      git init && \
-      echo -e '#!/bin/sh\necho fakerepo' > fakerepo.sh && \
-      cat fakerepo.sh && \
-      touch .git/git-daemon-export-ok && \
-      git add fakerepo.sh .git/git-daemon-export-ok && \
-      git commit -m fakerepo && \
-      git daemon --verbose --export-all --base-path=/srv/repos --reuseaddr & \
-    "));
-
-    # Test gitrepo
-    $bbmaster->waitForUnit("network-online.target");
-    #$bbmaster->execute("nc -z gitrepo 9418");
-    print($bbmaster->execute(" \
-      rm -rfv /tmp/fakerepo && \
-      git clone git://gitrepo/fakerepo /tmp/fakerepo && \
-      pwd && \
-      ls -la && \
-      ls -la /tmp/fakerepo \
-    "));
-
-    # Test start master and connect worker
-    $bbmaster->waitForUnit("buildbot-master.service");
-    $bbmaster->waitUntilSucceeds("curl -s --head http://bbmaster:8010") =~ /200 OK/;
-    $bbworker->waitForUnit("network-online.target");
-    $bbworker->execute("nc -z bbmaster 8010");
-    $bbworker->execute("nc -z bbmaster 9989");
-    $bbworker->waitForUnit("buildbot-worker.service");
-    print($bbworker->execute("ls -la /home/bbworker/worker"));
-
-
-    # Test stop buildbot master and worker
-    print($bbmaster->execute(" \
-      systemctl -l --no-pager status buildbot-master && \
-      systemctl stop buildbot-master \
-    "));
-    $bbworker->fail("nc -z bbmaster 8010");
-    $bbworker->fail("nc -z bbmaster 9989");
-    print($bbworker->execute(" \
-      systemctl -l --no-pager status buildbot-worker && \
-      systemctl stop buildbot-worker && \
-      ls -la /home/bbworker/worker \
-    "));
-
-
-    # Test buildbot daemon mode
-    # NOTE: daemon mode tests disabled due to broken PYTHONPATH child inheritence
-    #
-    #$bbmaster->execute("buildbot create-master /tmp");
-    #$bbmaster->execute("mv -fv /tmp/master.cfg.sample /tmp/master.cfg");
-    #$bbmaster->execute("sed -i 's/8010/8011/' /tmp/master.cfg");
-    #$bbmaster->execute("buildbot start /tmp");
-    #$bbworker->execute("nc -z bbmaster 8011");
-    #$bbworker->waitUntilSucceeds("curl -s --head http://bbmaster:8011") =~ /200 OK/;
-    #$bbmaster->execute("buildbot stop /tmp");
-    #$bbworker->fail("nc -z bbmaster 8011");
-
-  '';
-
-  meta.maintainers = with pkgs.stdenv.lib.maintainers; [ nand0p ];
-
-})
+    testScript = ''
+      #Start up and populate fake repo
+      $gitrepo->waitForUnit("multi-user.target");
+      print($gitrepo->execute(" \
+        git config --global user.name 'Nobody Fakeuser' && \
+        git config --global user.email 'nobody\@fakerepo.com' && \
+        rm -rvf /srv/repos/fakerepo.git /tmp/fakerepo && \
+        mkdir -pv /srv/repos/fakerepo ~/.ssh && \
+        ssh-keyscan -H gitrepo > ~/.ssh/known_hosts && \
+        cat ~/.ssh/known_hosts && \
+        cd /srv/repos/fakerepo && \
+        git init && \
+        echo -e '#!/bin/sh\necho fakerepo' > fakerepo.sh && \
+        cat fakerepo.sh && \
+        touch .git/git-daemon-export-ok && \
+        git add fakerepo.sh .git/git-daemon-export-ok && \
+        git commit -m fakerepo && \
+        git daemon --verbose --export-all --base-path=/srv/repos --reuseaddr & \
+      "));
+
+      # Test gitrepo
+      $bbmaster->waitForUnit("network-online.target");
+      #$bbmaster->execute("nc -z gitrepo 9418");
+      print($bbmaster->execute(" \
+        rm -rfv /tmp/fakerepo && \
+        git clone git://gitrepo/fakerepo /tmp/fakerepo && \
+        pwd && \
+        ls -la && \
+        ls -la /tmp/fakerepo \
+      "));
+
+      # Test start master and connect worker
+      $bbmaster->waitForUnit("buildbot-master.service");
+      $bbmaster->waitUntilSucceeds("curl -s --head http://bbmaster:8010") =~ /200 OK/;
+      $bbworker->waitForUnit("network-online.target");
+      $bbworker->execute("nc -z bbmaster 8010");
+      $bbworker->execute("nc -z bbmaster 9989");
+      $bbworker->waitForUnit("buildbot-worker.service");
+      print($bbworker->execute("ls -la /home/bbworker/worker"));
+
+
+      # Test stop buildbot master and worker
+      print($bbmaster->execute(" \
+        systemctl -l --no-pager status buildbot-master && \
+        systemctl stop buildbot-master \
+      "));
+      $bbworker->fail("nc -z bbmaster 8010");
+      $bbworker->fail("nc -z bbmaster 9989");
+      print($bbworker->execute(" \
+        systemctl -l --no-pager status buildbot-worker && \
+        systemctl stop buildbot-worker && \
+        ls -la /home/bbworker/worker \
+      "));
+
+
+      # Test buildbot daemon mode
+      $bbmaster->execute("buildbot create-master /tmp");
+      $bbmaster->execute("mv -fv /tmp/master.cfg.sample /tmp/master.cfg");
+      $bbmaster->execute("sed -i 's/8010/8011/' /tmp/master.cfg");
+      $bbmaster->execute("buildbot start /tmp");
+      $bbworker->execute("nc -z bbmaster 8011");
+      $bbworker->waitUntilSucceeds("curl -s --head http://bbmaster:8011") =~ /200 OK/;
+      $bbmaster->execute("buildbot stop /tmp");
+      $bbworker->fail("nc -z bbmaster 8011");
+
+    '';
+
+    meta.maintainers = with pkgs.stdenv.lib.maintainers; [ nand0p ];
+
+  };
+in {
+  python2 = mkBuildbotTest pkgs.python2;
+  python3 = mkBuildbotTest pkgs.python3;
+}
diff --git a/pkgs/development/tools/build-managers/buildbot/default.nix b/pkgs/development/python-modules/buildbot/default.nix
index 7e89fdc348e..0110aa92c06 100644
--- a/pkgs/development/tools/build-managers/buildbot/default.nix
+++ b/pkgs/development/python-modules/buildbot/default.nix
@@ -1,72 +1,68 @@
-{ stdenv, openssh, buildbot-worker, buildbot-pkg, pythonPackages, runCommand, makeWrapper }:
+{ stdenv, lib, buildPythonPackage, fetchPypi, makeWrapper, isPy3k,
+  python, twisted, jinja2, zope_interface, future, sqlalchemy,
+  sqlalchemy_migrate, dateutil, txaio, autobahn, pyjwt, treq, txrequests,
+  txgithub, pyjade, boto3, moto, mock, lz4, setuptoolsTrial, isort, pylint,
+  flake8, buildbot-worker, buildbot-pkg, glibcLocales }:
 
 let
-  withPlugins = plugins: runCommand "wrapped-${package.name}" {
-    buildInputs = [ makeWrapper ] ++ plugins;
-    propagatedBuildInputs = package.propagatedBuildInputs;
-    passthru.withPlugins = moarPlugins: withPlugins (moarPlugins ++ plugins);
-  } ''
-    makeWrapper ${package}/bin/buildbot $out/bin/buildbot \
-      --prefix PYTHONPATH : "${package}/lib/python2.7/site-packages:$PYTHONPATH"
-    ln -sfv ${package}/lib $out/lib
-  '';
+  withPlugins = plugins: buildPythonPackage {
+    name = "${package.name}-with-plugins";
+    phases = [ "installPhase" "fixupPhase" ];
+    buildInputs = [ makeWrapper ];
+    propagatedBuildInputs = plugins ++ package.propagatedBuildInputs;
 
-  package = pythonPackages.buildPythonApplication rec {
+    installPhase = ''
+      makeWrapper ${package}/bin/buildbot $out/bin/buildbot \
+        --prefix PYTHONPATH : "${package}/${python.sitePackages}:$PYTHONPATH"
+      ln -sfv ${package}/lib $out/lib
+    '';
+
+    passthru = package.passthru // {
+      withPlugins = morePlugins: withPlugins (morePlugins ++ plugins);
+    };
+  };
+
+  package = buildPythonPackage rec {
     pname = "buildbot";
     version = "1.4.0";
 
-    src = pythonPackages.fetchPypi {
+    src = fetchPypi {
       inherit pname version;
       sha256 = "0dfa926nh642i3bnpzlz0q347zicyx6wswjfqbniri59ya64fncx";
     };
 
-    buildInputs = with pythonPackages; [
-      lz4
-      txrequests
-      pyjade
-      boto3
-      moto
-      txgithub
-      mock
-      setuptoolsTrial
-      isort
-      pylint
-      astroid
-      pyflakes
-      openssh
-      buildbot-worker
-      buildbot-pkg
-      treq
-    ];
-
-    propagatedBuildInputs = with pythonPackages; [
+    propagatedBuildInputs = [
       # core
       twisted
       jinja2
       zope_interface
+      future
       sqlalchemy
       sqlalchemy_migrate
-      future
       dateutil
       txaio
       autobahn
       pyjwt
-      distro
 
       # tls
-      pyopenssl
-      service-identity
-      idna
-
-      # docs
-      sphinx
-      sphinxcontrib-blockdiag
-      sphinxcontrib-spelling
-      pyenchant
-      docutils
-      ramlfications
-      sphinx-jinja
+      twisted.extras.tls
+    ];
 
+    checkInputs = [
+      treq
+      txrequests
+      pyjade
+      boto3
+      moto
+      mock
+      lz4
+      setuptoolsTrial
+      isort
+      pylint
+      flake8
+      buildbot-worker
+      buildbot-pkg
+      glibcLocales
     ];
 
     patches = [
@@ -75,6 +71,8 @@ let
       ./skip_test_linux_distro.patch
     ];
 
+    LC_ALL = "en_US.UTF-8";
+
     # TimeoutErrors on slow machines -> aarch64
     doCheck = !stdenv.isAarch64;
 
@@ -82,9 +80,11 @@ let
       substituteInPlace buildbot/scripts/logwatcher.py --replace '/usr/bin/tail' "$(type -P tail)"
     '';
 
-    passthru = { inherit withPlugins; };
+    passthru = {
+      inherit withPlugins;
+    };
 
-    meta = with stdenv.lib; {
+    meta = with lib; {
       homepage = http://buildbot.net/;
       description = "Buildbot is an open-source continuous integration framework for automating software build, test, and release processes";
       maintainers = with maintainers; [ nand0p ryansydnor ];
diff --git a/pkgs/development/tools/build-managers/buildbot/pkg.nix b/pkgs/development/python-modules/buildbot/pkg.nix
index 452c2ffea04..b9358b1c420 100644
--- a/pkgs/development/tools/build-managers/buildbot/pkg.nix
+++ b/pkgs/development/python-modules/buildbot/pkg.nix
@@ -1,4 +1,4 @@
-{ stdenv, buildPythonPackage, fetchPypi, setuptools }:
+{ lib, buildPythonPackage, fetchPypi, setuptools }:
 
 buildPythonPackage rec {
   pname = "buildbot-pkg";
@@ -15,7 +15,7 @@ buildPythonPackage rec {
     substituteInPlace buildbot_pkg.py --replace "os.listdir = listdir" ""
   '';
 
-  meta = with stdenv.lib; {
+  meta = with lib; {
     homepage = http://buildbot.net/;
     description = "Buildbot Packaging Helper";
     maintainers = with maintainers; [ nand0p ryansydnor ];
diff --git a/pkgs/development/tools/build-managers/buildbot/plugins.nix b/pkgs/development/python-modules/buildbot/plugins.nix
index 91499b6e755..bdc67d178d4 100644
--- a/pkgs/development/tools/build-managers/buildbot/plugins.nix
+++ b/pkgs/development/python-modules/buildbot/plugins.nix
@@ -1,19 +1,19 @@
-{ stdenv, pythonPackages, buildbot-pkg }:
+{ lib, buildPythonPackage, fetchPypi, buildbot, buildbot-pkg }:
 
 {
-  www = pythonPackages.buildPythonPackage rec {
+  www = buildPythonPackage rec {
     pname = "buildbot_www";
-    version = buildbot-pkg.version;
+    inherit (buildbot-pkg) version;
 
     # NOTE: wheel is used due to buildbot circular dependency
     format = "wheel";
 
-    src = pythonPackages.fetchPypi {
+    src = fetchPypi {
       inherit pname version format;
       sha256 = "1m5dsp1gn9m5vfh5hnqp8g6hmhw1f1ydnassd33nhk521f2akz0v";
     };
 
-    meta = with stdenv.lib; {
+    meta = with lib; {
       homepage = http://buildbot.net/;
       description = "Buildbot UI";
       maintainers = with maintainers; [ nand0p ryansydnor ];
@@ -21,18 +21,19 @@
     };
   };
 
-  console-view = pythonPackages.buildPythonPackage rec {
+  console-view = buildPythonPackage rec {
     pname = "buildbot-console-view";
-    version = buildbot-pkg.version;
+    inherit (buildbot-pkg) version;
 
-    src = pythonPackages.fetchPypi {
+    src = fetchPypi {
       inherit pname version;
       sha256 = "0vblaxmihgb4w9aa5q0wcgvxs7qzajql8s22w0pl9qs494g05s9r";
     };
 
-    propagatedBuildInputs = with pythonPackages; [ buildbot-pkg ];
+    propagatedBuildInputs = [ buildbot-pkg ];
+    checkInputs = [ buildbot ];
 
-    meta = with stdenv.lib; {
+    meta = with lib; {
       homepage = http://buildbot.net/;
       description = "Buildbot Console View Plugin";
       maintainers = with maintainers; [ nand0p ryansydnor ];
@@ -40,18 +41,19 @@
     };
   };
 
-  waterfall-view = pythonPackages.buildPythonPackage rec {
+  waterfall-view = buildPythonPackage rec {
     pname = "buildbot-waterfall-view";
-    version = buildbot-pkg.version;
+    inherit (buildbot-pkg) version;
 
-    src = pythonPackages.fetchPypi {
+    src = fetchPypi {
       inherit pname version;
       sha256 = "18v1a6dapwjc2s9hi0cv3ry3s048w84md908zwaa3033gz3zwzy7";
     };
 
-    propagatedBuildInputs = with pythonPackages; [ buildbot-pkg ];
+    propagatedBuildInputs = [ buildbot-pkg ];
+    checkInputs = [ buildbot ];
 
-    meta = with stdenv.lib; {
+    meta = with lib; {
       homepage = http://buildbot.net/;
       description = "Buildbot Waterfall View Plugin";
       maintainers = with maintainers; [ nand0p ryansydnor ];
@@ -59,18 +61,19 @@
     };
   };
 
-  grid-view = pythonPackages.buildPythonPackage rec {
+  grid-view = buildPythonPackage rec {
     pname = "buildbot-grid-view";
-    version = buildbot-pkg.version;
+    inherit (buildbot-pkg) version;
 
-    src = pythonPackages.fetchPypi {
+    src = fetchPypi {
       inherit pname version;
       sha256 = "0iawsy892v6rn88hsgiiwaf689jqzhnb2wbxh6zkz3c0hvq4g0qd";
     };
 
-    propagatedBuildInputs = with pythonPackages; [ buildbot-pkg ];
+    propagatedBuildInputs = [ buildbot-pkg ];
+    checkInputs = [ buildbot ];
 
-    meta = with stdenv.lib; {
+    meta = with lib; {
       homepage = http://buildbot.net/;
       description = "Buildbot Grid View Plugin";
       maintainers = with maintainers; [ nand0p ];
@@ -78,23 +81,23 @@
     };
   };
 
-  wsgi-dashboards = pythonPackages.buildPythonPackage rec {
+  wsgi-dashboards = buildPythonPackage rec {
     pname = "buildbot-wsgi-dashboards";
-    version = buildbot-pkg.version;
+    inherit (buildbot-pkg) version;
 
-    src = pythonPackages.fetchPypi {
+    src = fetchPypi {
       inherit pname version;
       sha256 = "00cpjna3bffh1qbq6a3sqffd1g7qhbrmn9gpzxf9k38jam6jgfpz";
     };
 
-    propagatedBuildInputs = with pythonPackages; [ buildbot-pkg ];
+    propagatedBuildInputs = [ buildbot-pkg ];
+    checkInputs = [ buildbot ];
 
-    meta = with stdenv.lib; {
+    meta = with lib; {
       homepage = http://buildbot.net/;
       description = "Buildbot WSGI dashboards Plugin";
       maintainers = with maintainers; [ ];
       license = licenses.gpl2;
     };
   };
-
 }
diff --git a/pkgs/development/tools/build-managers/buildbot/skip_test_linux_distro.patch b/pkgs/development/python-modules/buildbot/skip_test_linux_distro.patch
index 8fe5c7b56b4..8fe5c7b56b4 100644
--- a/pkgs/development/tools/build-managers/buildbot/skip_test_linux_distro.patch
+++ b/pkgs/development/python-modules/buildbot/skip_test_linux_distro.patch
diff --git a/pkgs/development/tools/build-managers/buildbot/worker.nix b/pkgs/development/python-modules/buildbot/worker.nix
index 02beb686399..4e54276f8ae 100644
--- a/pkgs/development/tools/build-managers/buildbot/worker.nix
+++ b/pkgs/development/python-modules/buildbot/worker.nix
@@ -1,22 +1,23 @@
-{ stdenv, pythonPackages }:
+{ lib, buildPythonPackage, fetchPypi, python, setuptoolsTrial, mock, twisted, future }:
 
-pythonPackages.buildPythonApplication (rec {
+buildPythonPackage (rec {
   pname = "buildbot-worker";
   version = "1.4.0";
 
-  src = pythonPackages.fetchPypi {
+  src = fetchPypi {
     inherit pname version;
     sha256 = "12zvf4c39b6s4g1f2w407q8kkw602m88rc1ggi4w9pkw3bwbxrgy";
   };
 
-  buildInputs = with pythonPackages; [ setuptoolsTrial mock ];
-  propagatedBuildInputs = with pythonPackages; [ twisted future ];
+  propagatedBuildInputs = [ twisted future ];
+
+  checkInputs = [ setuptoolsTrial mock ];
 
   postPatch = ''
     substituteInPlace buildbot_worker/scripts/logwatcher.py --replace '/usr/bin/tail' "$(type -P tail)"
   '';
 
-  meta = with stdenv.lib; {
+  meta = with lib; {
     homepage = http://buildbot.net/;
     description = "Buildbot Worker Daemon";
     maintainers = with maintainers; [ nand0p ryansydnor ];
diff --git a/pkgs/top-level/aliases.nix b/pkgs/top-level/aliases.nix
index fe0d3816413..0abf0e60846 100644
--- a/pkgs/top-level/aliases.nix
+++ b/pkgs/top-level/aliases.nix
@@ -49,6 +49,11 @@ mapAliases ({
   bashCompletion = bash-completion; # Added 2016-09-28
   bridge_utils = bridge-utils;  # added 2015-02-20
   btrfsProgs = btrfs-progs; # added 2016-01-03
+  buildbot = pythonPackages.buildbot; # added 2018-10-11
+  buildbot-full = pythonPackages.buildbot-full; # added 2018-10-11
+  buildbot-pkg = pythonPackages.buildbot-pkg; # added 2018-10-11
+  buildbot-ui = pythonPackages.buildbot-ui; # added 2018-10-11
+  buildbot-worker = pythonPackages.buildbot-worker; # added 2018-10-11
   bundler_HEAD = bundler; # added 2015-11-15
   cantarell_fonts = cantarell-fonts; # added 2018-03-03
   checkbashism = checkbashisms; # added 2016-08-16
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index d911181ae37..b34501a8f69 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -8135,21 +8135,6 @@ with pkgs;
 
   buck = callPackage ../development/tools/build-managers/buck { };
 
-  buildbot = callPackage ../development/tools/build-managers/buildbot {
-    pythonPackages = python2Packages;
-  };
-  buildbot-worker = callPackage ../development/tools/build-managers/buildbot/worker.nix {
-    pythonPackages = python2Packages;
-  };
-  buildbot-pkg = callPackage ../development/tools/build-managers/buildbot/pkg.nix {
-    inherit (python2Packages) buildPythonPackage fetchPypi setuptools;
-  };
-  buildbot-plugins = callPackages ../development/tools/build-managers/buildbot/plugins.nix {
-    pythonPackages = python2Packages;
-  };
-  buildbot-ui = buildbot.withPlugins (with self.buildbot-plugins; [ www ]);
-  buildbot-full = buildbot.withPlugins (with self.buildbot-plugins; [ www console-view waterfall-view grid-view wsgi-dashboards ]);
-
   buildkite-agent = buildkite-agent2;
   buildkite-agent2 = callPackage ../development/tools/continuous-integration/buildkite-agent/2.x.nix { };
   buildkite-agent3 = callPackage ../development/tools/continuous-integration/buildkite-agent/3.x.nix { };
diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix
index 73d2cf6d2ac..3822f08bad7 100644
--- a/pkgs/top-level/python-packages.nix
+++ b/pkgs/top-level/python-packages.nix
@@ -1421,6 +1421,13 @@ in {
 
   bugzilla = callPackage ../development/python-modules/bugzilla { };
 
+  buildbot = callPackage ../development/python-modules/buildbot { };
+  buildbot-plugins = pkgs.recurseIntoAttrs (callPackage ../development/python-modules/buildbot/plugins.nix { });
+  buildbot-ui = self.buildbot.withPlugins (with self.buildbot-plugins; [ www ]);
+  buildbot-full = self.buildbot.withPlugins (with self.buildbot-plugins; [ www console-view waterfall-view grid-view wsgi-dashboards ]);
+  buildbot-worker = callPackage ../development/python-modules/buildbot/worker.nix { };
+  buildbot-pkg = callPackage ../development/python-modules/buildbot/pkg.nix { };
+
   check-manifest = callPackage ../development/python-modules/check-manifest { };
 
   devpi-common = callPackage ../development/python-modules/devpi-common { };