diff --git a/nix/master.nix b/nix/master.nix
index 3d37e48..d461208 100644
--- a/nix/master.nix
+++ b/nix/master.nix
@@ -178,7 +178,8 @@ in
         in
         "${if hasSSL then "https" else "http"}://${cfg.domain}/";
       dbUrl = config.services.buildbot-nix.master.dbUrl;
-      package = (pkgs.buildbot.overrideAttrs (old: {
+      # Can be dropped after we have 24.05 everywhere
+      package = lib.mkIf (lib.versionOlder pkgs.buildbot.version "3.10.0") (pkgs.buildbot.overrideAttrs (old: {
         patches = old.patches ++ [ ./0001-allow-secrets-to-be-group-readable.patch ];
       }));
       pythonPackages = ps: [