Merge pull request #15 from phaer/fixes

small fixes

buildbot_nix/__init__.py

@@ -20,14 +20,16 @@ from buildbot.process.project import Project
 from buildbot.process.properties import Interpolate, Properties
 from buildbot.process.results import ALL_RESULTS, statusToString
 from buildbot.steps.trigger import Trigger
-from github_projects import (  # noqa: E402
+from twisted.internet import defer, threads
+from twisted.python.failure import Failure
+
+from .github_projects import (  # noqa: E402
     GithubProject,
     create_project_hook,
     load_projects,
     refresh_projects,
+    slugify_project_name,
 )
-from twisted.internet import defer, threads
-from twisted.python.failure import Failure
 
 
 class BuildTrigger(Trigger):
@@ -63,7 +65,7 @@ class BuildTrigger(Trigger):
             "github.base.repo.full_name",
             build_props.getProperty("github.repository.full_name"),
         )
-        project_id = repo_name.replace("/", "-")
+        project_id = slugify_project_name(repo_name)
         source = f"nix-eval-{project_id}"
 
         sch = self.schedulerNames[0]
@@ -151,7 +153,7 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
                 "github.base.repo.full_name",
                 build_props.getProperty("github.repository.full_name"),
             )
-            project_id = repo_name.replace("/", "-")
+            project_id = slugify_project_name(repo_name)
             scheduler = f"{project_id}-nix-build"
             filtered_jobs = []
             for job in jobs:
@@ -310,7 +312,7 @@ def nix_update_flake_config(
     """
     factory = util.BuildFactory()
     url_with_secret = util.Interpolate(
-        f"https://git:%(secret:{github_token_secret})s@github.com/{project.name}"
+        f"https://git:%(secret:{github_token_secret})s@github.com/%(prop:project)s"
     )
     factory.addStep(
         steps.Git(
@@ -546,7 +548,7 @@ def read_secret_file(secret_name: str) -> str:
     if directory is None:
         print("directory not set", file=sys.stderr)
         sys.exit(1)
-    return Path(directory).joinpath(secret_name).read_text()
+    return Path(directory).joinpath(secret_name).read_text().rstrip()
 
 
 @dataclass
@@ -614,7 +616,15 @@ def config_for_project(
             ),
             # allow to manually trigger a nix-build
             schedulers.ForceScheduler(
-                name=f"{project.id}-force", builderNames=[f"{project.name}/nix-eval"]
+                name=f"{project.id}-force",
+                builderNames=[f"{project.name}/nix-eval"],
+                properties=[
+                    util.StringParameter(
+                        name="project",
+                        label="Name of the GitHub repository.",
+                        default=project.name,
+                    )
+                ],
             ),
             # allow to manually update flakes
             schedulers.ForceScheduler(
@@ -761,20 +771,6 @@ class NixConfigurator(ConfiguratorBase):
         config["secretsProviders"] = config.get("secretsProviders", [])
         config["secretsProviders"].append(systemd_secrets)
         config["www"] = config.get("www", {})
-        config["www"]["avatar_methods"] = config["www"].get("avatar_methods", [])
-        config["www"]["avatar_methods"].append(util.AvatarGitHub())
-        config["www"]["auth"] = util.GitHubAuth(
-            self.github.oauth_id, read_secret_file(self.github.oauth_secret_name)
-        )
-        config["www"]["authz"] = util.Authz(
-            roleMatchers=[
-                util.RolesFromUsername(roles=["admin"], usernames=self.github.admins)
-            ],
-            allowRules=[
-                util.AnyEndpointMatcher(role="admin", defaultDeny=False),
-                util.AnyControlEndpointMatcher(role="admins"),
-            ],
-        )
         config["www"]["change_hook_dialects"] = config["www"].get(
             "change_hook_dialects", {}
         )
@@ -784,3 +780,21 @@ class NixConfigurator(ConfiguratorBase):
             "token": self.github.token(),
             "github_property_whitelist": "*",
         }
+
+        if not config["www"].get("auth"):
+            config["www"]["avatar_methods"] = config["www"].get("avatar_methods", [])
+            config["www"]["avatar_methods"].append(util.AvatarGitHub())
+            config["www"]["auth"] = util.GitHubAuth(
+                self.github.oauth_id, read_secret_file(self.github.oauth_secret_name)
+            )
+            config["www"]["authz"] = util.Authz(
+                roleMatchers=[
+                    util.RolesFromUsername(
+                        roles=["admin"], usernames=self.github.admins
+                    )
+                ],
+                allowRules=[
+                    util.AnyEndpointMatcher(role="admin", defaultDeny=False),
+                    util.AnyControlEndpointMatcher(role="admins"),
+                ],
+            )

buildbot_nix/github_projects.py

@@ -69,6 +69,10 @@ def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]:
     return items
 
 
+def slugify_project_name(name: str) -> str:
+    return name.replace(".", "-").replace("/", "-")
+
+
 class GithubProject:
     def __init__(self, data: dict[str, Any]) -> None:
         self.data = data
@@ -91,8 +95,7 @@ class GithubProject:
 
     @property
     def id(self) -> str:
-        n = self.data["full_name"]
+        return slugify_project_name(self.data["full_name"])
-        return n.replace("/", "-")
 
     @property
     def default_branch(self) -> str:
@@ -133,10 +136,20 @@ def create_project_hook(
 
 
 def refresh_projects(github_token: str, repo_cache_file: Path) -> None:
-    repos = paginated_github_request(
+    repos = []
+
+    for repo in paginated_github_request(
         "https://api.github.com/user/repos?per_page=100",
         github_token,
-    )
+    ):
+        if not repo["permissions"]["admin"]:
+            name = repo["full_name"]
+            log.msg(
+                f"skipping {name} because we do not have admin privileges, needed for hook management"
+            )
+        else:
+            repos.append(repo)
+
     with NamedTemporaryFile("w", delete=False, dir=repo_cache_file.parent) as f:
         try:
             f.write(json.dumps(repos))
@@ -148,9 +161,8 @@ def refresh_projects(github_token: str, repo_cache_file: Path) -> None:
 
 
 def load_projects(github_token: str, repo_cache_file: Path) -> list[GithubProject]:
-    if repo_cache_file.exists():
+    if not repo_cache_file.exists():
-        log.msg("fetching github repositories from cache")
+        log.msg("fetching github repositories")
-        repos: list[dict[str, Any]] = json.loads(repo_cache_file.read_text())
-    else:
         refresh_projects(github_token, repo_cache_file)
+    repos: list[dict[str, Any]] = json.loads(repo_cache_file.read_text())
     return [GithubProject(repo) for repo in repos]

nix/worker.nix

@@ -56,7 +56,7 @@ in
         pkgs.nix-eval-jobs
       ];
       environment.PYTHONPATH = "${python.withPackages (_: [cfg.package])}/${python.sitePackages}";
-      environment.MASTER_URL = ''tcp:host=localhost:port=9989'';
+      environment.MASTER_URL = cfg.masterUrl;
       environment.BUILDBOT_DIR = buildbotDir;
 
       serviceConfig = {