Merge pull request #15 from phaer/fixes

small fixes
This commit is contained in:
Jörg Thalheim 2023-10-27 11:12:47 +02:00 committed by GitHub
commit d16fbb0f71
Failed to generate hash of commit
3 changed files with 57 additions and 31 deletions

View file

@ -20,14 +20,16 @@ from buildbot.process.project import Project
from buildbot.process.properties import Interpolate, Properties
from buildbot.process.results import ALL_RESULTS, statusToString
from buildbot.steps.trigger import Trigger
from github_projects import ( # noqa: E402
from twisted.internet import defer, threads
from twisted.python.failure import Failure
from .github_projects import ( # noqa: E402
GithubProject,
create_project_hook,
load_projects,
refresh_projects,
slugify_project_name,
)
from twisted.internet import defer, threads
from twisted.python.failure import Failure
class BuildTrigger(Trigger):
@ -63,7 +65,7 @@ class BuildTrigger(Trigger):
"github.base.repo.full_name",
build_props.getProperty("github.repository.full_name"),
)
project_id = repo_name.replace("/", "-")
project_id = slugify_project_name(repo_name)
source = f"nix-eval-{project_id}"
sch = self.schedulerNames[0]
@ -151,7 +153,7 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
"github.base.repo.full_name",
build_props.getProperty("github.repository.full_name"),
)
project_id = repo_name.replace("/", "-")
project_id = slugify_project_name(repo_name)
scheduler = f"{project_id}-nix-build"
filtered_jobs = []
for job in jobs:
@ -310,7 +312,7 @@ def nix_update_flake_config(
"""
factory = util.BuildFactory()
url_with_secret = util.Interpolate(
f"https://git:%(secret:{github_token_secret})s@github.com/{project.name}"
f"https://git:%(secret:{github_token_secret})s@github.com/%(prop:project)s"
)
factory.addStep(
steps.Git(
@ -546,7 +548,7 @@ def read_secret_file(secret_name: str) -> str:
if directory is None:
print("directory not set", file=sys.stderr)
sys.exit(1)
return Path(directory).joinpath(secret_name).read_text()
return Path(directory).joinpath(secret_name).read_text().rstrip()
@dataclass
@ -614,7 +616,15 @@ def config_for_project(
),
# allow to manually trigger a nix-build
schedulers.ForceScheduler(
name=f"{project.id}-force", builderNames=[f"{project.name}/nix-eval"]
name=f"{project.id}-force",
builderNames=[f"{project.name}/nix-eval"],
properties=[
util.StringParameter(
name="project",
label="Name of the GitHub repository.",
default=project.name,
)
],
),
# allow to manually update flakes
schedulers.ForceScheduler(
@ -761,20 +771,6 @@ class NixConfigurator(ConfiguratorBase):
config["secretsProviders"] = config.get("secretsProviders", [])
config["secretsProviders"].append(systemd_secrets)
config["www"] = config.get("www", {})
config["www"]["avatar_methods"] = config["www"].get("avatar_methods", [])
config["www"]["avatar_methods"].append(util.AvatarGitHub())
config["www"]["auth"] = util.GitHubAuth(
self.github.oauth_id, read_secret_file(self.github.oauth_secret_name)
)
config["www"]["authz"] = util.Authz(
roleMatchers=[
util.RolesFromUsername(roles=["admin"], usernames=self.github.admins)
],
allowRules=[
util.AnyEndpointMatcher(role="admin", defaultDeny=False),
util.AnyControlEndpointMatcher(role="admins"),
],
)
config["www"]["change_hook_dialects"] = config["www"].get(
"change_hook_dialects", {}
)
@ -784,3 +780,21 @@ class NixConfigurator(ConfiguratorBase):
"token": self.github.token(),
"github_property_whitelist": "*",
}
if not config["www"].get("auth"):
config["www"]["avatar_methods"] = config["www"].get("avatar_methods", [])
config["www"]["avatar_methods"].append(util.AvatarGitHub())
config["www"]["auth"] = util.GitHubAuth(
self.github.oauth_id, read_secret_file(self.github.oauth_secret_name)
)
config["www"]["authz"] = util.Authz(
roleMatchers=[
util.RolesFromUsername(
roles=["admin"], usernames=self.github.admins
)
],
allowRules=[
util.AnyEndpointMatcher(role="admin", defaultDeny=False),
util.AnyControlEndpointMatcher(role="admins"),
],
)

View file

@ -69,6 +69,10 @@ def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]:
return items
def slugify_project_name(name: str) -> str:
return name.replace(".", "-").replace("/", "-")
class GithubProject:
def __init__(self, data: dict[str, Any]) -> None:
self.data = data
@ -91,8 +95,7 @@ class GithubProject:
@property
def id(self) -> str:
n = self.data["full_name"]
return n.replace("/", "-")
return slugify_project_name(self.data["full_name"])
@property
def default_branch(self) -> str:
@ -133,10 +136,20 @@ def create_project_hook(
def refresh_projects(github_token: str, repo_cache_file: Path) -> None:
repos = paginated_github_request(
repos = []
for repo in paginated_github_request(
"https://api.github.com/user/repos?per_page=100",
github_token,
)
):
if not repo["permissions"]["admin"]:
name = repo["full_name"]
log.msg(
f"skipping {name} because we do not have admin privileges, needed for hook management"
)
else:
repos.append(repo)
with NamedTemporaryFile("w", delete=False, dir=repo_cache_file.parent) as f:
try:
f.write(json.dumps(repos))
@ -148,9 +161,8 @@ def refresh_projects(github_token: str, repo_cache_file: Path) -> None:
def load_projects(github_token: str, repo_cache_file: Path) -> list[GithubProject]:
if repo_cache_file.exists():
log.msg("fetching github repositories from cache")
repos: list[dict[str, Any]] = json.loads(repo_cache_file.read_text())
else:
if not repo_cache_file.exists():
log.msg("fetching github repositories")
refresh_projects(github_token, repo_cache_file)
repos: list[dict[str, Any]] = json.loads(repo_cache_file.read_text())
return [GithubProject(repo) for repo in repos]

View file

@ -56,7 +56,7 @@ in
pkgs.nix-eval-jobs
];
environment.PYTHONPATH = "${python.withPackages (_: [cfg.package])}/${python.sitePackages}";
environment.MASTER_URL = ''tcp:host=localhost:port=9989'';
environment.MASTER_URL = cfg.masterUrl;
environment.BUILDBOT_DIR = buildbotDir;
serviceConfig = {