From 046a13aceb3dbe5daf6d7fd00f4d6adba5a58177 Mon Sep 17 00:00:00 2001 From: Pavel Tsakalidis Date: Tue, 2 Jun 2026 20:46:00 +0100 Subject: [PATCH 1/6] Implement org/repo secret/variable support --- butler.py | 4 +- src/commands/download/download.py | 38 ----- src/commands/download/download_helper.py | 39 +++++ src/commands/secvars/command.py | 51 ++++++ src/commands/secvars/secvars.py | 188 +++++++++++++++++++++++ src/database/database.py | 10 +- src/database/helpers/db_repos.py | 3 + src/database/helpers/db_secvars.py | 37 +++++ src/database/models.py | 20 +++ src/github/client.py | 68 ++++++++ src/libs/components/secvar.py | 90 +++++++++++ src/libs/constants.py | 21 ++- src/tests/conftest.py | 2 + 13 files changed, 529 insertions(+), 42 deletions(-) create mode 100644 src/commands/secvars/command.py create mode 100644 src/commands/secvars/secvars.py create mode 100644 src/database/helpers/db_secvars.py create mode 100644 src/libs/components/secvar.py diff --git a/butler.py b/butler.py index 0713075..5a0728f 100644 --- a/butler.py +++ b/butler.py @@ -2,17 +2,19 @@ from src.commands.database.command import CommandDatabase from src.commands.process.command import CommandProcess from src.commands.report.command import CommandReport +from src.commands.secvars.command import CommandSecretsAndVariables from src.libs.utils import Utils from src.libs.exceptions import InvalidCommandLine from src.commands.download.command import CommandDownload -__VERSION__ = '0.10.2 Beta' +__VERSION__ = '0.12.0' commands = { 'download': CommandDownload, 'database': CommandDatabase, 'process': CommandProcess, 'report': CommandReport, + 'secvars': CommandSecretsAndVariables } parser = argparse.ArgumentParser(prog="butler", description=f"Butler - GitHub Actions Oversight v{__VERSION__}") diff --git a/src/commands/download/download.py b/src/commands/download/download.py index 522e1a4..d6b3d61 100644 --- a/src/commands/download/download.py +++ b/src/commands/download/download.py @@ -160,44 +160,6 @@ def _collect_repos(self, repos: list[RepoComponent]) -> None: self.log.trace(f"Writing to database...") self.database.commit() - def _save_repo(self, repo: RepoComponent) -> None: - with self.lock: - self.log.info(f"Saving repository {repo}") - if repo.org.id == 0: - repo.org.id = self._create_org(repo.org).id - - if len(repo.ref) > 0: - repo.poll_status = PollStatus.PENDING - repo_db = self.database.repos().create(repo) - return - - # At this point, there is no `ref` in the object. - # Search if the repo is already in the database. - repo_db = self.database.repos().find(repo.org.id, repo.name, None) - if self._repo_already_stored(repo_db): - return - - # Either there's no database record, or the stored one also has an empty ref. - fresh_repo = self._fetch_repo(repo) - with self.lock: - if fresh_repo.org.name.lower() == repo.org.name.lower() and fresh_repo.name.lower() == repo.name.lower(): - fresh_repo.org.id = repo.org.id - fresh_repo.poll_status = PollStatus.SCANNED if repo.status == RepoStatus.MISSING else PollStatus.PENDING - if repo_db: - fresh_repo.id = repo_db.id - self.database.repos().update(fresh_repo) - else: - self.database.repos().create(fresh_repo) - return - - # Here, the fetched repo is different to the one passed to the function, this happens when a repo is redirected. - fresh_repo.org.id = self._create_org(fresh_repo.org).id - fresh_repo_db = self.database.repos().create(fresh_repo) - - repo.redirect_id = fresh_repo_db.id - repo.status = RepoStatus.REDIRECT - self.database.repos().create(repo) - def _resolve_commits(self) -> None: while True: batch = self.database.next_commit_to_resolve(self.threads) diff --git a/src/commands/download/download_helper.py b/src/commands/download/download_helper.py index 8f1957f..38ec618 100644 --- a/src/commands/download/download_helper.py +++ b/src/commands/download/download_helper.py @@ -1,3 +1,4 @@ +from src.libs.constants import PollStatus from src.database.models import OrganisationModel, RepositoryModel from src.github.exceptions import HttpNotFound from src.libs.components.org import OrgComponent @@ -167,3 +168,41 @@ def _create_child_workflow_from_workflow(self, uses: str, workflow_instance: Wor else: org, repo, workflow = self._create_child_workflow_from_action(uses) return org, repo, workflow + + def _save_repo(self, repo: RepoComponent) -> None: + with self.lock: + self.log.info(f"Saving repository {repo}") + if repo.org.id == 0: + repo.org.id = self._create_org(repo.org).id + + if len(repo.ref) > 0: + repo.poll_status = PollStatus.PENDING + repo_db = self.database.repos().create(repo) + return + + # At this point, there is no `ref` in the object. + # Search if the repo is already in the database. + repo_db = self.database.repos().find(repo.org.id, repo.name, None) + if self._repo_already_stored(repo_db): + return + + # Either there's no database record, or the stored one also has an empty ref. + fresh_repo = self._fetch_repo(repo) + with self.lock: + if fresh_repo.org.name.lower() == repo.org.name.lower() and fresh_repo.name.lower() == repo.name.lower(): + fresh_repo.org.id = repo.org.id + fresh_repo.poll_status = PollStatus.SCANNED if repo.status == RepoStatus.MISSING else PollStatus.PENDING + if repo_db: + fresh_repo.id = repo_db.id + self.database.repos().update(fresh_repo) + else: + self.database.repos().create(fresh_repo) + return + + # Here, the fetched repo is different to the one passed to the function, this happens when a repo is redirected. + fresh_repo.org.id = self._create_org(fresh_repo.org).id + fresh_repo_db = self.database.repos().create(fresh_repo) + + repo.redirect_id = fresh_repo_db.id + repo.status = RepoStatus.REDIRECT + self.database.repos().create(repo) diff --git a/src/commands/secvars/command.py b/src/commands/secvars/command.py new file mode 100644 index 0000000..b58d669 --- /dev/null +++ b/src/commands/secvars/command.py @@ -0,0 +1,51 @@ +import os +import argparse +from src.commands.secvars.secvars import ServiceSecretsAndVariables +from src.commands.command import Command +from src.database.database import Database +from src.libs.exceptions import InvalidCommandLine +from src.github.client import GitHubClient + + +class CommandSecretsAndVariables(Command): + @staticmethod + def load_command_line(subparsers: any) -> None: + subparser = subparsers.add_parser("secvars", help="Download Secrets and Variables from GitHub") + + subparser.add_argument("--org", type=str, help="Organisation to download secrets and variables for") + subparser.add_argument("--database", default="database.db", type=str, help="Path to SQLite database to create or connect to") + subparser.add_argument("--resume-next", default=True, action="store_true", help="Resume downloads on server errors") + subparser.add_argument("--threads", default=1, type=int, help="Enable multithreading") + + Command.define_shared_arguments(subparser) + + def load_arguments(self, arguments: argparse.Namespace) -> dict: + return { + 'org': '' if arguments.org is None or len(arguments.org.strip()) == 0 else arguments.org.strip(), + 'database': '' if arguments.database is None or len(arguments.database.strip()) == 0 else os.path.realpath(arguments.database.strip()), + 'resume_next': arguments.resume_next or False, + 'threads': int(arguments.threads), + } + + def validate_command_arguments(self, arguments: dict) -> None: + # Validate database. + if len(arguments['database']) == 0: + raise InvalidCommandLine(f"--database cannot be empty") + elif not arguments['database'].lower().endswith('.sqlite3') and not arguments['database'].lower().endswith('.db'): + raise InvalidCommandLine(f"--database {arguments['database']} is not a SQLite database (must end with .sqlite3 or .db)") + + if len(arguments['org']) == 0: + raise InvalidCommandLine(f"--org cannot be empty") + + if arguments['threads'] <= 0: + arguments['threads'] = 1 + + def execute(self, arguments: dict) -> bool: + database = Database(arguments['database'], arguments['db_debug'], arguments['db_debug_auto_commit']) + + service = ServiceSecretsAndVariables(self.log, database) + service.github_client = GitHubClient(self.tokens, self.log) + service.org = arguments['org'] + service.resume_next = arguments['resume_next'] + service.threads = arguments['threads'] + return service.run() diff --git a/src/commands/secvars/secvars.py b/src/commands/secvars/secvars.py new file mode 100644 index 0000000..28cdd8f --- /dev/null +++ b/src/commands/secvars/secvars.py @@ -0,0 +1,188 @@ +import threading +import concurrent.futures +from contextlib import nullcontext +from src.database.models import OrganisationModel, RepositoryModel +from src.libs.constants import SecretVariableCategory, SecretVariableType, SecretVariableVisibility +from src.libs.components.secvar import SecretVariableComponent +from src.commands.download.download_helper import DownloadHelper +from src.commands.service import Service +from src.github.client import GitHubClient +from src.github.exceptions import TooManyRequests, ApiRateLimitExceeded, OrgNotFound +from src.libs.components.org import OrgComponent +from src.libs.constants import PollStatus, OrgStatus +from src.libs.exceptions import InvalidCommandLine +from src.libs.utils import Utils + + +class ServiceSecretsAndVariables(Service, DownloadHelper): + org: str = None + lock: threading.Lock = None + resume_next: bool = None + github_client: GitHubClient = None + threads: int = None + + _combinations: list = [ + {'label': 'actions / secrets', 'category': SecretVariableCategory.ACTIONS, 'type': SecretVariableType.SECRET}, + {'label': 'actions / variables', 'category': SecretVariableCategory.ACTIONS, 'type': SecretVariableType.VARIABLE}, + {'label': 'agents / secrets', 'category': SecretVariableCategory.AGENTS, 'type': SecretVariableType.SECRET}, + {'label': 'agents / variables', 'category': SecretVariableCategory.AGENTS, 'type': SecretVariableType.VARIABLE}, + {'label': 'codespaces / secrets', 'category': SecretVariableCategory.CODESPACES, 'type': SecretVariableType.SECRET}, + {'label': 'dependabot / secrets', 'category': SecretVariableCategory.DEPENDABOT, 'type': SecretVariableType.SECRET}, + ] + + def run(self) -> bool: + # Thanks SQLite3 :> + self.lock = threading.Lock() if self.threads > 1 else nullcontext() + + while True: + try: + self.log.info(f"Collecting repositories for {self.org}...") + self._collect_targets() + + org = self.database.orgs().find(self.org) + if not org: + raise InvalidCommandLine(f"Organisation {self.org} not found") + + self.log.info(f"Collecting secrets and variables for {org.name}") + self._collect_secrets_and_variables(org) + + break + except (TooManyRequests, ApiRateLimitExceeded) as e: + if self.resume_next: + self.github_client.halt_and_continue(5) + continue + raise + except Exception as e: + if self.resume_next and 'Server Error' in str(e): + self.github_client.halt_and_continue(2) + continue + raise + + if self.database.debug: + self.log.info(f"Total SQL Queries: {self.database.total_queries}") + + self.log.info(f"Total API Calls: {self.github_client._api.total_requests}") + return True + + def _collect_targets(self) -> None: + orgs, repos = Utils.filter_orgs_and_repos([self.org]) + self.log.debug(f"Input has {len(orgs)} organisations and {len(repos)} repositories") + self._collect_orgs(orgs) + + def _collect_orgs(self, orgs: list[OrgComponent]) -> None: + count = 0 + for org in orgs: + count += 1 + self.log.info(f"Processing {org} ({count}/{len(orgs)})") + + org_db = self._create_org(org) + org.id = org_db.id + + if org_db.poll_status == PollStatus.SCANNED: + self.log.debug(f"Organisation {org_db.name} already scanned - skipping") + continue + elif org_db.poll_status == PollStatus.NONE: + self.log.info(f"Organisation {org_db.name} is new or was not marked as pending before") + self.database.orgs().set_poll_status(org_db.id, PollStatus.PENDING) + + try: + for batch in self.github_client.get_org_repos(org.name, True, True): + for repo in batch: + repo.org.id = org_db.id + self._save_repo(repo) + except OrgNotFound as e: + self.log.error(f"Organisation {org.name} not found") + self.database.orgs().set_status(org_db.id, OrgStatus.MISSING) + + self.database.orgs().set_poll_status(org_db.id, PollStatus.SCANNED) + + self.log.trace(f"Writing to database...") + self.database.commit() + + self.log.trace(f"Writing to database...") + self.database.commit() + + def _collect_secrets_and_variables(self, org: OrganisationModel) -> None: + components = [] + for item in self._combinations: + self.log.info(f"Getting organisation {item['label']}") + try: + results = self.github_client.get_secrets(org.name, item['category'], item['type'], None, None) + components.extend(self._create_components(org.id, results, item['category'], item['type'])) + except Exception as e: + self.log.warning(f"Could not get {item['label']}") + + self.log.info(f"Writing to database") + for component in components: + self.database.secvars().create(0, component) + + self.database.commit() + + self.log.info(f"Getting organisation repos") + repos = self.database.repos().all(org.id) + self.log.info(f"Got {len(repos)} repos") + + with concurrent.futures.ThreadPoolExecutor(max_workers=self.threads) as executor: + # Submit all repositories to the executor + future_to_repo = {executor.submit(self._fetch_secvar_repo_single, org, repo): repo for repo in repos} + + for future in concurrent.futures.as_completed(future_to_repo): + repo = future_to_repo[future] + try: + # Retrieve the data returned by fetch_repo_data + returned_repo, components = future.result() + + # 3. Write to the database safely in the main thread + if components: + self.log.info(f"Writing {len(components)} components to database for {returned_repo.name}") + with self.lock: + for component in components: + self.database.secvars().create(repo.id, component) + + self.database.commit() + + except Exception as e: + self.log.error(f"Unhandled exception processing repo {repo.name}: {e}") + + def _fetch_secvar_repo_single(self, org: OrganisationModel, repo: RepositoryModel) -> tuple[object, list]: + components = [] + self.log.info(f"Getting secrets and variables for {repo.name}") + for item in self._combinations: + self.log.info(f"Getting data for repo {repo.name} and {item['label']}") + try: + results = self.github_client.get_secrets(org.name, item['category'], item['type'], None, repo.name) + components.extend(self._create_components(org.id, results, item['category'], item['type'])) + except Exception as e: + self.log.warning(f"Could not get {item['label']} for {repo.name}") + + return repo, components + + def _create_components(self, org_id: int, items: list, category: SecretVariableCategory, type: SecretVariableType) -> list[SecretVariableComponent]: + components = [] + for item in items: + component = SecretVariableComponent() + component.category = category + component.type = type + component.name = item['name'] + if component.type == SecretVariableType.VARIABLE: + component.value = item['value'] + component.created_at = item['created_at'] + component.updated_at = item['updated_at'] + + if 'visibility' in item: + match item['visibility']: + case 'all': + component.visibility = SecretVariableVisibility.ALL + case 'private': + component.visibility = SecretVariableVisibility.PRIVATE + case 'selected': + component.visibility = SecretVariableVisibility.SELECTED + + repos = [] + for repo in item['repos']: + repo = self.database.repos().find(org_id, repo, None) + repos.append(repo) + component.repos = repos + + components.append(component) + return components diff --git a/src/database/database.py b/src/database/database.py index 573a15a..801459e 100644 --- a/src/database/database.py +++ b/src/database/database.py @@ -1,4 +1,5 @@ import os +from src.database.helpers.db_secvars import DBSecretsAndVariables from src.database.database_helper import DatabaseHelper from src.database.helpers.db_config import DBConfig from src.database.helpers.db_jobs import DBJob @@ -14,7 +15,7 @@ class Database(DatabaseHelper): - __VERSION__: str = '1.0.1' + __VERSION__: str = '1.1.0' _engine: Engine = None _sessionmaker: sessionmaker = None _session = None @@ -26,6 +27,7 @@ class Database(DatabaseHelper): _steps: DBStep | None = None _vars: DBVars | None = None _config: DBConfig | None = None + _secvars: DBSecretsAndVariables | None = None _total_queries: int = 0 _debug: bool = False @@ -98,7 +100,6 @@ def _update_views(self) -> None: """ self.execute(sql) - def orgs(self) -> DBOrg: if not self._orgs: self._orgs = DBOrg(self.session, self.auto_commit) @@ -134,6 +135,11 @@ def config(self) -> DBConfig: self._config = DBConfig(self.session, self.auto_commit) return self._config + def secvars(self) -> DBSecretsAndVariables: + if not self._secvars: + self._secvars = DBSecretsAndVariables(self.session, self.auto_commit) + return self._secvars + def commit(self) -> None: self.session.commit() diff --git a/src/database/helpers/db_repos.py b/src/database/helpers/db_repos.py index 31722ed..53d29d6 100644 --- a/src/database/helpers/db_repos.py +++ b/src/database/helpers/db_repos.py @@ -97,3 +97,6 @@ def set_ref_resolved_fields(self, id: int, resolved_ref: str, resolved_ref_type: def count(self) -> int: return self.session.query(RepositoryModel).count() + + def all(self, org_id: int) -> any: + return self.session.query(RepositoryModel).filter(RepositoryModel.org_id == org_id).all() diff --git a/src/database/helpers/db_secvars.py b/src/database/helpers/db_secvars.py new file mode 100644 index 0000000..4fc3576 --- /dev/null +++ b/src/database/helpers/db_secvars.py @@ -0,0 +1,37 @@ +from sqlalchemy import func +from src.libs.constants import SecretVariableVisibility +from src.libs.components.secvar import SecretVariableComponent +from src.database.helpers.db_base import DBBase +from src.database.models import SecretsVariablesModel, SecretsVariablesReposModel + + +class DBSecretsAndVariables(DBBase): + def find(self, repo_id: int, category: int, type: int, name: str) -> SecretsVariablesModel | None: + return self.session.query(SecretsVariablesModel).filter( + SecretsVariablesModel.repo_id == repo_id, + SecretsVariablesModel.category == category, + SecretsVariablesModel.type == type, + func.lower(SecretsVariablesModel.name) == func.lower(name) + ).first() + + def create(self, repo_id: int, secvar: SecretVariableComponent) -> SecretsVariablesModel: + record = self.find(repo_id, secvar.category, secvar.type, secvar.name) + if not record: + record = SecretsVariablesModel(repo_id=repo_id) + record.category = secvar.category + record.type = secvar.type + record.name = secvar.name + record.value = secvar.value + record.visibility = secvar.visibility + record.created_at = secvar.created_at + record.updated_at = secvar.updated_at + self.add(record) + self.save() + + if secvar.visibility == SecretVariableVisibility.SELECTED: + for repo in secvar.repos: + repo_record = SecretsVariablesReposModel(org_secret_variable_id=record.id, repo_id=repo.id) + self.add(repo_record) + self.save() + + return record diff --git a/src/database/models.py b/src/database/models.py index a0afcc4..e630ed7 100644 --- a/src/database/models.py +++ b/src/database/models.py @@ -119,3 +119,23 @@ class ConfigModel(Base): name = Column(String, primary_key=True, index=True) value = Column(String, default='', index=True) + +class SecretsVariablesModel(Base): + __tablename__ = 'secrets_and_variables' + + id = Column(Integer, primary_key=True, autoincrement=True) + repo_id = Column(Integer, default=0, index=True) + category = Column(Integer, default=0, index=True) + type = Column(Integer, default=0, index=True) + name = Column(String, default='', index=True) + value = Column(String, default='', index=True) + visibility = Column(Integer, default=0, index=True) + created_at = Column(Text, default='', index=True) + updated_at = Column(Text, default='', index=True) + +class SecretsVariablesReposModel(Base): + __tablename__ = 'secrets_and_variables_repos' + + id = Column(Integer, primary_key=True, autoincrement=True) + org_secret_variable_id = Column(Integer, default=0, index=True) + repo_id = Column(Integer, default=0, index=True) diff --git a/src/github/client.py b/src/github/client.py index 2b2a8d0..9abb1d8 100644 --- a/src/github/client.py +++ b/src/github/client.py @@ -2,6 +2,7 @@ import base64 import time from loguru import logger +from src.libs.constants import SecretVariableCategory, SecretVariableType from src.github.api import GitHubApi from src.github.exceptions import ( AccountNotFound, UnknownAccountType, HttpNotFound, GitHubException, HttpEmptyRepo, HttpAccessBlocked, @@ -357,3 +358,70 @@ def halt_and_continue(self, minutes: int) -> None: self.log.info(f"Reached API rate limit - waiting {minutes} minutes") time.sleep(minutes * 60) self._api.refresh_tokens() + + def get_secrets(self, org: str, category: SecretVariableCategory, type: SecretVariableType, environment: str | None, repo: str | None) -> list[dict]: + params = {'page': 1, 'per_page': 100} + url = self._get_secrets_url(org, category, type, environment, repo) + all_results = [] + while True: + response_headers = {} + try: + results = self._api.get(url, params, None, response_headers) + except HttpNotFound: + raise RepoNotFound(f"{org}/{category}/{type}/{environment}/{repo}") + + name = 'secrets' if 'secrets' in results else 'variables' + for item in results[name]: + if item.get('visibility', '') == 'selected': + item['repos'] = self._get_selected_repositories(item['selected_repositories_url']) + all_results.append(item) + + # Invalidate as we'll be using the 'next url' from the headers. + params = None + url = self._get_next_url(response_headers) + if not url: + break + + return all_results + + def _get_selected_repositories(self, url: str) -> list: + url = url.replace('https://api.github.com', '') + try: + results = self._api.get(url, {'per_page': 999}) + except HttpNotFound: + raise RepoNotFound(url) + + repos = [] + for result in results['repositories']: + repos.append(result['name']) + return repos + + def _get_secrets_url(self, org: str, category: SecretVariableCategory, type: SecretVariableType, environment: str | None, repo: str | None) -> str: + category_mapping = { + # /orgs/{org}/actions/{secrets,variables} + # /repos/{owner}/{repo}/actions/{secrets,variables} + SecretVariableCategory.ACTIONS: 'actions', + + # /orgs/{org}/agents/{secrets,variables} + SecretVariableCategory.AGENTS: 'agents', + # /repos/{owner}/{repo}/codespaces/secrets + SecretVariableCategory.CODESPACES: 'codespaces', + # /orgs/{org}/dependabot/secrets + SecretVariableCategory.DEPENDABOT: 'dependabot', + # /repos/{owner}/{repo}/environments/{environment_name}/secrets + SecretVariableCategory.ENVIRONMENTS: 'environments', + } + + type_mapping = { + SecretVariableType.SECRET: 'secrets', + SecretVariableType.VARIABLE: 'variables', + } + + base_path = f"/repos/{org}/{repo}" if repo else f"/orgs/{org}" + components = [ + base_path, + category_mapping[category], + environment, + type_mapping[type] + ] + return "/".join(comp for comp in components if comp) diff --git a/src/libs/components/secvar.py b/src/libs/components/secvar.py new file mode 100644 index 0000000..45d5766 --- /dev/null +++ b/src/libs/components/secvar.py @@ -0,0 +1,90 @@ +from datetime import datetime +from src.libs.constants import SecretVariableCategory, SecretVariableType, SecretVariableVisibility + + +class SecretVariableComponent: + _id: int = None + _repos: list = None + _category: SecretVariableCategory = None + _type: SecretVariableType = None + _name: str = None + _value: str = None + _visibility: SecretVariableVisibility = None + _created_at: datetime = None + _updated_at: datetime = None + + @property + def id(self) -> int: + return self._id or 0 + + @id.setter + def id(self, value: int): + self._id = value + + @property + def repos(self) -> list: + return self._repos + + @repos.setter + def repos(self, value: list): + self._repos = value + + @property + def category(self) -> SecretVariableCategory: + return self._category + + @category.setter + def category(self, value: SecretVariableCategory): + self._category = value + + @property + def type(self) -> SecretVariableType: + return self._type + + @type.setter + def type(self, value: SecretVariableType): + self._type = value + + @property + def name(self) -> str: + return self._name + + @name.setter + def name(self, value: str): + self._name = value + + @property + def value(self) -> str: + return self._value or '' + + @value.setter + def value(self, value: str): + self._value = value + + @property + def visibility(self) -> SecretVariableVisibility: + return self._visibility + + @visibility.setter + def visibility(self, value: SecretVariableVisibility): + self._visibility = value + + @property + def created_at(self) -> datetime: + return self._created_at + + @created_at.setter + def created_at(self, value: datetime | str): + if isinstance(value, str): + value = datetime.fromisoformat(value) + self._created_at = value + + @property + def updated_at(self) -> datetime: + return self._updated_at + + @updated_at.setter + def updated_at(self, value: datetime | str): + if isinstance(value, str): + value = datetime.fromisoformat(value) + self._updated_at = value diff --git a/src/libs/constants.py b/src/libs/constants.py index 6061e73..15fe060 100644 --- a/src/libs/constants.py +++ b/src/libs/constants.py @@ -60,4 +60,23 @@ class VariableMappingType(IntEnum): class VariableMappingGroupType(IntEnum): ENV = 1 # INPUTS = 2 - # OUTPUTS = 3 \ No newline at end of file + # OUTPUTS = 3 + +class SecretVariableCategory(IntEnum): + NONE = 0 + ACTIONS = 1 + AGENTS = 2 + CODESPACES = 3 + DEPENDABOT = 4 + ENVIRONMENTS = 5 + +class SecretVariableType(IntEnum): + NONE = 0 + SECRET = 1 + VARIABLE = 2 + +class SecretVariableVisibility(IntEnum): + NONE = 0 + ALL = 1 + PRIVATE = 2 + SELECTED = 3 diff --git a/src/tests/conftest.py b/src/tests/conftest.py index 9e81811..f1ae9e8 100644 --- a/src/tests/conftest.py +++ b/src/tests/conftest.py @@ -47,6 +47,8 @@ def client(logger): @pytest.fixture def database(): db_file = os.path.join(tempfile.gettempdir(), 'tests.db') + if os.path.isfile(db_file): + os.unlink(db_file) database = Database(db_file) yield database From b75bce6ae6f3764044e546c7187d06cc762f0d6e Mon Sep 17 00:00:00 2001 From: Pavel Tsakalidis Date: Sat, 13 Jun 2026 11:35:22 +0100 Subject: [PATCH 2/6] Add value mappings, update secvar query --- src/commands/report/queries/org-secrets.yaml | 87 ++++++++++++++++++++ src/commands/report/query_processor.py | 2 + src/commands/report/table_elements.py | 54 ++++++++---- 3 files changed, 128 insertions(+), 15 deletions(-) create mode 100644 src/commands/report/queries/org-secrets.yaml diff --git a/src/commands/report/queries/org-secrets.yaml b/src/commands/report/queries/org-secrets.yaml new file mode 100644 index 0000000..5a40de1 --- /dev/null +++ b/src/commands/report/queries/org-secrets.yaml @@ -0,0 +1,87 @@ +version: '1.1' +name: 'Organisation Secrets & Variables' +description: 'List of all secrets across the organisation and its repos' +filename: 'all-secrets-and-variables' +group: 'secrets' +sql: | + SELECT + COALESCE(r.name, 'Defined in Organisation') AS repo_name, + (CASE + WHEN s.category = 1 THEN 'actions' + WHEN s.category = 2 THEN 'agents' + WHEN s.category = 3 THEN 'codespaces' + WHEN s.category = 4 THEN 'dependabot' + WHEN s.category = 5 THEN 'environments' + ELSE 'unknown' + END) AS category, + (CASE + WHEN s.type = 1 THEN 'secret' + WHEN s.type = 2 THEN 'variable' + ELSE 'unknown' + END) AS type, + s.name, + s.value, + (CASE + WHEN s.visibility = 1 THEN 'all' + WHEN s.visibility = 2 THEN 'private' + WHEN s.visibility = 3 THEN 'selected' + ELSE '' + END) AS visibility, + (CASE WHEN org_secrets.id IS NOT NULL THEN 1 ELSE 0 END) AS overrides_org + FROM secrets_and_variables s + LEFT JOIN repositories r ON r.id = s.repo_id + LEFT JOIN organisations o ON o.id = r.org_id + LEFT JOIN ( + SELECT s.id, s.category, s.type, s.name + FROM secrets_and_variables s + WHERE s.repo_id = 0 + ) org_secrets ON org_secrets.category = s.category AND org_secrets.type = s.type AND org_secrets.name = s.name AND r.id > 0 + WHERE + o.id = :org OR o.id IS NULL +columns: + repo_name: + label: 'Repository' + filters: + column_control_alias: 'list' + category: + label: 'Category' + filters: + column_control_alias: 'list-no-search' + value_mapping: + actions: 'Actions' + agents: 'Agents' + codespaces: 'Codespaces' + dependabot: 'Dependabot' + environments: 'Environments' + type: + label: 'Type' + filters: + column_control_alias: 'list-no-search' + value_mapping: + secret: 'Secret' + variable: 'Variable' + name: + label: 'Name' + filters: + column_control_alias: 'list' + value: + label: 'Value' + filters: + column_control_alias: 'list' + visibility: + label: 'Visibility' + filters: + column_control_alias: 'list-no-search' + value_mapping: + all: 'All Repos' + private: 'Private Repos' + selected: 'Selected Repos' + overrides_org: + label: 'Overrides Org Value' + align: 'text-center' + filters: + column_control_alias: 'list-no-search' + type: 'icon' + format: + style_true: 'text-warning' + diff --git a/src/commands/report/query_processor.py b/src/commands/report/query_processor.py index dacdaf6..82f4424 100644 --- a/src/commands/report/query_processor.py +++ b/src/commands/report/query_processor.py @@ -74,6 +74,8 @@ def _prepare_query(self, sql: str) -> tuple[str, dict]: params = {} if '$_TRUSTED_ORGS_$' in sql: org_ids = self._get_trusted_org_ids() + if len(org_ids) == 0: + org_ids = [0] trusted_params = {f'trusted_org_{i}': org_id for i, org_id in enumerate(org_ids)} params.update(trusted_params) sql = sql.replace("$_TRUSTED_ORGS_$", ", ".join(f":{k}" for k in trusted_params)) diff --git a/src/commands/report/table_elements.py b/src/commands/report/table_elements.py index 3f877aa..866e712 100644 --- a/src/commands/report/table_elements.py +++ b/src/commands/report/table_elements.py @@ -11,9 +11,7 @@ class TableColumnOptions: _style_false: str = None _column_control: str = None _column_control_alias: str = None - - def __init__(self, name: str): - self._name = name + _value_mapping: dict = None @property def name(self) -> str: @@ -117,10 +115,19 @@ def column_control_alias(self) -> str: def column_control_alias(self, value: str): self._column_control_alias = value + @property + def value_mapping(self) -> dict: + return self._value_mapping or {} + + @value_mapping.setter + def value_mapping(self, value: dict): + self._value_mapping = value + def __init__(self, name: str): self.name = name self.visible = True self.type = 'text' + self.value_mapping = {} class TableColumn: _name: str = None @@ -220,6 +227,7 @@ def _load_columns(self, options: dict) -> dict: column.style_false = column_options.get('format', {}).get('style_false', '') column.column_control = column_options.get('filters', {}).get('column_control', '[]') column.column_control_alias = column_options.get('filters', {}).get('column_control_alias', '') + column.value_mapping = column_options.get('value_mapping', {}) all_columns[name] = column return all_columns @@ -270,24 +278,40 @@ def load_rows(self, raw_results: list): continue column = TableColumn(raw_column) - column.contents = raw_value + column.contents = self._map_value(raw_column, raw_value) if raw_column in self.columns: column.align = self.columns[raw_column].align if self.columns[raw_column].type == 'link': - column.type = 'link' - column.url = raw_result[self.columns[raw_column].link] if self.columns[raw_column].link in raw_result else self.columns[raw_column].link + column = self._format_link(column, raw_result, raw_column) elif self.columns[raw_column].type == 'icon': - column.type = 'icon' - if raw_value: - column.icon = self.columns[raw_column].icon_true - column.style = self.columns[raw_column].style_true - column.contents = 'yes' - else: - column.icon = self.columns[raw_column].icon_false - column.style = self.columns[raw_column].style_false - column.contents = 'no' + column = self._format_icon(column, raw_column, raw_value) row.append(column) self.rows.append(row) + + def _map_value(self, raw_column: str, raw_value: str) -> str: + if len(self.columns[raw_column].value_mapping) > 0: + if raw_value in self.columns[raw_column].value_mapping: + return self.columns[raw_column].value_mapping[raw_value] + elif '_' in self.columns[raw_column].value_mapping: + return self.columns[raw_column].value_mapping['_'] + return raw_value + + def _format_link(self, column: TableColumn, raw_result: dict, raw_column: str) -> TableColumn: + column.type = 'link' + column.url = raw_result[self.columns[raw_column].link] if self.columns[raw_column].link in raw_result else self.columns[raw_column].link + return column + + def _format_icon(self, column: TableColumn, raw_column: str, raw_value: str | None) -> TableColumn: + column.type = 'icon' + if raw_value: + column.icon = self.columns[raw_column].icon_true + column.style = self.columns[raw_column].style_true + column.contents = 'yes' + else: + column.icon = self.columns[raw_column].icon_false + column.style = self.columns[raw_column].style_false + column.contents = 'no' + return column From e7edb7abc5483c101ddeef266d86e035213aaeaa Mon Sep 17 00:00:00 2001 From: Pavel Tsakalidis Date: Sat, 13 Jun 2026 12:13:51 +0100 Subject: [PATCH 3/6] Update query --- src/commands/report/queries/org-secrets.yaml | 13 ++++++++++++- src/commands/report/table_elements.py | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/commands/report/queries/org-secrets.yaml b/src/commands/report/queries/org-secrets.yaml index 5a40de1..d857853 100644 --- a/src/commands/report/queries/org-secrets.yaml +++ b/src/commands/report/queries/org-secrets.yaml @@ -27,7 +27,8 @@ sql: | WHEN s.visibility = 3 THEN 'selected' ELSE '' END) AS visibility, - (CASE WHEN org_secrets.id IS NOT NULL THEN 1 ELSE 0 END) AS overrides_org + (CASE WHEN org_secrets.id IS NOT NULL THEN 1 ELSE 0 END) AS overrides_org, + COALESCE(selected_repos.repos, '') AS selected_repos FROM secrets_and_variables s LEFT JOIN repositories r ON r.id = s.repo_id LEFT JOIN organisations o ON o.id = r.org_id @@ -36,9 +37,19 @@ sql: | FROM secrets_and_variables s WHERE s.repo_id = 0 ) org_secrets ON org_secrets.category = s.category AND org_secrets.type = s.type AND org_secrets.name = s.name AND r.id > 0 + LEFT JOIN ( + SELECT s.id, GROUP_CONCAT(r.name, ',') AS repos + FROM secrets_and_variables s + JOIN secrets_and_variables_repos sr ON sr.org_secret_variable_id = s.id + JOIN repositories r ON r.id = sr.repo_id + WHERE s.repo_id = 0 AND s.visibility = 3 + GROUP BY s.id + ) selected_repos ON selected_repos.id = s.id WHERE o.id = :org OR o.id IS NULL columns: + selected_repos: hide + repo_name: label: 'Repository' filters: diff --git a/src/commands/report/table_elements.py b/src/commands/report/table_elements.py index 866e712..be1086e 100644 --- a/src/commands/report/table_elements.py +++ b/src/commands/report/table_elements.py @@ -292,7 +292,7 @@ def load_rows(self, raw_results: list): self.rows.append(row) def _map_value(self, raw_column: str, raw_value: str) -> str: - if len(self.columns[raw_column].value_mapping) > 0: + if raw_column in self.columns and len(self.columns[raw_column].value_mapping) > 0: if raw_value in self.columns[raw_column].value_mapping: return self.columns[raw_column].value_mapping[raw_value] elif '_' in self.columns[raw_column].value_mapping: From 08d2ad1964f7f15811045677545c3f20e2273c5d Mon Sep 17 00:00:00 2001 From: Pavel Tsakalidis Date: Sat, 13 Jun 2026 12:29:40 +0100 Subject: [PATCH 4/6] Rename table column --- src/commands/report/queries/org-secrets.yaml | 3 +-- src/database/helpers/db_secvars.py | 2 +- src/database/models.py | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/commands/report/queries/org-secrets.yaml b/src/commands/report/queries/org-secrets.yaml index d857853..8dbb764 100644 --- a/src/commands/report/queries/org-secrets.yaml +++ b/src/commands/report/queries/org-secrets.yaml @@ -40,7 +40,7 @@ sql: | LEFT JOIN ( SELECT s.id, GROUP_CONCAT(r.name, ',') AS repos FROM secrets_and_variables s - JOIN secrets_and_variables_repos sr ON sr.org_secret_variable_id = s.id + JOIN secrets_and_variables_repos sr ON sr.secret_variable_id = s.id JOIN repositories r ON r.id = sr.repo_id WHERE s.repo_id = 0 AND s.visibility = 3 GROUP BY s.id @@ -95,4 +95,3 @@ columns: type: 'icon' format: style_true: 'text-warning' - diff --git a/src/database/helpers/db_secvars.py b/src/database/helpers/db_secvars.py index 4fc3576..4ab6017 100644 --- a/src/database/helpers/db_secvars.py +++ b/src/database/helpers/db_secvars.py @@ -30,7 +30,7 @@ def create(self, repo_id: int, secvar: SecretVariableComponent) -> SecretsVariab if secvar.visibility == SecretVariableVisibility.SELECTED: for repo in secvar.repos: - repo_record = SecretsVariablesReposModel(org_secret_variable_id=record.id, repo_id=repo.id) + repo_record = SecretsVariablesReposModel(secret_variable_id=record.id, repo_id=repo.id) self.add(repo_record) self.save() diff --git a/src/database/models.py b/src/database/models.py index e630ed7..7e26396 100644 --- a/src/database/models.py +++ b/src/database/models.py @@ -137,5 +137,5 @@ class SecretsVariablesReposModel(Base): __tablename__ = 'secrets_and_variables_repos' id = Column(Integer, primary_key=True, autoincrement=True) - org_secret_variable_id = Column(Integer, default=0, index=True) + secret_variable_id = Column(Integer, default=0, index=True) repo_id = Column(Integer, default=0, index=True) From fae0824d816892c5ef5bf6cfc6d214ae6b84d2b9 Mon Sep 17 00:00:00 2001 From: Pavel Tsakalidis Date: Sat, 13 Jun 2026 13:12:04 +0100 Subject: [PATCH 5/6] Added HTML popup support --- src/commands/report/queries/org-secrets.yaml | 3 ++ src/commands/report/table_elements.py | 51 ++++++++++++++++++++ src/commands/report/templates/generic.html | 13 +++++ src/commands/report/templates/parent.html | 2 +- 4 files changed, 68 insertions(+), 1 deletion(-) diff --git a/src/commands/report/queries/org-secrets.yaml b/src/commands/report/queries/org-secrets.yaml index 8dbb764..6cb415a 100644 --- a/src/commands/report/queries/org-secrets.yaml +++ b/src/commands/report/queries/org-secrets.yaml @@ -87,6 +87,9 @@ columns: all: 'All Repos' private: 'Private Repos' selected: 'Selected Repos' + popup: + title: 'Repositories' + field: 'selected_repos' overrides_org: label: 'Overrides Org Value' align: 'text-center' diff --git a/src/commands/report/table_elements.py b/src/commands/report/table_elements.py index be1086e..b461ca2 100644 --- a/src/commands/report/table_elements.py +++ b/src/commands/report/table_elements.py @@ -12,6 +12,8 @@ class TableColumnOptions: _column_control: str = None _column_control_alias: str = None _value_mapping: dict = None + _popup_field: str = None + _popup_title: str = None @property def name(self) -> str: @@ -123,6 +125,22 @@ def value_mapping(self) -> dict: def value_mapping(self, value: dict): self._value_mapping = value + @property + def popup_field(self) -> str: + return self._popup_field or '' + + @popup_field.setter + def popup_field(self, value: str): + self._popup_field = value + + @property + def popup_title(self) -> str: + return self._popup_title + + @popup_title.setter + def popup_title(self, value: str): + self._popup_title = value + def __init__(self, name: str): self.name = name self.visible = True @@ -137,6 +155,8 @@ class TableColumn: _align: str = None _icon: str = None _style: str = None + _popup_html: str = None + _popup_title: str = None @property def name(self) -> str: @@ -198,6 +218,22 @@ def style(self) -> str: def style(self, value: str): self._style = value + @property + def popup_html(self) -> str: + return self._popup_html or '' + + @popup_html.setter + def popup_html(self, value: str): + self._popup_html = value + + @property + def popup_title(self) -> str: + return self._popup_title + + @popup_title.setter + def popup_title(self, value: str): + self._popup_title = value + class Table: columns: dict = None @@ -228,6 +264,8 @@ def _load_columns(self, options: dict) -> dict: column.column_control = column_options.get('filters', {}).get('column_control', '[]') column.column_control_alias = column_options.get('filters', {}).get('column_control_alias', '') column.value_mapping = column_options.get('value_mapping', {}) + column.popup_field = column_options.get('popup', {}).get('field', '') + column.popup_title = column_options.get('popup', {}).get('title', '') all_columns[name] = column return all_columns @@ -287,6 +325,8 @@ def load_rows(self, raw_results: list): column = self._format_link(column, raw_result, raw_column) elif self.columns[raw_column].type == 'icon': column = self._format_icon(column, raw_column, raw_value) + elif len(self.columns[raw_column].popup_field) > 0: + column = self._format_popup(column, raw_result, raw_column) row.append(column) self.rows.append(row) @@ -315,3 +355,14 @@ def _format_icon(self, column: TableColumn, raw_column: str, raw_value: str | No column.style = self.columns[raw_column].style_false column.contents = 'no' return column + + def _format_popup(self, column: TableColumn, raw_result: dict, raw_column: str) -> TableColumn: + column.type = 'popup' + column.popup_title = self.columns[raw_column].popup_title + column.popup_html = '' + + values = [value.strip() for value in raw_result[self.columns[raw_column].popup_field].split(',') if value.strip()] + if len(values) > 0: + html = "
    " + "".join(f"
  • {item}
    • " for item in values) + "
" + column.popup_html = html + return column diff --git a/src/commands/report/templates/generic.html b/src/commands/report/templates/generic.html index db8c373..5d64da7 100644 --- a/src/commands/report/templates/generic.html +++ b/src/commands/report/templates/generic.html @@ -24,6 +24,14 @@ {{ column.contents }} {% elif column.type == 'icon' %} + {% elif column.type == 'popup' %} + {% if column.popup_html|length > 0 %} + + {{ column.contents }} + + {% else %} + {{ column.contents }} + {% endif %} {% else %} {{ column.contents }} {% endif %} @@ -70,4 +78,9 @@ {% endif %} }); } + + $(document).ready(function() { + const popoverTriggerList = document.querySelectorAll('[data-bs-toggle="popover"]') + const popoverList = [...popoverTriggerList].map(popoverTriggerEl => new bootstrap.Popover(popoverTriggerEl)) + }); diff --git a/src/commands/report/templates/parent.html b/src/commands/report/templates/parent.html index 4fc6cd8..a9c9996 100644 --- a/src/commands/report/templates/parent.html +++ b/src/commands/report/templates/parent.html @@ -14,7 +14,7 @@ - + From dfd02ba0d52f470997ce3d8a4056d39d1fb84a19 Mon Sep 17 00:00:00 2001 From: Pavel Tsakalidis Date: Sat, 13 Jun 2026 15:16:54 +0100 Subject: [PATCH 6/6] Add links --- src/commands/report/queries/org-secrets.yaml | 20 +++++++++++++++++++- src/commands/report/templates/generic.html | 2 +- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/commands/report/queries/org-secrets.yaml b/src/commands/report/queries/org-secrets.yaml index 6cb415a..c2b0aa5 100644 --- a/src/commands/report/queries/org-secrets.yaml +++ b/src/commands/report/queries/org-secrets.yaml @@ -28,7 +28,19 @@ sql: | ELSE '' END) AS visibility, (CASE WHEN org_secrets.id IS NOT NULL THEN 1 ELSE 0 END) AS overrides_org, - COALESCE(selected_repos.repos, '') AS selected_repos + COALESCE(selected_repos.repos, '') AS selected_repos, + (CASE + WHEN r.id IS NOT NULL AND s.category = 1 THEN CONCAT('https://github.com/', o.name, '/', r.name, '/settings/secrets/actions') + WHEN r.id IS NOT NULL AND s.category = 2 THEN CONCAT('https://github.com/', o.name, '/', r.name, '/settings/secrets/agents') + WHEN r.id IS NOT NULL AND s.category = 3 THEN CONCAT('https://github.com/', o.name, '/', r.name, '/settings/secrets/codespaces') + WHEN r.id IS NOT NULL AND s.category = 4 THEN CONCAT('https://github.com/', o.name, '/', r.name, '/settings/secrets/dependabot') + + ELSE '' + END) AS category_url, + (CASE + WHEN r.id IS NOT NULL THEN CONCAT('https://github.com/', o.name, '/', r.name) + ELSE '' + END) AS repo_url FROM secrets_and_variables s LEFT JOIN repositories r ON r.id = s.repo_id LEFT JOIN organisations o ON o.id = r.org_id @@ -49,13 +61,19 @@ sql: | o.id = :org OR o.id IS NULL columns: selected_repos: hide + category_url: hide + repo_url: hide repo_name: label: 'Repository' + type: 'link' + link: 'repo_url' filters: column_control_alias: 'list' category: label: 'Category' + type: 'link' + link: 'category_url' filters: column_control_alias: 'list-no-search' value_mapping: diff --git a/src/commands/report/templates/generic.html b/src/commands/report/templates/generic.html index 5d64da7..e3f808d 100644 --- a/src/commands/report/templates/generic.html +++ b/src/commands/report/templates/generic.html @@ -20,7 +20,7 @@ {% for column in row %} - {% if column.type == 'link' %} + {% if column.type == 'link' and column.url|length > 0 %} {{ column.contents }} {% elif column.type == 'icon' %}