Implement org/repo secret/variable support

butler.py

@@ -2,17 +2,19 @@
 from src.commands.database.command import CommandDatabase
 from src.commands.process.command import CommandProcess
 from src.commands.report.command import CommandReport
+from src.commands.secvars.command import CommandSecretsAndVariables
 from src.libs.utils import Utils
 from src.libs.exceptions import InvalidCommandLine
 from src.commands.download.command import CommandDownload
 
 
-__VERSION__ = '0.10.2 Beta'
+__VERSION__ = '0.12.0'
 commands = {
     'download': CommandDownload,
     'database': CommandDatabase,
     'process': CommandProcess,
     'report': CommandReport,
+    'secvars': CommandSecretsAndVariables
 }
 
 parser = argparse.ArgumentParser(prog="butler", description=f"Butler - GitHub Actions Oversight v{__VERSION__}")

src/commands/download/download.py

@@ -160,44 +160,6 @@ def _collect_repos(self, repos: list[RepoComponent]) -> None:
         self.log.trace(f"Writing to database...")
         self.database.commit()
 
-    def _save_repo(self, repo: RepoComponent) -> None:
-        with self.lock:
-            self.log.info(f"Saving repository {repo}")
-            if repo.org.id == 0:
-                repo.org.id = self._create_org(repo.org).id
-
-            if len(repo.ref) > 0:
-                repo.poll_status = PollStatus.PENDING
-                repo_db = self.database.repos().create(repo)
-                return
-
-            # At this point, there is no `ref` in the object.
-            # Search if the repo is already in the database.
-            repo_db = self.database.repos().find(repo.org.id, repo.name, None)
-            if self._repo_already_stored(repo_db):
-                return
-
-        # Either there's no database record, or the stored one also has an empty ref.
-        fresh_repo = self._fetch_repo(repo)
-        with self.lock:
-            if fresh_repo.org.name.lower() == repo.org.name.lower() and fresh_repo.name.lower() == repo.name.lower():
-                fresh_repo.org.id = repo.org.id
-                fresh_repo.poll_status = PollStatus.SCANNED if repo.status == RepoStatus.MISSING else PollStatus.PENDING
-                if repo_db:
-                    fresh_repo.id = repo_db.id
-                    self.database.repos().update(fresh_repo)
-                else:
-                    self.database.repos().create(fresh_repo)
-                return
-
-            # Here, the fetched repo is different to the one passed to the function, this happens when a repo is redirected.
-            fresh_repo.org.id = self._create_org(fresh_repo.org).id
-            fresh_repo_db = self.database.repos().create(fresh_repo)
-
-            repo.redirect_id = fresh_repo_db.id
-            repo.status = RepoStatus.REDIRECT
-            self.database.repos().create(repo)
-
     def _resolve_commits(self) -> None:
         while True:
             batch = self.database.next_commit_to_resolve(self.threads)

src/commands/download/download_helper.py

@@ -1,3 +1,4 @@
+from src.libs.constants import PollStatus
 from src.database.models import OrganisationModel, RepositoryModel
 from src.github.exceptions import HttpNotFound
 from src.libs.components.org import OrgComponent
@@ -167,3 +168,41 @@ def _create_child_workflow_from_workflow(self, uses: str, workflow_instance: Wor
         else:
             org, repo, workflow = self._create_child_workflow_from_action(uses)
         return org, repo, workflow
+
+    def _save_repo(self, repo: RepoComponent) -> None:
+        with self.lock:
+            self.log.info(f"Saving repository {repo}")
+            if repo.org.id == 0:
+                repo.org.id = self._create_org(repo.org).id
+
+            if len(repo.ref) > 0:
+                repo.poll_status = PollStatus.PENDING
+                repo_db = self.database.repos().create(repo)
+                return
+
+            # At this point, there is no `ref` in the object.
+            # Search if the repo is already in the database.
+            repo_db = self.database.repos().find(repo.org.id, repo.name, None)
+            if self._repo_already_stored(repo_db):
+                return
+
+        # Either there's no database record, or the stored one also has an empty ref.
+        fresh_repo = self._fetch_repo(repo)
+        with self.lock:
+            if fresh_repo.org.name.lower() == repo.org.name.lower() and fresh_repo.name.lower() == repo.name.lower():
+                fresh_repo.org.id = repo.org.id
+                fresh_repo.poll_status = PollStatus.SCANNED if repo.status == RepoStatus.MISSING else PollStatus.PENDING
+                if repo_db:
+                    fresh_repo.id = repo_db.id
+                    self.database.repos().update(fresh_repo)
+                else:
+                    self.database.repos().create(fresh_repo)
+                return
+
+            # Here, the fetched repo is different to the one passed to the function, this happens when a repo is redirected.
+            fresh_repo.org.id = self._create_org(fresh_repo.org).id
+            fresh_repo_db = self.database.repos().create(fresh_repo)
+
+            repo.redirect_id = fresh_repo_db.id
+            repo.status = RepoStatus.REDIRECT
+            self.database.repos().create(repo)

src/commands/report/queries/org-secrets.yaml

@@ -0,0 +1,118 @@
+version: '1.1'
+name: 'Organisation Secrets & Variables'
+description: 'List of all secrets across the organisation and its repos'
+filename: 'all-secrets-and-variables'
+group: 'secrets'
+sql: |
+  SELECT
+  	COALESCE(r.name, 'Defined in Organisation') AS repo_name,
+  	(CASE
+  		WHEN s.category = 1 THEN 'actions'
+  		WHEN s.category = 2 THEN 'agents'
+  		WHEN s.category = 3 THEN 'codespaces'
+  		WHEN s.category = 4 THEN 'dependabot'
+  		WHEN s.category = 5 THEN 'environments'
+  		ELSE 'unknown'
+  	END) AS category,
+  	(CASE
+  		WHEN s.type = 1 THEN 'secret'
+  		WHEN s.type = 2 THEN 'variable'
+  		ELSE 'unknown'
+  	END) AS type,
+    s.name,
+  	s.value,
+  	(CASE
+  		WHEN s.visibility = 1 THEN 'all'
+  		WHEN s.visibility = 2 THEN 'private'
+  		WHEN s.visibility = 3 THEN 'selected'
+  		ELSE ''
+  	END) AS visibility,
+    (CASE WHEN org_secrets.id IS NOT NULL THEN 1 ELSE 0 END) AS overrides_org,
+    COALESCE(selected_repos.repos, '') AS selected_repos,
+    (CASE 
+      WHEN r.id IS NOT NULL AND s.category = 1 THEN CONCAT('https://github.com/', o.name, '/', r.name, '/settings/secrets/actions')
+      WHEN r.id IS NOT NULL AND s.category = 2 THEN CONCAT('https://github.com/', o.name, '/', r.name, '/settings/secrets/agents')
+      WHEN r.id IS NOT NULL AND s.category = 3 THEN CONCAT('https://github.com/', o.name, '/', r.name, '/settings/secrets/codespaces')
+      WHEN r.id IS NOT NULL AND s.category = 4 THEN CONCAT('https://github.com/', o.name, '/', r.name, '/settings/secrets/dependabot')
+
+      ELSE ''
+    END) AS category_url,
+    (CASE
+      WHEN r.id IS NOT NULL THEN CONCAT('https://github.com/', o.name, '/', r.name)
+      ELSE ''
+    END) AS repo_url
+  FROM secrets_and_variables s
+  LEFT JOIN repositories r ON r.id = s.repo_id
+  LEFT JOIN organisations o ON o.id = r.org_id
+  LEFT JOIN (
+    SELECT s.id, s.category, s.type, s.name
+    FROM secrets_and_variables s
+    WHERE s.repo_id = 0
+  ) org_secrets ON org_secrets.category = s.category AND org_secrets.type = s.type AND org_secrets.name = s.name AND r.id > 0
+  LEFT JOIN (
+    SELECT s.id, GROUP_CONCAT(r.name, ',') AS repos
+    FROM secrets_and_variables s
+    JOIN secrets_and_variables_repos sr ON sr.secret_variable_id = s.id
+    JOIN repositories r ON r.id = sr.repo_id
+    WHERE s.repo_id = 0 AND s.visibility = 3
+    GROUP BY s.id
+  ) selected_repos ON selected_repos.id = s.id
+  WHERE
+  	o.id = :org OR o.id IS NULL
+columns:
+  selected_repos: hide
+  category_url: hide
+  repo_url: hide
+
+  repo_name:
+    label: 'Repository'
+    type: 'link'
+    link: 'repo_url'
+    filters:
+      column_control_alias: 'list'
+  category:
+    label: 'Category'
+    type: 'link'
+    link: 'category_url'
+    filters:
+      column_control_alias: 'list-no-search'
+    value_mapping:
+      actions: 'Actions'
+      agents: 'Agents'
+      codespaces: 'Codespaces'
+      dependabot: 'Dependabot'
+      environments: 'Environments'
+  type:
+    label: 'Type'
+    filters:
+      column_control_alias: 'list-no-search'
+    value_mapping:
+      secret: 'Secret'
+      variable: 'Variable'
+  name:
+    label: 'Name'
+    filters:
+      column_control_alias: 'list'
+  value:
+    label: 'Value'
+    filters:
+      column_control_alias: 'list'
+  visibility:
+    label: 'Visibility'
+    filters:
+      column_control_alias: 'list-no-search'
+    value_mapping:
+      all: 'All Repos'
+      private: 'Private Repos'
+      selected: 'Selected Repos'
+    popup:
+      title: 'Repositories'
+      field: 'selected_repos'
+  overrides_org:
+    label: 'Overrides Org Value'
+    align: 'text-center'
+    filters:
+      column_control_alias: 'list-no-search'
+    type: 'icon'
+    format:
+      style_true: 'text-warning'

src/commands/report/query_processor.py

@@ -74,6 +74,8 @@ def _prepare_query(self, sql: str) -> tuple[str, dict]:
         params = {}
         if '$_TRUSTED_ORGS_$' in sql:
             org_ids = self._get_trusted_org_ids()
+            if len(org_ids) == 0:
+                org_ids = [0]
             trusted_params = {f'trusted_org_{i}': org_id for i, org_id in enumerate(org_ids)}
             params.update(trusted_params)
             sql = sql.replace("$_TRUSTED_ORGS_$", ", ".join(f":{k}" for k in trusted_params))

src/commands/report/table_elements.py

@@ -11,9 +11,9 @@ class TableColumnOptions:
     _style_false: str = None
     _column_control: str = None
     _column_control_alias: str = None
-
-    def __init__(self, name: str):
-        self._name = name
+    _value_mapping: dict = None
+    _popup_field: str = None
+    _popup_title: str = None
 
     @property
     def name(self) -> str:
@@ -117,10 +117,35 @@ def column_control_alias(self) -> str:
     def column_control_alias(self, value: str):
         self._column_control_alias = value
 
+    @property
+    def value_mapping(self) -> dict:
+        return self._value_mapping or {}
+
+    @value_mapping.setter
+    def value_mapping(self, value: dict):
+        self._value_mapping = value
+
+    @property
+    def popup_field(self) -> str:
+        return self._popup_field or ''
+
+    @popup_field.setter
+    def popup_field(self, value: str):
+        self._popup_field = value
+
+    @property
+    def popup_title(self) -> str:
+        return self._popup_title
+
+    @popup_title.setter
+    def popup_title(self, value: str):
+        self._popup_title = value
+
     def __init__(self, name: str):
         self.name = name
         self.visible = True
         self.type = 'text'
+        self.value_mapping = {}
 
 class TableColumn:
     _name: str = None
@@ -130,6 +155,8 @@ class TableColumn:
     _align: str = None
     _icon: str = None
     _style: str = None
+    _popup_html: str = None
+    _popup_title: str = None
 
     @property
     def name(self) -> str:
@@ -191,6 +218,22 @@ def style(self) -> str:
     def style(self, value: str):
         self._style = value
 
+    @property
+    def popup_html(self) -> str:
+        return self._popup_html or ''
+
+    @popup_html.setter
+    def popup_html(self, value: str):
+        self._popup_html = value
+
+    @property
+    def popup_title(self) -> str:
+        return self._popup_title
+
+    @popup_title.setter
+    def popup_title(self, value: str):
+        self._popup_title = value
+
 class Table:
     columns: dict = None
 
@@ -220,6 +263,9 @@ def _load_columns(self, options: dict) -> dict:
                 column.style_false = column_options.get('format', {}).get('style_false', '')
                 column.column_control = column_options.get('filters', {}).get('column_control', '[]')
                 column.column_control_alias = column_options.get('filters', {}).get('column_control_alias', '')
+                column.value_mapping = column_options.get('value_mapping', {})
+                column.popup_field = column_options.get('popup', {}).get('field', '')
+                column.popup_title = column_options.get('popup', {}).get('title', '')
             all_columns[name] = column
         return all_columns
 
@@ -270,24 +316,53 @@ def load_rows(self, raw_results: list):
                     continue
 
                 column = TableColumn(raw_column)
-                column.contents = raw_value
+                column.contents = self._map_value(raw_column, raw_value)
 
                 if raw_column in self.columns:
                     column.align = self.columns[raw_column].align
 
                     if self.columns[raw_column].type == 'link':
-                        column.type = 'link'
-                        column.url = raw_result[self.columns[raw_column].link] if self.columns[raw_column].link in raw_result else self.columns[raw_column].link
+                        column = self._format_link(column, raw_result, raw_column)
                     elif self.columns[raw_column].type == 'icon':
-                        column.type = 'icon'
-                        if raw_value:
-                            column.icon = self.columns[raw_column].icon_true
-                            column.style = self.columns[raw_column].style_true
-                            column.contents = 'yes'
-                        else:
-                            column.icon = self.columns[raw_column].icon_false
-                            column.style = self.columns[raw_column].style_false
-                            column.contents = 'no'
+                        column = self._format_icon(column, raw_column, raw_value)
+                    elif len(self.columns[raw_column].popup_field) > 0:
+                        column = self._format_popup(column, raw_result, raw_column)
 
                 row.append(column)
             self.rows.append(row)
+
+    def _map_value(self, raw_column: str, raw_value: str) -> str:
+        if raw_column in self.columns and len(self.columns[raw_column].value_mapping) > 0:
+            if raw_value in self.columns[raw_column].value_mapping:
+                return self.columns[raw_column].value_mapping[raw_value]
+            elif '_' in self.columns[raw_column].value_mapping:
+                return self.columns[raw_column].value_mapping['_']
+        return raw_value
+
+    def _format_link(self, column: TableColumn, raw_result: dict, raw_column: str) -> TableColumn:
+        column.type = 'link'
+        column.url = raw_result[self.columns[raw_column].link] if self.columns[raw_column].link in raw_result else self.columns[raw_column].link
+        return column
+
+    def _format_icon(self, column: TableColumn, raw_column: str, raw_value: str | None) -> TableColumn:
+        column.type = 'icon'
+        if raw_value:
+            column.icon = self.columns[raw_column].icon_true
+            column.style = self.columns[raw_column].style_true
+            column.contents = 'yes'
+        else:
+            column.icon = self.columns[raw_column].icon_false
+            column.style = self.columns[raw_column].style_false
+            column.contents = 'no'
+        return column
+
+    def _format_popup(self, column: TableColumn, raw_result: dict, raw_column: str) -> TableColumn:
+        column.type = 'popup'
+        column.popup_title = self.columns[raw_column].popup_title
+        column.popup_html = ''
+
+        values = [value.strip() for value in raw_result[self.columns[raw_column].popup_field].split(',') if value.strip()]
+        if len(values) > 0:
+            html = "<ul class='list-group'>" + "".join(f"<li class='list-group-item'>{item}</li>" for item in values) + "</ul>"
+            column.popup_html = html
+        return column