{% for column in row %}
|
- {% if column.type == 'link' %}
+ {% if column.type == 'link' and column.url|length > 0 %}
{{ column.contents }}
{% elif column.type == 'icon' %}
+ {% elif column.type == 'popup' %}
+ {% if column.popup_html|length > 0 %}
+
+ {{ column.contents }}
+
+ {% else %}
+ {{ column.contents }}
+ {% endif %}
{% else %}
{{ column.contents }}
{% endif %}
@@ -70,4 +78,9 @@
{% endif %}
});
}
+
+ $(document).ready(function() {
+ const popoverTriggerList = document.querySelectorAll('[data-bs-toggle="popover"]')
+ const popoverList = [...popoverTriggerList].map(popoverTriggerEl => new bootstrap.Popover(popoverTriggerEl))
+ });
diff --git a/src/commands/report/templates/parent.html b/src/commands/report/templates/parent.html
index 4fc6cd8..a9c9996 100644
--- a/src/commands/report/templates/parent.html
+++ b/src/commands/report/templates/parent.html
@@ -14,7 +14,7 @@
-
+
diff --git a/src/commands/secvars/command.py b/src/commands/secvars/command.py
new file mode 100644
index 0000000..b58d669
--- /dev/null
+++ b/src/commands/secvars/command.py
@@ -0,0 +1,51 @@
+import os
+import argparse
+from src.commands.secvars.secvars import ServiceSecretsAndVariables
+from src.commands.command import Command
+from src.database.database import Database
+from src.libs.exceptions import InvalidCommandLine
+from src.github.client import GitHubClient
+
+
+class CommandSecretsAndVariables(Command):
+ @staticmethod
+ def load_command_line(subparsers: any) -> None:
+ subparser = subparsers.add_parser("secvars", help="Download Secrets and Variables from GitHub")
+
+ subparser.add_argument("--org", type=str, help="Organisation to download secrets and variables for")
+ subparser.add_argument("--database", default="database.db", type=str, help="Path to SQLite database to create or connect to")
+ subparser.add_argument("--resume-next", default=True, action="store_true", help="Resume downloads on server errors")
+ subparser.add_argument("--threads", default=1, type=int, help="Enable multithreading")
+
+ Command.define_shared_arguments(subparser)
+
+ def load_arguments(self, arguments: argparse.Namespace) -> dict:
+ return {
+ 'org': '' if arguments.org is None or len(arguments.org.strip()) == 0 else arguments.org.strip(),
+ 'database': '' if arguments.database is None or len(arguments.database.strip()) == 0 else os.path.realpath(arguments.database.strip()),
+ 'resume_next': arguments.resume_next or False,
+ 'threads': int(arguments.threads),
+ }
+
+ def validate_command_arguments(self, arguments: dict) -> None:
+ # Validate database.
+ if len(arguments['database']) == 0:
+ raise InvalidCommandLine(f"--database cannot be empty")
+ elif not arguments['database'].lower().endswith('.sqlite3') and not arguments['database'].lower().endswith('.db'):
+ raise InvalidCommandLine(f"--database {arguments['database']} is not a SQLite database (must end with .sqlite3 or .db)")
+
+ if len(arguments['org']) == 0:
+ raise InvalidCommandLine(f"--org cannot be empty")
+
+ if arguments['threads'] <= 0:
+ arguments['threads'] = 1
+
+ def execute(self, arguments: dict) -> bool:
+ database = Database(arguments['database'], arguments['db_debug'], arguments['db_debug_auto_commit'])
+
+ service = ServiceSecretsAndVariables(self.log, database)
+ service.github_client = GitHubClient(self.tokens, self.log)
+ service.org = arguments['org']
+ service.resume_next = arguments['resume_next']
+ service.threads = arguments['threads']
+ return service.run()
diff --git a/src/commands/secvars/secvars.py b/src/commands/secvars/secvars.py
new file mode 100644
index 0000000..28cdd8f
--- /dev/null
+++ b/src/commands/secvars/secvars.py
@@ -0,0 +1,188 @@
+import threading
+import concurrent.futures
+from contextlib import nullcontext
+from src.database.models import OrganisationModel, RepositoryModel
+from src.libs.constants import SecretVariableCategory, SecretVariableType, SecretVariableVisibility
+from src.libs.components.secvar import SecretVariableComponent
+from src.commands.download.download_helper import DownloadHelper
+from src.commands.service import Service
+from src.github.client import GitHubClient
+from src.github.exceptions import TooManyRequests, ApiRateLimitExceeded, OrgNotFound
+from src.libs.components.org import OrgComponent
+from src.libs.constants import PollStatus, OrgStatus
+from src.libs.exceptions import InvalidCommandLine
+from src.libs.utils import Utils
+
+
+class ServiceSecretsAndVariables(Service, DownloadHelper):
+ org: str = None
+ lock: threading.Lock = None
+ resume_next: bool = None
+ github_client: GitHubClient = None
+ threads: int = None
+
+ _combinations: list = [
+ {'label': 'actions / secrets', 'category': SecretVariableCategory.ACTIONS, 'type': SecretVariableType.SECRET},
+ {'label': 'actions / variables', 'category': SecretVariableCategory.ACTIONS, 'type': SecretVariableType.VARIABLE},
+ {'label': 'agents / secrets', 'category': SecretVariableCategory.AGENTS, 'type': SecretVariableType.SECRET},
+ {'label': 'agents / variables', 'category': SecretVariableCategory.AGENTS, 'type': SecretVariableType.VARIABLE},
+ {'label': 'codespaces / secrets', 'category': SecretVariableCategory.CODESPACES, 'type': SecretVariableType.SECRET},
+ {'label': 'dependabot / secrets', 'category': SecretVariableCategory.DEPENDABOT, 'type': SecretVariableType.SECRET},
+ ]
+
+ def run(self) -> bool:
+ # Thanks SQLite3 :>
+ self.lock = threading.Lock() if self.threads > 1 else nullcontext()
+
+ while True:
+ try:
+ self.log.info(f"Collecting repositories for {self.org}...")
+ self._collect_targets()
+
+ org = self.database.orgs().find(self.org)
+ if not org:
+ raise InvalidCommandLine(f"Organisation {self.org} not found")
+
+ self.log.info(f"Collecting secrets and variables for {org.name}")
+ self._collect_secrets_and_variables(org)
+
+ break
+ except (TooManyRequests, ApiRateLimitExceeded) as e:
+ if self.resume_next:
+ self.github_client.halt_and_continue(5)
+ continue
+ raise
+ except Exception as e:
+ if self.resume_next and 'Server Error' in str(e):
+ self.github_client.halt_and_continue(2)
+ continue
+ raise
+
+ if self.database.debug:
+ self.log.info(f"Total SQL Queries: {self.database.total_queries}")
+
+ self.log.info(f"Total API Calls: {self.github_client._api.total_requests}")
+ return True
+
+ def _collect_targets(self) -> None:
+ orgs, repos = Utils.filter_orgs_and_repos([self.org])
+ self.log.debug(f"Input has {len(orgs)} organisations and {len(repos)} repositories")
+ self._collect_orgs(orgs)
+
+ def _collect_orgs(self, orgs: list[OrgComponent]) -> None:
+ count = 0
+ for org in orgs:
+ count += 1
+ self.log.info(f"Processing {org} ({count}/{len(orgs)})")
+
+ org_db = self._create_org(org)
+ org.id = org_db.id
+
+ if org_db.poll_status == PollStatus.SCANNED:
+ self.log.debug(f"Organisation {org_db.name} already scanned - skipping")
+ continue
+ elif org_db.poll_status == PollStatus.NONE:
+ self.log.info(f"Organisation {org_db.name} is new or was not marked as pending before")
+ self.database.orgs().set_poll_status(org_db.id, PollStatus.PENDING)
+
+ try:
+ for batch in self.github_client.get_org_repos(org.name, True, True):
+ for repo in batch:
+ repo.org.id = org_db.id
+ self._save_repo(repo)
+ except OrgNotFound as e:
+ self.log.error(f"Organisation {org.name} not found")
+ self.database.orgs().set_status(org_db.id, OrgStatus.MISSING)
+
+ self.database.orgs().set_poll_status(org_db.id, PollStatus.SCANNED)
+
+ self.log.trace(f"Writing to database...")
+ self.database.commit()
+
+ self.log.trace(f"Writing to database...")
+ self.database.commit()
+
+ def _collect_secrets_and_variables(self, org: OrganisationModel) -> None:
+ components = []
+ for item in self._combinations:
+ self.log.info(f"Getting organisation {item['label']}")
+ try:
+ results = self.github_client.get_secrets(org.name, item['category'], item['type'], None, None)
+ components.extend(self._create_components(org.id, results, item['category'], item['type']))
+ except Exception as e:
+ self.log.warning(f"Could not get {item['label']}")
+
+ self.log.info(f"Writing to database")
+ for component in components:
+ self.database.secvars().create(0, component)
+
+ self.database.commit()
+
+ self.log.info(f"Getting organisation repos")
+ repos = self.database.repos().all(org.id)
+ self.log.info(f"Got {len(repos)} repos")
+
+ with concurrent.futures.ThreadPoolExecutor(max_workers=self.threads) as executor:
+ # Submit all repositories to the executor
+ future_to_repo = {executor.submit(self._fetch_secvar_repo_single, org, repo): repo for repo in repos}
+
+ for future in concurrent.futures.as_completed(future_to_repo):
+ repo = future_to_repo[future]
+ try:
+ # Retrieve the data returned by fetch_repo_data
+ returned_repo, components = future.result()
+
+ # 3. Write to the database safely in the main thread
+ if components:
+ self.log.info(f"Writing {len(components)} components to database for {returned_repo.name}")
+ with self.lock:
+ for component in components:
+ self.database.secvars().create(repo.id, component)
+
+ self.database.commit()
+
+ except Exception as e:
+ self.log.error(f"Unhandled exception processing repo {repo.name}: {e}")
+
+ def _fetch_secvar_repo_single(self, org: OrganisationModel, repo: RepositoryModel) -> tuple[object, list]:
+ components = []
+ self.log.info(f"Getting secrets and variables for {repo.name}")
+ for item in self._combinations:
+ self.log.info(f"Getting data for repo {repo.name} and {item['label']}")
+ try:
+ results = self.github_client.get_secrets(org.name, item['category'], item['type'], None, repo.name)
+ components.extend(self._create_components(org.id, results, item['category'], item['type']))
+ except Exception as e:
+ self.log.warning(f"Could not get {item['label']} for {repo.name}")
+
+ return repo, components
+
+ def _create_components(self, org_id: int, items: list, category: SecretVariableCategory, type: SecretVariableType) -> list[SecretVariableComponent]:
+ components = []
+ for item in items:
+ component = SecretVariableComponent()
+ component.category = category
+ component.type = type
+ component.name = item['name']
+ if component.type == SecretVariableType.VARIABLE:
+ component.value = item['value']
+ component.created_at = item['created_at']
+ component.updated_at = item['updated_at']
+
+ if 'visibility' in item:
+ match item['visibility']:
+ case 'all':
+ component.visibility = SecretVariableVisibility.ALL
+ case 'private':
+ component.visibility = SecretVariableVisibility.PRIVATE
+ case 'selected':
+ component.visibility = SecretVariableVisibility.SELECTED
+
+ repos = []
+ for repo in item['repos']:
+ repo = self.database.repos().find(org_id, repo, None)
+ repos.append(repo)
+ component.repos = repos
+
+ components.append(component)
+ return components
diff --git a/src/database/database.py b/src/database/database.py
index 573a15a..801459e 100644
--- a/src/database/database.py
+++ b/src/database/database.py
@@ -1,4 +1,5 @@
import os
+from src.database.helpers.db_secvars import DBSecretsAndVariables
from src.database.database_helper import DatabaseHelper
from src.database.helpers.db_config import DBConfig
from src.database.helpers.db_jobs import DBJob
@@ -14,7 +15,7 @@
class Database(DatabaseHelper):
- __VERSION__: str = '1.0.1'
+ __VERSION__: str = '1.1.0'
_engine: Engine = None
_sessionmaker: sessionmaker = None
_session = None
@@ -26,6 +27,7 @@ class Database(DatabaseHelper):
_steps: DBStep | None = None
_vars: DBVars | None = None
_config: DBConfig | None = None
+ _secvars: DBSecretsAndVariables | None = None
_total_queries: int = 0
_debug: bool = False
@@ -98,7 +100,6 @@ def _update_views(self) -> None:
"""
self.execute(sql)
-
def orgs(self) -> DBOrg:
if not self._orgs:
self._orgs = DBOrg(self.session, self.auto_commit)
@@ -134,6 +135,11 @@ def config(self) -> DBConfig:
self._config = DBConfig(self.session, self.auto_commit)
return self._config
+ def secvars(self) -> DBSecretsAndVariables:
+ if not self._secvars:
+ self._secvars = DBSecretsAndVariables(self.session, self.auto_commit)
+ return self._secvars
+
def commit(self) -> None:
self.session.commit()
diff --git a/src/database/helpers/db_repos.py b/src/database/helpers/db_repos.py
index 31722ed..53d29d6 100644
--- a/src/database/helpers/db_repos.py
+++ b/src/database/helpers/db_repos.py
@@ -97,3 +97,6 @@ def set_ref_resolved_fields(self, id: int, resolved_ref: str, resolved_ref_type:
def count(self) -> int:
return self.session.query(RepositoryModel).count()
+
+ def all(self, org_id: int) -> any:
+ return self.session.query(RepositoryModel).filter(RepositoryModel.org_id == org_id).all()
diff --git a/src/database/helpers/db_secvars.py b/src/database/helpers/db_secvars.py
new file mode 100644
index 0000000..4ab6017
--- /dev/null
+++ b/src/database/helpers/db_secvars.py
@@ -0,0 +1,37 @@
+from sqlalchemy import func
+from src.libs.constants import SecretVariableVisibility
+from src.libs.components.secvar import SecretVariableComponent
+from src.database.helpers.db_base import DBBase
+from src.database.models import SecretsVariablesModel, SecretsVariablesReposModel
+
+
+class DBSecretsAndVariables(DBBase):
+ def find(self, repo_id: int, category: int, type: int, name: str) -> SecretsVariablesModel | None:
+ return self.session.query(SecretsVariablesModel).filter(
+ SecretsVariablesModel.repo_id == repo_id,
+ SecretsVariablesModel.category == category,
+ SecretsVariablesModel.type == type,
+ func.lower(SecretsVariablesModel.name) == func.lower(name)
+ ).first()
+
+ def create(self, repo_id: int, secvar: SecretVariableComponent) -> SecretsVariablesModel:
+ record = self.find(repo_id, secvar.category, secvar.type, secvar.name)
+ if not record:
+ record = SecretsVariablesModel(repo_id=repo_id)
+ record.category = secvar.category
+ record.type = secvar.type
+ record.name = secvar.name
+ record.value = secvar.value
+ record.visibility = secvar.visibility
+ record.created_at = secvar.created_at
+ record.updated_at = secvar.updated_at
+ self.add(record)
+ self.save()
+
+ if secvar.visibility == SecretVariableVisibility.SELECTED:
+ for repo in secvar.repos:
+ repo_record = SecretsVariablesReposModel(secret_variable_id=record.id, repo_id=repo.id)
+ self.add(repo_record)
+ self.save()
+
+ return record
diff --git a/src/database/models.py b/src/database/models.py
index a0afcc4..7e26396 100644
--- a/src/database/models.py
+++ b/src/database/models.py
@@ -119,3 +119,23 @@ class ConfigModel(Base):
name = Column(String, primary_key=True, index=True)
value = Column(String, default='', index=True)
+
+class SecretsVariablesModel(Base):
+ __tablename__ = 'secrets_and_variables'
+
+ id = Column(Integer, primary_key=True, autoincrement=True)
+ repo_id = Column(Integer, default=0, index=True)
+ category = Column(Integer, default=0, index=True)
+ type = Column(Integer, default=0, index=True)
+ name = Column(String, default='', index=True)
+ value = Column(String, default='', index=True)
+ visibility = Column(Integer, default=0, index=True)
+ created_at = Column(Text, default='', index=True)
+ updated_at = Column(Text, default='', index=True)
+
+class SecretsVariablesReposModel(Base):
+ __tablename__ = 'secrets_and_variables_repos'
+
+ id = Column(Integer, primary_key=True, autoincrement=True)
+ secret_variable_id = Column(Integer, default=0, index=True)
+ repo_id = Column(Integer, default=0, index=True)
diff --git a/src/github/client.py b/src/github/client.py
index 2b2a8d0..9abb1d8 100644
--- a/src/github/client.py
+++ b/src/github/client.py
@@ -2,6 +2,7 @@
import base64
import time
from loguru import logger
+from src.libs.constants import SecretVariableCategory, SecretVariableType
from src.github.api import GitHubApi
from src.github.exceptions import (
AccountNotFound, UnknownAccountType, HttpNotFound, GitHubException, HttpEmptyRepo, HttpAccessBlocked,
@@ -357,3 +358,70 @@ def halt_and_continue(self, minutes: int) -> None:
self.log.info(f"Reached API rate limit - waiting {minutes} minutes")
time.sleep(minutes * 60)
self._api.refresh_tokens()
+
+ def get_secrets(self, org: str, category: SecretVariableCategory, type: SecretVariableType, environment: str | None, repo: str | None) -> list[dict]:
+ params = {'page': 1, 'per_page': 100}
+ url = self._get_secrets_url(org, category, type, environment, repo)
+ all_results = []
+ while True:
+ response_headers = {}
+ try:
+ results = self._api.get(url, params, None, response_headers)
+ except HttpNotFound:
+ raise RepoNotFound(f"{org}/{category}/{type}/{environment}/{repo}")
+
+ name = 'secrets' if 'secrets' in results else 'variables'
+ for item in results[name]:
+ if item.get('visibility', '') == 'selected':
+ item['repos'] = self._get_selected_repositories(item['selected_repositories_url'])
+ all_results.append(item)
+
+ # Invalidate as we'll be using the 'next url' from the headers.
+ params = None
+ url = self._get_next_url(response_headers)
+ if not url:
+ break
+
+ return all_results
+
+ def _get_selected_repositories(self, url: str) -> list:
+ url = url.replace('https://api.github.com', '')
+ try:
+ results = self._api.get(url, {'per_page': 999})
+ except HttpNotFound:
+ raise RepoNotFound(url)
+
+ repos = []
+ for result in results['repositories']:
+ repos.append(result['name'])
+ return repos
+
+ def _get_secrets_url(self, org: str, category: SecretVariableCategory, type: SecretVariableType, environment: str | None, repo: str | None) -> str:
+ category_mapping = {
+ # /orgs/{org}/actions/{secrets,variables}
+ # /repos/{owner}/{repo}/actions/{secrets,variables}
+ SecretVariableCategory.ACTIONS: 'actions',
+
+ # /orgs/{org}/agents/{secrets,variables}
+ SecretVariableCategory.AGENTS: 'agents',
+ # /repos/{owner}/{repo}/codespaces/secrets
+ SecretVariableCategory.CODESPACES: 'codespaces',
+ # /orgs/{org}/dependabot/secrets
+ SecretVariableCategory.DEPENDABOT: 'dependabot',
+ # /repos/{owner}/{repo}/environments/{environment_name}/secrets
+ SecretVariableCategory.ENVIRONMENTS: 'environments',
+ }
+
+ type_mapping = {
+ SecretVariableType.SECRET: 'secrets',
+ SecretVariableType.VARIABLE: 'variables',
+ }
+
+ base_path = f"/repos/{org}/{repo}" if repo else f"/orgs/{org}"
+ components = [
+ base_path,
+ category_mapping[category],
+ environment,
+ type_mapping[type]
+ ]
+ return "/".join(comp for comp in components if comp)
diff --git a/src/libs/components/secvar.py b/src/libs/components/secvar.py
new file mode 100644
index 0000000..45d5766
--- /dev/null
+++ b/src/libs/components/secvar.py
@@ -0,0 +1,90 @@
+from datetime import datetime
+from src.libs.constants import SecretVariableCategory, SecretVariableType, SecretVariableVisibility
+
+
+class SecretVariableComponent:
+ _id: int = None
+ _repos: list = None
+ _category: SecretVariableCategory = None
+ _type: SecretVariableType = None
+ _name: str = None
+ _value: str = None
+ _visibility: SecretVariableVisibility = None
+ _created_at: datetime = None
+ _updated_at: datetime = None
+
+ @property
+ def id(self) -> int:
+ return self._id or 0
+
+ @id.setter
+ def id(self, value: int):
+ self._id = value
+
+ @property
+ def repos(self) -> list:
+ return self._repos
+
+ @repos.setter
+ def repos(self, value: list):
+ self._repos = value
+
+ @property
+ def category(self) -> SecretVariableCategory:
+ return self._category
+
+ @category.setter
+ def category(self, value: SecretVariableCategory):
+ self._category = value
+
+ @property
+ def type(self) -> SecretVariableType:
+ return self._type
+
+ @type.setter
+ def type(self, value: SecretVariableType):
+ self._type = value
+
+ @property
+ def name(self) -> str:
+ return self._name
+
+ @name.setter
+ def name(self, value: str):
+ self._name = value
+
+ @property
+ def value(self) -> str:
+ return self._value or ''
+
+ @value.setter
+ def value(self, value: str):
+ self._value = value
+
+ @property
+ def visibility(self) -> SecretVariableVisibility:
+ return self._visibility
+
+ @visibility.setter
+ def visibility(self, value: SecretVariableVisibility):
+ self._visibility = value
+
+ @property
+ def created_at(self) -> datetime:
+ return self._created_at
+
+ @created_at.setter
+ def created_at(self, value: datetime | str):
+ if isinstance(value, str):
+ value = datetime.fromisoformat(value)
+ self._created_at = value
+
+ @property
+ def updated_at(self) -> datetime:
+ return self._updated_at
+
+ @updated_at.setter
+ def updated_at(self, value: datetime | str):
+ if isinstance(value, str):
+ value = datetime.fromisoformat(value)
+ self._updated_at = value
diff --git a/src/libs/constants.py b/src/libs/constants.py
index 6061e73..15fe060 100644
--- a/src/libs/constants.py
+++ b/src/libs/constants.py
@@ -60,4 +60,23 @@ class VariableMappingType(IntEnum):
class VariableMappingGroupType(IntEnum):
ENV = 1
# INPUTS = 2
- # OUTPUTS = 3
\ No newline at end of file
+ # OUTPUTS = 3
+
+class SecretVariableCategory(IntEnum):
+ NONE = 0
+ ACTIONS = 1
+ AGENTS = 2
+ CODESPACES = 3
+ DEPENDABOT = 4
+ ENVIRONMENTS = 5
+
+class SecretVariableType(IntEnum):
+ NONE = 0
+ SECRET = 1
+ VARIABLE = 2
+
+class SecretVariableVisibility(IntEnum):
+ NONE = 0
+ ALL = 1
+ PRIVATE = 2
+ SELECTED = 3
diff --git a/src/tests/conftest.py b/src/tests/conftest.py
index 9e81811..f1ae9e8 100644
--- a/src/tests/conftest.py
+++ b/src/tests/conftest.py
@@ -47,6 +47,8 @@ def client(logger):
@pytest.fixture
def database():
db_file = os.path.join(tempfile.gettempdir(), 'tests.db')
+ if os.path.isfile(db_file):
+ os.unlink(db_file)
database = Database(db_file)
yield database
|