From 19447a252066446f7ec8119b0d4720ec4446aa48 Mon Sep 17 00:00:00 2001 From: makrelas Date: Fri, 21 Oct 2022 14:20:17 +0200 Subject: [PATCH 01/17] ci: pip upgrade in container fix --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f11095bf..5eb8fd1f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ FROM base AS dev WORKDIR /workdir RUN apk add --no-cache gcc linux-headers musl-dev make RUN python -m venv /opt/venv -RUN pip install --upgrade pip +RUN python -m pip install --upgrade pip # ========= FROM dev AS deps From d2ddcfc0d6beb1a4ecf2d2fa2efa31a99a0685e5 Mon Sep 17 00:00:00 2001 From: makrelas Date: Fri, 21 Oct 2022 16:14:26 +0200 Subject: [PATCH 02/17] refactor: sync_apps refactor Move Root repository and Application Tenant configurations to be treated as object, including additional option of tenant repository traversal to obtain custom configurations. RootRepo and AppTennantConfig curenlty are explicitly defined in sync_apps.py as this is a POC to gather feedback about the direction to go. --- Makefile | 3 +- docs/commands/sync-apps.md | 2 + gitopscli/appconfig_api/__init__.py | 0 gitopscli/appconfig_api/app_tenant_config.py | 110 +++++++ gitopscli/appconfig_api/root_repo.py | 71 ++++ gitopscli/appconfig_api/traverse_config.py | 6 + gitopscli/commands/sync_apps.py | 329 +++++++++++++------ tests/commands/test_sync_apps.py | 90 +++-- 8 files changed, 481 insertions(+), 130 deletions(-) create mode 100644 gitopscli/appconfig_api/__init__.py create mode 100644 gitopscli/appconfig_api/app_tenant_config.py create mode 100644 gitopscli/appconfig_api/root_repo.py create mode 100644 gitopscli/appconfig_api/traverse_config.py diff --git a/Makefile b/Makefile index 4d95e881..5d637adf 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,8 @@ coverage: coverage html coverage report -checks: format-check lint mypy test +#checks: format-check lint mypy test +checks: format-check test image: DOCKER_BUILDKIT=1 docker build --progress=plain -t gitopscli:latest . diff --git a/docs/commands/sync-apps.md b/docs/commands/sync-apps.md index c0185d80..325b70f2 100644 --- a/docs/commands/sync-apps.md +++ b/docs/commands/sync-apps.md @@ -28,6 +28,8 @@ root-config-repo/ └── bootstrap └── values.yaml ``` +### app specific values +app specific values may be set using a values.yaml file directly in the app directory. gitopscli will process these values and remove key that would be blacklisted for security purpose and then store them in the result files under app key. **bootstrap/values.yaml** ```yaml diff --git a/gitopscli/appconfig_api/__init__.py b/gitopscli/appconfig_api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/gitopscli/appconfig_api/app_tenant_config.py b/gitopscli/appconfig_api/app_tenant_config.py new file mode 100644 index 00000000..5912d986 --- /dev/null +++ b/gitopscli/appconfig_api/app_tenant_config.py @@ -0,0 +1,110 @@ +import os +from dataclasses import dataclass +from ruamel.yaml import YAML +from gitopscli.appconfig_api.traverse_config import traverse_config +from gitopscli.io_api.yaml_util import yaml_file_load, yaml_load + + +@dataclass +class AppTenantConfig: + # TODO: rethink objects and class initialization methods + config_type: str # is instance initialized as config located in root/team repo + data: YAML + # schema important fields + # config - entrypoint + # config.repository - tenant repository url + # config.applications - tenant applications list + # config.applications.{}.userconfig - user configuration + name: str # tenant name + config_source_repository: str # team tenant repository url + # user_config: dict #contents of custom_tenant_config.yaml in team repository + file_path: str + file_name: str + config_api_version: tuple + + def __init__( + self, config_type, config_source_repository=None, data=None, name=None, file_path=None, file_name=None + ): + self.config_type = config_type + self.config_source_repository = config_source_repository + if self.config_type == "root": + self.data = data + self.file_path = file_path + self.file_name = file_name + elif self.config_type == "team": + self.config_source_repository.clone() + self.data = self.generate_config_from_team_repo() + self.config_api_version = self.__get_config_api_version() + self.name = name + + def __get_config_api_version(self): + # maybe count the keys? + if "config" in self.data.keys(): + return ("v2", ("config", "applications")) + return ("v1", ("applications",)) + + def generate_config_from_team_repo(self): + # recognize type of repo + team_config_git_repo = self.config_source_repository + repo_dir = team_config_git_repo.get_full_file_path(".") + applist = { + name + for name in os.listdir(repo_dir) + if os.path.isdir(os.path.join(repo_dir, name)) and not name.startswith(".") + } + # TODO: Create YAML() object without writing template strings + # Currently this is the easiest method, although it can be better + template_yaml = """ + config: + repository: {} + applications: {{}} + """.format( + team_config_git_repo.get_clone_url() + ) + data = yaml_load(template_yaml) + for app in applist: + template_yaml = """ + {}: {{}} + """.format( + app + ) + customconfig = self.get_custom_config(app) + app_as_yaml_object = yaml_load(template_yaml) + # dict path hardcoded as object generated will always be in v2 or later + data["config"]["applications"].update(app_as_yaml_object) + data["config"]["applications"][app].insert(1, "customAppConfig", customconfig) + + return data + + # TODO: rewrite! as config should be inside of the app folder + # TODO: method should contain all aps, not only one, requires rewriting of merging during root repo init + def get_custom_config(self, appname): + team_config_git_repo = self.config_source_repository + # try: + custom_config_file = team_config_git_repo.get_full_file_path(f"{appname}/app_value_file.yaml") + # except Exception as ex: + # handle missing file + # handle broken file/nod adhering to allowed + # return ex + # sanitize + # TODO: how to keep whole content with comments + # TODO: handling generic values for all apps + if os.path.exists(custom_config_file): + custom_config_content = yaml_file_load(custom_config_file) + return custom_config_content + return None + + def list_apps(self): + return traverse_config(self.data, self.config_api_version) + + def add_app(self): + # adds app to the app tenant config + pass + + def modify_app(self): + # modifies existing app in tenant config + pass + + def delete_app(self): + # deletes app from tenant config + pass diff --git a/gitopscli/appconfig_api/root_repo.py b/gitopscli/appconfig_api/root_repo.py new file mode 100644 index 00000000..470522e8 --- /dev/null +++ b/gitopscli/appconfig_api/root_repo.py @@ -0,0 +1,71 @@ +from dataclasses import dataclass +import logging +from typing import Any +from gitopscli.git_api import GitRepo +from gitopscli.gitops_exception import GitOpsException +from gitopscli.io_api.yaml_util import yaml_file_load +from gitopscli.appconfig_api.app_tenant_config import AppTenantConfig +from gitopscli.appconfig_api.traverse_config import traverse_config + + +@dataclass +class RootRepo: + name: str # root repository name + tenant_list: dict # TODO of AppTenantConfig #list of the tenant configs in the root repository (in apps folder) + bootstrap: set # list of tenants to be bootstrapped, derived form values.yaml in bootstrap root repo dict + app_list: set # llist of apps without custormer separation + + def __init__(self, root_config_git_repo: GitRepo): + repo_clone_url = root_config_git_repo.get_clone_url() + root_config_git_repo.clone() + bootstrap_values_file = root_config_git_repo.get_full_file_path("bootstrap/values.yaml") + self.bootstrap = self.__get_bootstrap_entries(bootstrap_values_file) + self.name = repo_clone_url.split("/")[-1].removesuffix(".git") + self.tenant_list = self.__generate_tenant_app_dict(root_config_git_repo) + self.app_list = self.__get_all_apps_list() + + def __get_bootstrap_entries(self, bootstrap_values_file: str) -> Any: + try: + bootstrap_yaml = yaml_file_load(bootstrap_values_file) + except FileNotFoundError as ex: + raise GitOpsException("File 'bootstrap/values.yaml' not found in root repository.") from ex + if "bootstrap" not in bootstrap_yaml: + raise GitOpsException("Cannot find key 'bootstrap' in 'bootstrap/values.yaml'") + for bootstrap_entry in bootstrap_yaml["bootstrap"]: + if "name" not in bootstrap_entry: + raise GitOpsException("Every bootstrap entry must have a 'name' property.") + return bootstrap_yaml["bootstrap"] + + def __generate_tenant_app_dict(self, root_config_git_repo: GitRepo): + tenant_app_dict = {} + for bootstrap_entry in self.bootstrap: + tenant_apps_config_file_name = "apps/" + bootstrap_entry["name"] + ".yaml" + logging.info("Analyzing %s in root repository", tenant_apps_config_file_name) + tenant_apps_config_file = root_config_git_repo.get_full_file_path(tenant_apps_config_file_name) + try: + tenant_apps_config_content = yaml_file_load(tenant_apps_config_file) + except FileNotFoundError as ex: + raise GitOpsException(f"File '{tenant_apps_config_file_name}' not found in root repository.") from ex + # TODO exception handling for malformed yaml + if "config" in tenant_apps_config_content: + tenant_apps_config_content = tenant_apps_config_content["config"] + if "repository" not in tenant_apps_config_content: + raise GitOpsException(f"Cannot find key 'repository' in '{tenant_apps_config_file_name}'") + # if "config" in tenant_apps_config_content: + logging.info("adding {}".format(bootstrap_entry["name"])) + atc = AppTenantConfig( + data=tenant_apps_config_content, + name=bootstrap_entry["name"], + config_type="root", + file_path=tenant_apps_config_file, + file_name=tenant_apps_config_file_name, + ) + tenant_app_dict.update({bootstrap_entry["name"]: atc}) + return tenant_app_dict + + def __get_all_apps_list(self): + all_apps_list = dict() + for tenant in self.tenant_list: + value = traverse_config(self.tenant_list[tenant].data, self.tenant_list[tenant].config_api_version) + all_apps_list.update({tenant: list((dict(value).keys()))}) + return all_apps_list diff --git a/gitopscli/appconfig_api/traverse_config.py b/gitopscli/appconfig_api/traverse_config.py new file mode 100644 index 00000000..7ea32421 --- /dev/null +++ b/gitopscli/appconfig_api/traverse_config.py @@ -0,0 +1,6 @@ +def traverse_config(data, configver): + path = configver[1] + lookup = data + for key in path: + lookup = lookup[key] + return lookup diff --git a/gitopscli/commands/sync_apps.py b/gitopscli/commands/sync_apps.py index 1121fc1e..bfa8b76d 100644 --- a/gitopscli/commands/sync_apps.py +++ b/gitopscli/commands/sync_apps.py @@ -1,13 +1,204 @@ import logging -import os from dataclasses import dataclass -from typing import Any, Set, Tuple from gitopscli.git_api import GitApiConfig, GitRepo, GitRepoApiFactory -from gitopscli.io_api.yaml_util import merge_yaml_element, yaml_file_load +from gitopscli.io_api.yaml_util import merge_yaml_element from gitopscli.gitops_exception import GitOpsException from .command import Command +# from gitopscli.appconfig_api.app_tenant_config import AppTenantConfig +# from gitopscli.appconfig_api.root_repo import RootRepo +# from gitopscli.appconfig_api.traverse_config import traverse_config + +########################################################################################### +######## TO BE REPLACED WITH IMPORT FROM appconfig_api +# TODO: Custom config reader +# TODO: Test custom config read, creation of objects AppTenantConfig and RootRepo +import os +from typing import Any +from dataclasses import dataclass +from ruamel.yaml import YAML +from gitopscli.appconfig_api.traverse_config import traverse_config +from gitopscli.io_api.yaml_util import yaml_file_load, yaml_load + + +@dataclass +class AppTenantConfig: + # TODO: rethink objects and class initialization methods + config_type: str # is instance initialized as config located in root/team repo + data: YAML + # schema important fields + # config - entrypoint + # config.repository - tenant repository url + # config.applications - tenant applications list + # config.applications.{}.userconfig - user configuration + name: str # tenant name + config_source_repository: str # team tenant repository url + # user_config: dict #contents of custom_tenant_config.yaml in team repository + file_path: str + file_name: str + config_api_version: tuple + + def __init__( + self, config_type, config_source_repository=None, data=None, name=None, file_path=None, file_name=None + ): + self.config_type = config_type + self.config_source_repository = config_source_repository + if self.config_type == "root": + self.data = data + self.file_path = file_path + self.file_name = file_name + elif self.config_type == "team": + self.config_source_repository.clone() + self.data = self.generate_config_from_team_repo() + self.config_api_version = self.__get_config_api_version() + self.name = name + + def __get_config_api_version(self): + # maybe count the keys? + if "config" in self.data.keys(): + return ("v2", ("config", "applications")) + return ("v1", ("applications",)) + + def generate_config_from_team_repo(self): + # recognize type of repo + team_config_git_repo = self.config_source_repository + repo_dir = team_config_git_repo.get_full_file_path(".") + applist = { + name + for name in os.listdir(repo_dir) + if os.path.isdir(os.path.join(repo_dir, name)) and not name.startswith(".") + } + # TODO: Create YAML() object without writing template strings + # Currently this is the easiest method, although it can be better + template_yaml = """ + config: + repository: {} + applications: {{}} + """.format( + team_config_git_repo.get_clone_url() + ) + data = yaml_load(template_yaml) + for app in applist: + template_yaml = """ + {}: {{}} + """.format( + app + ) + customconfig = self.get_custom_config(app) + app_as_yaml_object = yaml_load(template_yaml) + # dict path hardcoded as object generated will always be in v2 or later + data["config"]["applications"].update(app_as_yaml_object) + data["config"]["applications"][app].insert(1, "customAppConfig", customconfig) + + return data + + # TODO: rewrite! as config should be inside of the app folder + # TODO: method should contain all aps, not only one, requires rewriting of merging during root repo init + def get_custom_config(self, appname): + team_config_git_repo = self.config_source_repository + # try: + custom_config_file = team_config_git_repo.get_full_file_path(f"{appname}/app_value_file.yaml") + # except Exception as ex: + # handle missing file + # handle broken file/nod adhering to allowed + # return ex + # sanitize + # TODO: how to keep whole content with comments + # TODO: handling generic values for all apps + if os.path.exists(custom_config_file): + custom_config_content = yaml_file_load(custom_config_file) + return custom_config_content + return None + + def list_apps(self): + return traverse_config(self.data, self.config_api_version) + + def add_app(self): + # adds app to the app tenant config + pass + + def modify_app(self): + # modifies existing app in tenant config + pass + + def delete_app(self): + # deletes app from tenant config + pass + + +@dataclass +class RootRepo: + name: str # root repository name + tenant_list: dict # TODO of AppTenantConfig #list of the tenant configs in the root repository (in apps folder) + bootstrap: set # list of tenants to be bootstrapped, derived form values.yaml in bootstrap root repo dict + app_list: set # llist of apps without custormer separation + + def __init__(self, root_config_git_repo: GitRepo): + repo_clone_url = root_config_git_repo.get_clone_url() + root_config_git_repo.clone() + bootstrap_values_file = root_config_git_repo.get_full_file_path("bootstrap/values.yaml") + self.bootstrap = self.__get_bootstrap_entries(bootstrap_values_file) + self.name = repo_clone_url.split("/")[-1].removesuffix(".git") + self.tenant_list = self.__generate_tenant_app_dict(root_config_git_repo) + self.app_list = self.__get_all_apps_list() + + def __get_bootstrap_entries(self, bootstrap_values_file: str) -> Any: + try: + bootstrap_yaml = yaml_file_load(bootstrap_values_file) + except FileNotFoundError as ex: + raise GitOpsException("File 'bootstrap/values.yaml' not found in root repository.") from ex + if "bootstrap" not in bootstrap_yaml: + raise GitOpsException("Cannot find key 'bootstrap' in 'bootstrap/values.yaml'") + for bootstrap_entry in bootstrap_yaml["bootstrap"]: + if "name" not in bootstrap_entry: + raise GitOpsException("Every bootstrap entry must have a 'name' property.") + return bootstrap_yaml["bootstrap"] + + def __generate_tenant_app_dict(self, root_config_git_repo: GitRepo): + tenant_app_dict = {} + for bootstrap_entry in self.bootstrap: + tenant_apps_config_file_name = "apps/" + bootstrap_entry["name"] + ".yaml" + logging.info("Analyzing %s in root repository", tenant_apps_config_file_name) + tenant_apps_config_file = root_config_git_repo.get_full_file_path(tenant_apps_config_file_name) + try: + tenant_apps_config_content = yaml_file_load(tenant_apps_config_file) + except FileNotFoundError as ex: + raise GitOpsException(f"File '{tenant_apps_config_file_name}' not found in root repository.") from ex + # TODO exception handling for malformed yaml + if "config" in tenant_apps_config_content: + tenant_apps_config_content = tenant_apps_config_content["config"] + if "repository" not in tenant_apps_config_content: + raise GitOpsException(f"Cannot find key 'repository' in '{tenant_apps_config_file_name}'") + # if "config" in tenant_apps_config_content: + logging.info("adding {}".format(bootstrap_entry["name"])) + atc = AppTenantConfig( + data=tenant_apps_config_content, + name=bootstrap_entry["name"], + config_type="root", + file_path=tenant_apps_config_file, + file_name=tenant_apps_config_file_name, + ) + tenant_app_dict.update({bootstrap_entry["name"]: atc}) + return tenant_app_dict + + def __get_all_apps_list(self): + all_apps_list = dict() + for tenant in self.tenant_list: + value = traverse_config(self.tenant_list[tenant].data, self.tenant_list[tenant].config_api_version) + all_apps_list.update({tenant: list((dict(value).keys()))}) + return all_apps_list + + +def traverse_config(data, configver): + path = configver[1] + lookup = data + for key in path: + lookup = lookup[key] + return lookup + + +################################################################################################# class SyncAppsCommand(Command): @dataclass(frozen=True) class Args(GitApiConfig): @@ -38,80 +229,58 @@ def _sync_apps_command(args: SyncAppsCommand.Args) -> None: def __sync_apps(team_config_git_repo: GitRepo, root_config_git_repo: GitRepo, git_user: str, git_email: str) -> None: logging.info("Team config repository: %s", team_config_git_repo.get_clone_url()) logging.info("Root config repository: %s", root_config_git_repo.get_clone_url()) + team_config_app_name = team_config_git_repo.get_clone_url().split("/")[-1].removesuffix(".git") + root_repo = RootRepo(root_config_git_repo) + tenant_config_team_repo = AppTenantConfig("team", config_source_repository=team_config_git_repo) + + # dict conversion causes YAML object to be unordered + tenant_config_repo_apps = dict(tenant_config_team_repo.list_apps()) + if not team_config_app_name in list(root_repo.tenant_list.keys()): + raise GitOpsException("Couldn't find config file for apps repository in root repository's 'apps/' directory") + current_repo_apps = dict(root_repo.tenant_list[team_config_app_name].list_apps()) - repo_apps = __get_repo_apps(team_config_git_repo) - logging.info("Found %s app(s) in apps repository: %s", len(repo_apps), ", ".join(repo_apps)) + apps_from_other_repos = root_repo.app_list.copy() + apps_from_other_repos.pop(team_config_app_name) + for app in list(tenant_config_repo_apps.keys()): + for tenant in apps_from_other_repos.values(): + if app in tenant: + raise GitOpsException(f"Application '{app}' already exists in a different repository") + logging.info( + "Found %s app(s) in apps repository: %s", len(tenant_config_repo_apps), ", ".join(tenant_config_repo_apps) + ) logging.info("Searching apps repository in root repository's 'apps/' directory...") - ( - apps_config_file, - apps_config_file_name, - current_repo_apps, - apps_from_other_repos, - found_apps_path, - ) = __find_apps_config_from_repo(team_config_git_repo, root_config_git_repo) - if current_repo_apps == repo_apps: + apps_config_file = root_repo.tenant_list[team_config_app_name].file_path + apps_config_file_name = root_repo.tenant_list[team_config_app_name].file_name + # TODO FIX VALUE TO DIFFER BETWEEN OLD/NEW STYLE + found_apps_path = "config.applications" + + # removing all keys not being current app repo in order to compare app lists + # excluding keys added by root repo administrator, + # TODO: figure out how to handle that better + for app in list(current_repo_apps.keys()): + if current_repo_apps.get(app, dict()) is not None: + for key in list(current_repo_apps.get(app, dict())): + if key != "customAppConfig": + del current_repo_apps[app][key] + if current_repo_apps == tenant_config_repo_apps: logging.info("Root repository already up-to-date. I'm done here.") return - __check_if_app_already_exists(repo_apps, apps_from_other_repos) - logging.info("Sync applications in root repository's %s.", apps_config_file_name) - merge_yaml_element(apps_config_file, found_apps_path, {repo_app: {} for repo_app in repo_apps}) - __commit_and_push(team_config_git_repo, root_config_git_repo, git_user, git_email, apps_config_file_name) - - -def __find_apps_config_from_repo( - team_config_git_repo: GitRepo, root_config_git_repo: GitRepo -) -> Tuple[str, str, Set[str], Set[str], str]: - apps_from_other_repos: Set[str] = set() # Set for all entries in .applications from each config repository - found_app_config_file = None - found_app_config_file_name = None - found_apps_path = "applications" - found_app_config_apps: Set[str] = set() - bootstrap_entries = __get_bootstrap_entries(root_config_git_repo) - team_config_git_repo_clone_url = team_config_git_repo.get_clone_url() - for bootstrap_entry in bootstrap_entries: - if "name" not in bootstrap_entry: - raise GitOpsException("Every bootstrap entry must have a 'name' property.") - app_file_name = "apps/" + bootstrap_entry["name"] + ".yaml" - logging.info("Analyzing %s in root repository", app_file_name) - app_config_file = root_config_git_repo.get_full_file_path(app_file_name) - try: - app_config_content = yaml_file_load(app_config_file) - except FileNotFoundError as ex: - raise GitOpsException(f"File '{app_file_name}' not found in root repository.") from ex - if "config" in app_config_content: - app_config_content = app_config_content["config"] - found_apps_path = "config.applications" - if "repository" not in app_config_content: - raise GitOpsException(f"Cannot find key 'repository' in '{app_file_name}'") - if app_config_content["repository"] == team_config_git_repo_clone_url: - logging.info("Found apps repository in %s", app_file_name) - found_app_config_file = app_config_file - found_app_config_file_name = app_file_name - found_app_config_apps = __get_applications_from_app_config(app_config_content) - else: - apps_from_other_repos.update(__get_applications_from_app_config(app_config_content)) - - if found_app_config_file is None or found_app_config_file_name is None: - raise GitOpsException("Couldn't find config file for apps repository in root repository's 'apps/' directory") - - return ( - found_app_config_file, - found_app_config_file_name, - found_app_config_apps, - apps_from_other_repos, + merge_yaml_element( + apps_config_file, found_apps_path, + { + repo_app: traverse_config(tenant_config_team_repo.data, tenant_config_team_repo.config_api_version).get( + repo_app, "{}" + ) + for repo_app in tenant_config_repo_apps + }, ) - -def __get_applications_from_app_config(app_config: Any) -> Set[str]: - apps = [] - if "applications" in app_config and app_config["applications"] is not None: - apps += app_config["applications"].keys() - return set(apps) + __commit_and_push(team_config_git_repo, root_config_git_repo, git_user, git_email, apps_config_file_name) def __commit_and_push( @@ -120,31 +289,3 @@ def __commit_and_push( author = team_config_git_repo.get_author_from_last_commit() root_config_git_repo.commit(git_user, git_email, f"{author} updated " + app_file_name) root_config_git_repo.push() - - -def __get_bootstrap_entries(root_config_git_repo: GitRepo) -> Any: - root_config_git_repo.clone() - bootstrap_values_file = root_config_git_repo.get_full_file_path("bootstrap/values.yaml") - try: - bootstrap_yaml = yaml_file_load(bootstrap_values_file) - except FileNotFoundError as ex: - raise GitOpsException("File 'bootstrap/values.yaml' not found in root repository.") from ex - if "bootstrap" not in bootstrap_yaml: - raise GitOpsException("Cannot find key 'bootstrap' in 'bootstrap/values.yaml'") - return bootstrap_yaml["bootstrap"] - - -def __get_repo_apps(team_config_git_repo: GitRepo) -> Set[str]: - team_config_git_repo.clone() - repo_dir = team_config_git_repo.get_full_file_path(".") - return { - name - for name in os.listdir(repo_dir) - if os.path.isdir(os.path.join(repo_dir, name)) and not name.startswith(".") - } - - -def __check_if_app_already_exists(apps_dirs: Set[str], apps_from_other_repos: Set[str]) -> None: - for app_key in apps_dirs: - if app_key in apps_from_other_repos: - raise GitOpsException(f"Application '{app_key}' already exists in a different repository") diff --git a/tests/commands/test_sync_apps.py b/tests/commands/test_sync_apps.py index 590dabf7..27dbbd9d 100644 --- a/tests/commands/test_sync_apps.py +++ b/tests/commands/test_sync_apps.py @@ -1,3 +1,4 @@ +import posixpath import logging import os import unittest @@ -6,6 +7,9 @@ from gitopscli.commands.sync_apps import SyncAppsCommand from gitopscli.io_api.yaml_util import merge_yaml_element, yaml_file_load from gitopscli.gitops_exception import GitOpsException + +# from collections import OrderedDict as ordereddict +from ruamel.yaml.compat import ordereddict from .mock_mixin import MockMixin ARGS = SyncAppsCommand.Args( @@ -28,8 +32,9 @@ def setUp(self): self.os_mock = self.monkey_patch(os) self.os_mock.path.isdir.return_value = True - self.os_mock.path.join.side_effect = os.path.join + self.os_mock.path.join.side_effect = posixpath.join # tests are designed to emulate posix env self.os_mock.listdir.return_value = ["my-app"] + self.os_mock.path.exists.return_value = False self.logging_mock = self.monkey_patch(logging) self.logging_mock.info.return_value = None @@ -40,7 +45,7 @@ def setUp(self): self.team_config_git_repo_mock = self.create_mock(GitRepo, "GitRepo_team") self.team_config_git_repo_mock.__enter__.return_value = self.team_config_git_repo_mock self.team_config_git_repo_mock.__exit__.return_value = False - self.team_config_git_repo_mock.get_clone_url.return_value = "https://team.config.repo.git" + self.team_config_git_repo_mock.get_clone_url.return_value = "https://repository.url/team/team-non-prod.git" self.team_config_git_repo_mock.clone.return_value = None self.team_config_git_repo_mock.get_full_file_path.side_effect = lambda x: f"/tmp/team-config-repo/{x}" self.team_config_git_repo_mock.get_author_from_last_commit.return_value = "author" @@ -49,7 +54,7 @@ def setUp(self): self.root_config_git_repo_mock.__enter__.return_value = self.root_config_git_repo_mock self.root_config_git_repo_mock.__exit__.return_value = False self.root_config_git_repo_mock.get_full_file_path.side_effect = lambda x: f"/tmp/root-config-repo/{x}" - self.root_config_git_repo_mock.get_clone_url.return_value = "https://root.config.repo.git" + self.root_config_git_repo_mock.get_clone_url.return_value = "https://repository.url/root/root-config.git" self.root_config_git_repo_mock.clone.return_value = None self.root_config_git_repo_mock.commit.return_value = None self.root_config_git_repo_mock.push.return_value = None @@ -72,10 +77,13 @@ def setUp(self): "bootstrap": [{"name": "team-non-prod"}, {"name": "other-team-non-prod"}], }, "/tmp/root-config-repo/apps/team-non-prod.yaml": { - "config": {"repository": "https://team.config.repo.git", "applications": {"some-other-app-1": None}} + "config": { + "repository": "https://repository.url/team/team-non-prod.git", + "applications": {"some-other-app-1": None}, + } }, "/tmp/root-config-repo/apps/other-team-non-prod.yaml": { - "repository": "https://other-team.config.repo.git", + "repository": "https://repository.url/other-team/other-team-non-prod.git", "applications": {"some-other-app-2": None}, }, }[file_path] @@ -93,30 +101,37 @@ def test_sync_apps_happy_flow(self): call.GitRepo(self.team_config_git_repo_api_mock), call.GitRepo(self.root_config_git_repo_api_mock), call.GitRepo_team.get_clone_url(), - call.logging.info("Team config repository: %s", "https://team.config.repo.git"), + call.logging.info("Team config repository: %s", "https://repository.url/team/team-non-prod.git"), + call.GitRepo_root.get_clone_url(), + call.logging.info("Root config repository: %s", "https://repository.url/root/root-config.git"), + call.GitRepo_team.get_clone_url(), call.GitRepo_root.get_clone_url(), - call.logging.info("Root config repository: %s", "https://root.config.repo.git"), - call.GitRepo_team.clone(), - call.GitRepo_team.get_full_file_path("."), - call.os.listdir("/tmp/team-config-repo/."), - call.os.path.join("/tmp/team-config-repo/.", "my-app"), - call.os.path.isdir("/tmp/team-config-repo/./my-app"), - call.logging.info("Found %s app(s) in apps repository: %s", 1, "my-app"), - call.logging.info("Searching apps repository in root repository's 'apps/' directory..."), call.GitRepo_root.clone(), call.GitRepo_root.get_full_file_path("bootstrap/values.yaml"), call.yaml_file_load("/tmp/root-config-repo/bootstrap/values.yaml"), - call.GitRepo_team.get_clone_url(), call.logging.info("Analyzing %s in root repository", "apps/team-non-prod.yaml"), call.GitRepo_root.get_full_file_path("apps/team-non-prod.yaml"), call.yaml_file_load("/tmp/root-config-repo/apps/team-non-prod.yaml"), - call.logging.info("Found apps repository in %s", "apps/team-non-prod.yaml"), + call.logging.info("adding team-non-prod"), call.logging.info("Analyzing %s in root repository", "apps/other-team-non-prod.yaml"), call.GitRepo_root.get_full_file_path("apps/other-team-non-prod.yaml"), call.yaml_file_load("/tmp/root-config-repo/apps/other-team-non-prod.yaml"), + call.logging.info("adding other-team-non-prod"), + call.GitRepo_team.clone(), + call.GitRepo_team.get_full_file_path("."), + call.os.listdir("/tmp/team-config-repo/."), + call.os.path.join("/tmp/team-config-repo/.", "my-app"), + call.os.path.isdir("/tmp/team-config-repo/./my-app"), + call.GitRepo_team.get_clone_url(), + call.GitRepo_team.get_full_file_path("my-app/app_value_file.yaml"), + call.os.path.exists("/tmp/team-config-repo/my-app/app_value_file.yaml"), + call.logging.info("Found %s app(s) in apps repository: %s", 1, "my-app"), + call.logging.info("Searching apps repository in root repository's 'apps/' directory..."), call.logging.info("Sync applications in root repository's %s.", "apps/team-non-prod.yaml"), call.merge_yaml_element( - "/tmp/root-config-repo/apps/team-non-prod.yaml", "config.applications", {"my-app": {}} + "/tmp/root-config-repo/apps/team-non-prod.yaml", + "config.applications", + {"my-app": ordereddict([("customAppConfig", None)])}, ), call.GitRepo_team.get_author_from_last_commit(), call.GitRepo_root.commit("GIT_USER", "GIT_EMAIL", "author updated apps/team-non-prod.yaml"), @@ -129,11 +144,11 @@ def test_sync_apps_already_up_to_date(self): "bootstrap": [{"name": "team-non-prod"}, {"name": "other-team-non-prod"}], }, "/tmp/root-config-repo/apps/team-non-prod.yaml": { - "repository": "https://team.config.repo.git", - "applications": {"my-app": None}, # my-app already exists + "repository": "https://repository.url/team/team-non-prod.git", + "applications": {"my-app": ordereddict([("customAppConfig", None)])}, # my-app already exists }, "/tmp/root-config-repo/apps/other-team-non-prod.yaml": { - "repository": "https://other-team.config.repo.git", + "repository": "https://repository.url/other-team/other-team-non-prod.git", "applications": {}, }, }[file_path] @@ -145,27 +160,32 @@ def test_sync_apps_already_up_to_date(self): call.GitRepo(self.team_config_git_repo_api_mock), call.GitRepo(self.root_config_git_repo_api_mock), call.GitRepo_team.get_clone_url(), - call.logging.info("Team config repository: %s", "https://team.config.repo.git"), + call.logging.info("Team config repository: %s", "https://repository.url/team/team-non-prod.git"), + call.GitRepo_root.get_clone_url(), + call.logging.info("Root config repository: %s", "https://repository.url/root/root-config.git"), + call.GitRepo_team.get_clone_url(), call.GitRepo_root.get_clone_url(), - call.logging.info("Root config repository: %s", "https://root.config.repo.git"), - call.GitRepo_team.clone(), - call.GitRepo_team.get_full_file_path("."), - call.os.listdir("/tmp/team-config-repo/."), - call.os.path.join("/tmp/team-config-repo/.", "my-app"), - call.os.path.isdir("/tmp/team-config-repo/./my-app"), - call.logging.info("Found %s app(s) in apps repository: %s", 1, "my-app"), - call.logging.info("Searching apps repository in root repository's 'apps/' directory..."), call.GitRepo_root.clone(), call.GitRepo_root.get_full_file_path("bootstrap/values.yaml"), call.yaml_file_load("/tmp/root-config-repo/bootstrap/values.yaml"), - call.GitRepo_team.get_clone_url(), call.logging.info("Analyzing %s in root repository", "apps/team-non-prod.yaml"), call.GitRepo_root.get_full_file_path("apps/team-non-prod.yaml"), call.yaml_file_load("/tmp/root-config-repo/apps/team-non-prod.yaml"), - call.logging.info("Found apps repository in %s", "apps/team-non-prod.yaml"), + call.logging.info("adding team-non-prod"), call.logging.info("Analyzing %s in root repository", "apps/other-team-non-prod.yaml"), call.GitRepo_root.get_full_file_path("apps/other-team-non-prod.yaml"), call.yaml_file_load("/tmp/root-config-repo/apps/other-team-non-prod.yaml"), + call.logging.info("adding other-team-non-prod"), + call.GitRepo_team.clone(), + call.GitRepo_team.get_full_file_path("."), + call.os.listdir("/tmp/team-config-repo/."), + call.os.path.join("/tmp/team-config-repo/.", "my-app"), + call.os.path.isdir("/tmp/team-config-repo/./my-app"), + call.GitRepo_team.get_clone_url(), + call.GitRepo_team.get_full_file_path("my-app/app_value_file.yaml"), + call.os.path.exists("/tmp/team-config-repo/my-app/app_value_file.yaml"), + call.logging.info("Found %s app(s) in apps repository: %s", 1, "my-app"), + call.logging.info("Searching apps repository in root repository's 'apps/' directory..."), call.logging.info("Root repository already up-to-date. I'm done here."), ] @@ -236,7 +256,7 @@ def test_sync_apps_missing_repository_element_in_team_yaml(self): self.yaml_file_load_mock.side_effect = lambda file_path: { "/tmp/root-config-repo/bootstrap/values.yaml": {"bootstrap": [{"name": "team-non-prod"}]}, "/tmp/root-config-repo/apps/team-non-prod.yaml": { - # missing: "repository": "https://team.config.repo.git", + # missing: "repository": "https://repository.url/team/team-non-prod.git", "applications": {}, }, }[file_path] @@ -251,7 +271,7 @@ def test_sync_apps_undefined_team_repo(self): self.yaml_file_load_mock.side_effect = lambda file_path: { "/tmp/root-config-repo/bootstrap/values.yaml": {"bootstrap": [{"name": "other-team-non-prod"}]}, "/tmp/root-config-repo/apps/other-team-non-prod.yaml": { - "repository": "https://other-team.config.repo.git", # there is no repo matching the command's team repo + "repository": "https://repository.url/other-team/other-team-non-prod.git", # there is no repo matching the command's team repo "applications": {}, }, }[file_path] @@ -270,11 +290,11 @@ def test_sync_apps_app_name_collission(self): "bootstrap": [{"name": "team-non-prod"}, {"name": "other-team-non-prod"}], }, "/tmp/root-config-repo/apps/team-non-prod.yaml": { - "repository": "https://team.config.repo.git", + "repository": "https://repository.url/team/team-non-prod.git", "applications": {"some-other-app-1": None}, }, "/tmp/root-config-repo/apps/other-team-non-prod.yaml": { - "repository": "https://other-team.config.repo.git", + "repository": "https://repository.url/other-team/other-team-non-prod.git", "applications": {"my-app": None}, # the other-team already has an app named "my-app" }, }[file_path] From 352a146480871b27acbe08710d77892e8eac15e0 Mon Sep 17 00:00:00 2001 From: makrelas Date: Mon, 7 Nov 2022 17:31:21 +0100 Subject: [PATCH 03/17] fix: compatibility between old and new style app-tenant configurations --- gitopscli/commands/sync_apps.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/gitopscli/commands/sync_apps.py b/gitopscli/commands/sync_apps.py index bfa8b76d..4e95a533 100644 --- a/gitopscli/commands/sync_apps.py +++ b/gitopscli/commands/sync_apps.py @@ -38,9 +38,10 @@ class AppTenantConfig: file_path: str file_name: str config_api_version: tuple + def __init__( - self, config_type, config_source_repository=None, data=None, name=None, file_path=None, file_name=None + self, config_type, config_source_repository=None, data=None, name=None, file_path=None, file_name=None, found_apps_path=None ): self.config_type = config_type self.config_source_repository = config_source_repository @@ -53,6 +54,7 @@ def __init__( self.data = self.generate_config_from_team_repo() self.config_api_version = self.__get_config_api_version() self.name = name + self.found_apps_path = found_apps_path def __get_config_api_version(self): # maybe count the keys? @@ -166,8 +168,10 @@ def __generate_tenant_app_dict(self, root_config_git_repo: GitRepo): except FileNotFoundError as ex: raise GitOpsException(f"File '{tenant_apps_config_file_name}' not found in root repository.") from ex # TODO exception handling for malformed yaml + found_apps_path = "applications" if "config" in tenant_apps_config_content: tenant_apps_config_content = tenant_apps_config_content["config"] + found_apps_path = "config.applications" if "repository" not in tenant_apps_config_content: raise GitOpsException(f"Cannot find key 'repository' in '{tenant_apps_config_file_name}'") # if "config" in tenant_apps_config_content: @@ -178,6 +182,7 @@ def __generate_tenant_app_dict(self, root_config_git_repo: GitRepo): config_type="root", file_path=tenant_apps_config_file, file_name=tenant_apps_config_file_name, + found_apps_path=found_apps_path ) tenant_app_dict.update({bootstrap_entry["name"]: atc}) return tenant_app_dict @@ -252,9 +257,7 @@ def __sync_apps(team_config_git_repo: GitRepo, root_config_git_repo: GitRepo, gi logging.info("Searching apps repository in root repository's 'apps/' directory...") apps_config_file = root_repo.tenant_list[team_config_app_name].file_path - apps_config_file_name = root_repo.tenant_list[team_config_app_name].file_name - # TODO FIX VALUE TO DIFFER BETWEEN OLD/NEW STYLE - found_apps_path = "config.applications" + apps_config_file_name = root_repo.tenant_list[team_config_app_name].file_name # removing all keys not being current app repo in order to compare app lists # excluding keys added by root repo administrator, @@ -271,7 +274,7 @@ def __sync_apps(team_config_git_repo: GitRepo, root_config_git_repo: GitRepo, gi logging.info("Sync applications in root repository's %s.", apps_config_file_name) merge_yaml_element( apps_config_file, - found_apps_path, + root_repo.tenant_list[team_config_app_name].found_apps_path, { repo_app: traverse_config(tenant_config_team_repo.data, tenant_config_team_repo.config_api_version).get( repo_app, "{}" From ce5d5429f0c423d6cf5a6e01f8b0f6a6caa119f8 Mon Sep 17 00:00:00 2001 From: makrelas Date: Thu, 17 Nov 2022 14:31:09 +0100 Subject: [PATCH 04/17] removed currently not working appconfig_api located classes --- gitopscli/appconfig_api/app_tenant_config.py | 110 ------------------- gitopscli/appconfig_api/root_repo.py | 71 ------------ 2 files changed, 181 deletions(-) delete mode 100644 gitopscli/appconfig_api/app_tenant_config.py delete mode 100644 gitopscli/appconfig_api/root_repo.py diff --git a/gitopscli/appconfig_api/app_tenant_config.py b/gitopscli/appconfig_api/app_tenant_config.py deleted file mode 100644 index 5912d986..00000000 --- a/gitopscli/appconfig_api/app_tenant_config.py +++ /dev/null @@ -1,110 +0,0 @@ -import os -from dataclasses import dataclass -from ruamel.yaml import YAML -from gitopscli.appconfig_api.traverse_config import traverse_config -from gitopscli.io_api.yaml_util import yaml_file_load, yaml_load - - -@dataclass -class AppTenantConfig: - # TODO: rethink objects and class initialization methods - config_type: str # is instance initialized as config located in root/team repo - data: YAML - # schema important fields - # config - entrypoint - # config.repository - tenant repository url - # config.applications - tenant applications list - # config.applications.{}.userconfig - user configuration - name: str # tenant name - config_source_repository: str # team tenant repository url - # user_config: dict #contents of custom_tenant_config.yaml in team repository - file_path: str - file_name: str - config_api_version: tuple - - def __init__( - self, config_type, config_source_repository=None, data=None, name=None, file_path=None, file_name=None - ): - self.config_type = config_type - self.config_source_repository = config_source_repository - if self.config_type == "root": - self.data = data - self.file_path = file_path - self.file_name = file_name - elif self.config_type == "team": - self.config_source_repository.clone() - self.data = self.generate_config_from_team_repo() - self.config_api_version = self.__get_config_api_version() - self.name = name - - def __get_config_api_version(self): - # maybe count the keys? - if "config" in self.data.keys(): - return ("v2", ("config", "applications")) - return ("v1", ("applications",)) - - def generate_config_from_team_repo(self): - # recognize type of repo - team_config_git_repo = self.config_source_repository - repo_dir = team_config_git_repo.get_full_file_path(".") - applist = { - name - for name in os.listdir(repo_dir) - if os.path.isdir(os.path.join(repo_dir, name)) and not name.startswith(".") - } - # TODO: Create YAML() object without writing template strings - # Currently this is the easiest method, although it can be better - template_yaml = """ - config: - repository: {} - applications: {{}} - """.format( - team_config_git_repo.get_clone_url() - ) - data = yaml_load(template_yaml) - for app in applist: - template_yaml = """ - {}: {{}} - """.format( - app - ) - customconfig = self.get_custom_config(app) - app_as_yaml_object = yaml_load(template_yaml) - # dict path hardcoded as object generated will always be in v2 or later - data["config"]["applications"].update(app_as_yaml_object) - data["config"]["applications"][app].insert(1, "customAppConfig", customconfig) - - return data - - # TODO: rewrite! as config should be inside of the app folder - # TODO: method should contain all aps, not only one, requires rewriting of merging during root repo init - def get_custom_config(self, appname): - team_config_git_repo = self.config_source_repository - # try: - custom_config_file = team_config_git_repo.get_full_file_path(f"{appname}/app_value_file.yaml") - # except Exception as ex: - # handle missing file - # handle broken file/nod adhering to allowed - # return ex - # sanitize - # TODO: how to keep whole content with comments - # TODO: handling generic values for all apps - if os.path.exists(custom_config_file): - custom_config_content = yaml_file_load(custom_config_file) - return custom_config_content - return None - - def list_apps(self): - return traverse_config(self.data, self.config_api_version) - - def add_app(self): - # adds app to the app tenant config - pass - - def modify_app(self): - # modifies existing app in tenant config - pass - - def delete_app(self): - # deletes app from tenant config - pass diff --git a/gitopscli/appconfig_api/root_repo.py b/gitopscli/appconfig_api/root_repo.py deleted file mode 100644 index 470522e8..00000000 --- a/gitopscli/appconfig_api/root_repo.py +++ /dev/null @@ -1,71 +0,0 @@ -from dataclasses import dataclass -import logging -from typing import Any -from gitopscli.git_api import GitRepo -from gitopscli.gitops_exception import GitOpsException -from gitopscli.io_api.yaml_util import yaml_file_load -from gitopscli.appconfig_api.app_tenant_config import AppTenantConfig -from gitopscli.appconfig_api.traverse_config import traverse_config - - -@dataclass -class RootRepo: - name: str # root repository name - tenant_list: dict # TODO of AppTenantConfig #list of the tenant configs in the root repository (in apps folder) - bootstrap: set # list of tenants to be bootstrapped, derived form values.yaml in bootstrap root repo dict - app_list: set # llist of apps without custormer separation - - def __init__(self, root_config_git_repo: GitRepo): - repo_clone_url = root_config_git_repo.get_clone_url() - root_config_git_repo.clone() - bootstrap_values_file = root_config_git_repo.get_full_file_path("bootstrap/values.yaml") - self.bootstrap = self.__get_bootstrap_entries(bootstrap_values_file) - self.name = repo_clone_url.split("/")[-1].removesuffix(".git") - self.tenant_list = self.__generate_tenant_app_dict(root_config_git_repo) - self.app_list = self.__get_all_apps_list() - - def __get_bootstrap_entries(self, bootstrap_values_file: str) -> Any: - try: - bootstrap_yaml = yaml_file_load(bootstrap_values_file) - except FileNotFoundError as ex: - raise GitOpsException("File 'bootstrap/values.yaml' not found in root repository.") from ex - if "bootstrap" not in bootstrap_yaml: - raise GitOpsException("Cannot find key 'bootstrap' in 'bootstrap/values.yaml'") - for bootstrap_entry in bootstrap_yaml["bootstrap"]: - if "name" not in bootstrap_entry: - raise GitOpsException("Every bootstrap entry must have a 'name' property.") - return bootstrap_yaml["bootstrap"] - - def __generate_tenant_app_dict(self, root_config_git_repo: GitRepo): - tenant_app_dict = {} - for bootstrap_entry in self.bootstrap: - tenant_apps_config_file_name = "apps/" + bootstrap_entry["name"] + ".yaml" - logging.info("Analyzing %s in root repository", tenant_apps_config_file_name) - tenant_apps_config_file = root_config_git_repo.get_full_file_path(tenant_apps_config_file_name) - try: - tenant_apps_config_content = yaml_file_load(tenant_apps_config_file) - except FileNotFoundError as ex: - raise GitOpsException(f"File '{tenant_apps_config_file_name}' not found in root repository.") from ex - # TODO exception handling for malformed yaml - if "config" in tenant_apps_config_content: - tenant_apps_config_content = tenant_apps_config_content["config"] - if "repository" not in tenant_apps_config_content: - raise GitOpsException(f"Cannot find key 'repository' in '{tenant_apps_config_file_name}'") - # if "config" in tenant_apps_config_content: - logging.info("adding {}".format(bootstrap_entry["name"])) - atc = AppTenantConfig( - data=tenant_apps_config_content, - name=bootstrap_entry["name"], - config_type="root", - file_path=tenant_apps_config_file, - file_name=tenant_apps_config_file_name, - ) - tenant_app_dict.update({bootstrap_entry["name"]: atc}) - return tenant_app_dict - - def __get_all_apps_list(self): - all_apps_list = dict() - for tenant in self.tenant_list: - value = traverse_config(self.tenant_list[tenant].data, self.tenant_list[tenant].config_api_version) - all_apps_list.update({tenant: list((dict(value).keys()))}) - return all_apps_list From 164f8023adf2991a5fb1742df4b80add462d7f2a Mon Sep 17 00:00:00 2001 From: makrelas Date: Fri, 25 Nov 2022 18:38:52 +0100 Subject: [PATCH 05/17] added AppTenant and RootRepo factories to separate data from logic --- gitopscli/commands/sync_apps.py | 176 +++++++++++++++++--------------- 1 file changed, 91 insertions(+), 85 deletions(-) diff --git a/gitopscli/commands/sync_apps.py b/gitopscli/commands/sync_apps.py index 4e95a533..f074d5ff 100644 --- a/gitopscli/commands/sync_apps.py +++ b/gitopscli/commands/sync_apps.py @@ -22,49 +22,10 @@ from gitopscli.io_api.yaml_util import yaml_file_load, yaml_load -@dataclass -class AppTenantConfig: - # TODO: rethink objects and class initialization methods - config_type: str # is instance initialized as config located in root/team repo - data: YAML - # schema important fields - # config - entrypoint - # config.repository - tenant repository url - # config.applications - tenant applications list - # config.applications.{}.userconfig - user configuration - name: str # tenant name - config_source_repository: str # team tenant repository url - # user_config: dict #contents of custom_tenant_config.yaml in team repository - file_path: str - file_name: str - config_api_version: tuple - - - def __init__( - self, config_type, config_source_repository=None, data=None, name=None, file_path=None, file_name=None, found_apps_path=None - ): - self.config_type = config_type - self.config_source_repository = config_source_repository - if self.config_type == "root": - self.data = data - self.file_path = file_path - self.file_name = file_name - elif self.config_type == "team": - self.config_source_repository.clone() - self.data = self.generate_config_from_team_repo() - self.config_api_version = self.__get_config_api_version() - self.name = name - self.found_apps_path = found_apps_path - - def __get_config_api_version(self): - # maybe count the keys? - if "config" in self.data.keys(): - return ("v2", ("config", "applications")) - return ("v1", ("applications",)) - - def generate_config_from_team_repo(self): - # recognize type of repo - team_config_git_repo = self.config_source_repository +class AppTenantConfigFactory: + def generate_config_from_team_repo( + self, team_config_git_repo: GitRepo + ) -> Any: # TODO: supposed to be ordereddict than Any repo_dir = team_config_git_repo.get_full_file_path(".") applist = { name @@ -75,8 +36,8 @@ def generate_config_from_team_repo(self): # Currently this is the easiest method, although it can be better template_yaml = """ config: - repository: {} - applications: {{}} + repository: {} + applications: {{}} """.format( team_config_git_repo.get_clone_url() ) @@ -87,18 +48,17 @@ def generate_config_from_team_repo(self): """.format( app ) - customconfig = self.get_custom_config(app) + customconfig = self.get_custom_config(app, team_config_git_repo) app_as_yaml_object = yaml_load(template_yaml) # dict path hardcoded as object generated will always be in v2 or later data["config"]["applications"].update(app_as_yaml_object) data["config"]["applications"][app].insert(1, "customAppConfig", customconfig) - return data - # TODO: rewrite! as config should be inside of the app folder # TODO: method should contain all aps, not only one, requires rewriting of merging during root repo init - def get_custom_config(self, appname): - team_config_git_repo = self.config_source_repository + def get_custom_config(self, appname, team_config_git_repo ) -> str | None: + team_config_git_repo = team_config_git_repo + # try: custom_config_file = team_config_git_repo.get_full_file_path(f"{appname}/app_value_file.yaml") # except Exception as ex: @@ -113,38 +73,62 @@ def get_custom_config(self, appname): return custom_config_content return None - def list_apps(self): + def create( + self, + config_type: str, + name: str, + data: str = None, + file_path: str = None, + file_name: str = None, + config_source_repository: GitRepo = None, + found_apps_path: str = None + ): + if config_type == "root": + return AppTenantConfig(config_type, data, name, file_path, file_name, found_apps_path=found_apps_path) + + elif config_type == "team": + config_source_repository.clone() + data = self.generate_config_from_team_repo(config_source_repository) + return AppTenantConfig(config_type, data, name, config_source_repository=config_source_repository.get_clone_url()) + + +@dataclass +class AppTenantConfig: + config_type: str # is instance initialized as config located in root/team repo + data: Any # TODO: supposed to be ordereddict from ruamel + name: str # tenant name + file_path: str = None + file_name: str = None + config_source_repository: str = None # team tenant repository url + config_api_version: tuple[Any, ...] = None + found_apps_path: str = None + + def __post_init__(self): + self.config_api_version = self.__get_config_api_version() + + def __get_config_api_version(self) -> tuple[Any, ...]: + #NOT WORKING AS SHOULD, SHOILD REPLACE MANUAL FOUND_APPS_PATH maybe count the keys? + if "config" in self.data.keys(): + return ("v2", ("config", "applications")) + return ("v1", ("applications",)) + + def list_apps(self) -> None: return traverse_config(self.data, self.config_api_version) - def add_app(self): + def add_app(self) -> None: # adds app to the app tenant config pass - def modify_app(self): + def modify_app(self) -> None: # modifies existing app in tenant config pass - def delete_app(self): + def delete_app(self) -> None: # deletes app from tenant config pass -@dataclass -class RootRepo: - name: str # root repository name - tenant_list: dict # TODO of AppTenantConfig #list of the tenant configs in the root repository (in apps folder) - bootstrap: set # list of tenants to be bootstrapped, derived form values.yaml in bootstrap root repo dict - app_list: set # llist of apps without custormer separation - - def __init__(self, root_config_git_repo: GitRepo): - repo_clone_url = root_config_git_repo.get_clone_url() - root_config_git_repo.clone() - bootstrap_values_file = root_config_git_repo.get_full_file_path("bootstrap/values.yaml") - self.bootstrap = self.__get_bootstrap_entries(bootstrap_values_file) - self.name = repo_clone_url.split("/")[-1].removesuffix(".git") - self.tenant_list = self.__generate_tenant_app_dict(root_config_git_repo) - self.app_list = self.__get_all_apps_list() - +class RootRepoFactory: def __get_bootstrap_entries(self, bootstrap_values_file: str) -> Any: try: bootstrap_yaml = yaml_file_load(bootstrap_values_file) @@ -157,9 +141,9 @@ def __get_bootstrap_entries(self, bootstrap_values_file: str) -> Any: raise GitOpsException("Every bootstrap entry must have a 'name' property.") return bootstrap_yaml["bootstrap"] - def __generate_tenant_app_dict(self, root_config_git_repo: GitRepo): + def __generate_tenant_app_dict_from_root_repo(self, root_config_git_repo: GitRepo, bootstrap: Any): tenant_app_dict = {} - for bootstrap_entry in self.bootstrap: + for bootstrap_entry in bootstrap: tenant_apps_config_file_name = "apps/" + bootstrap_entry["name"] + ".yaml" logging.info("Analyzing %s in root repository", tenant_apps_config_file_name) tenant_apps_config_file = root_config_git_repo.get_full_file_path(tenant_apps_config_file_name) @@ -174,9 +158,8 @@ def __generate_tenant_app_dict(self, root_config_git_repo: GitRepo): found_apps_path = "config.applications" if "repository" not in tenant_apps_config_content: raise GitOpsException(f"Cannot find key 'repository' in '{tenant_apps_config_file_name}'") - # if "config" in tenant_apps_config_content: logging.info("adding {}".format(bootstrap_entry["name"])) - atc = AppTenantConfig( + atc = AppTenantConfigFactory().create( data=tenant_apps_config_content, name=bootstrap_entry["name"], config_type="root", @@ -187,13 +170,32 @@ def __generate_tenant_app_dict(self, root_config_git_repo: GitRepo): tenant_app_dict.update({bootstrap_entry["name"]: atc}) return tenant_app_dict - def __get_all_apps_list(self): + # TODO SHOULD THIS FULL METHOD INSTEAD OF POPULATING + def __get_all_apps_list(self, tenant_dict: Any): all_apps_list = dict() - for tenant in self.tenant_list: - value = traverse_config(self.tenant_list[tenant].data, self.tenant_list[tenant].config_api_version) + for tenant in tenant_dict: + value = traverse_config(tenant_dict[tenant].data, tenant_dict[tenant].config_api_version) all_apps_list.update({tenant: list((dict(value).keys()))}) return all_apps_list + def create(self, root_repo: GitRepo): + name = root_repo.get_clone_url().split("/")[-1].removesuffix(".git") + root_repo.clone() + bootstrap_values_file = root_repo.get_full_file_path("bootstrap/values.yaml") + bootstrap = self.__get_bootstrap_entries(bootstrap_values_file) + tenant_dict = self.__generate_tenant_app_dict_from_root_repo(root_repo, bootstrap) + + all_app_list = self.__get_all_apps_list(tenant_dict) + return RootRepo(name, tenant_dict, bootstrap, all_app_list) + + +@dataclass +class RootRepo: + name: str # root repository name + tenant_dict: dict # TODO of AppTenantConfig #list of the tenant configs in the root repository (in apps folder) + bootstrap: set # list of tenants to be bootstrapped, derived form values.yaml in bootstrap root repo dict + all_app_list: set # list of apps without custormer separation + def traverse_config(data, configver): path = configver[1] @@ -234,17 +236,21 @@ def _sync_apps_command(args: SyncAppsCommand.Args) -> None: def __sync_apps(team_config_git_repo: GitRepo, root_config_git_repo: GitRepo, git_user: str, git_email: str) -> None: logging.info("Team config repository: %s", team_config_git_repo.get_clone_url()) logging.info("Root config repository: %s", root_config_git_repo.get_clone_url()) + + root_repo = RootRepoFactory().create(root_repo=root_config_git_repo) team_config_app_name = team_config_git_repo.get_clone_url().split("/")[-1].removesuffix(".git") - root_repo = RootRepo(root_config_git_repo) - tenant_config_team_repo = AppTenantConfig("team", config_source_repository=team_config_git_repo) + tenant_config_team_repo = AppTenantConfigFactory().create( + config_type="team", name=team_config_app_name, config_source_repository=team_config_git_repo + ) # dict conversion causes YAML object to be unordered tenant_config_repo_apps = dict(tenant_config_team_repo.list_apps()) - if not team_config_app_name in list(root_repo.tenant_list.keys()): + + if not team_config_app_name in list(root_repo.tenant_dict.keys()): raise GitOpsException("Couldn't find config file for apps repository in root repository's 'apps/' directory") - current_repo_apps = dict(root_repo.tenant_list[team_config_app_name].list_apps()) + current_repo_apps = dict(root_repo.tenant_dict[team_config_app_name].list_apps()) - apps_from_other_repos = root_repo.app_list.copy() + apps_from_other_repos = root_repo.all_app_list.copy() apps_from_other_repos.pop(team_config_app_name) for app in list(tenant_config_repo_apps.keys()): for tenant in apps_from_other_repos.values(): @@ -256,8 +262,8 @@ def __sync_apps(team_config_git_repo: GitRepo, root_config_git_repo: GitRepo, gi ) logging.info("Searching apps repository in root repository's 'apps/' directory...") - apps_config_file = root_repo.tenant_list[team_config_app_name].file_path - apps_config_file_name = root_repo.tenant_list[team_config_app_name].file_name + apps_config_file = root_repo.tenant_dict[team_config_app_name].file_path + apps_config_file_name = root_repo.tenant_dict[team_config_app_name].file_name # removing all keys not being current app repo in order to compare app lists # excluding keys added by root repo administrator, @@ -274,7 +280,7 @@ def __sync_apps(team_config_git_repo: GitRepo, root_config_git_repo: GitRepo, gi logging.info("Sync applications in root repository's %s.", apps_config_file_name) merge_yaml_element( apps_config_file, - root_repo.tenant_list[team_config_app_name].found_apps_path, + root_repo.tenant_dict[team_config_app_name].found_apps_path, { repo_app: traverse_config(tenant_config_team_repo.data, tenant_config_team_repo.config_api_version).get( repo_app, "{}" From 02a20ffbf920201cd7f6ae93559013b9c6d61673 Mon Sep 17 00:00:00 2001 From: makrelas Date: Sun, 27 Nov 2022 23:38:08 +0100 Subject: [PATCH 06/17] rework of linters and tests --- Makefile | 6 +- gitopscli/appconfig_api/traverse_config.py | 5 +- gitopscli/commands/sync_apps.py | 107 ++++++++++----------- tests/commands/test_sync_apps.py | 14 +-- 4 files changed, 67 insertions(+), 65 deletions(-) diff --git a/Makefile b/Makefile index 5d637adf..2e054620 100644 --- a/Makefile +++ b/Makefile @@ -25,9 +25,9 @@ coverage: coverage run -m pytest coverage html coverage report - -#checks: format-check lint mypy test -checks: format-check test +#temporary mypy test lock +#checks: format-check lint mypy test +checks: format-check lint test image: DOCKER_BUILDKIT=1 docker build --progress=plain -t gitopscli:latest . diff --git a/gitopscli/appconfig_api/traverse_config.py b/gitopscli/appconfig_api/traverse_config.py index 7ea32421..79fed4bb 100644 --- a/gitopscli/appconfig_api/traverse_config.py +++ b/gitopscli/appconfig_api/traverse_config.py @@ -1,4 +1,7 @@ -def traverse_config(data, configver): +from typing import Any + + +def traverse_config(data, configver) -> Any: path = configver[1] lookup = data for key in path: diff --git a/gitopscli/commands/sync_apps.py b/gitopscli/commands/sync_apps.py index f074d5ff..6163a3b2 100644 --- a/gitopscli/commands/sync_apps.py +++ b/gitopscli/commands/sync_apps.py @@ -1,38 +1,26 @@ import logging +import os +from typing import Any, Optional from dataclasses import dataclass from gitopscli.git_api import GitApiConfig, GitRepo, GitRepoApiFactory from gitopscli.io_api.yaml_util import merge_yaml_element from gitopscli.gitops_exception import GitOpsException -from .command import Command - - -# from gitopscli.appconfig_api.app_tenant_config import AppTenantConfig -# from gitopscli.appconfig_api.root_repo import RootRepo -# from gitopscli.appconfig_api.traverse_config import traverse_config - -########################################################################################### -######## TO BE REPLACED WITH IMPORT FROM appconfig_api -# TODO: Custom config reader -# TODO: Test custom config read, creation of objects AppTenantConfig and RootRepo -import os -from typing import Any -from dataclasses import dataclass -from ruamel.yaml import YAML from gitopscli.appconfig_api.traverse_config import traverse_config from gitopscli.io_api.yaml_util import yaml_file_load, yaml_load +from .command import Command class AppTenantConfigFactory: def generate_config_from_team_repo( self, team_config_git_repo: GitRepo - ) -> Any: # TODO: supposed to be ordereddict than Any + ) -> Any: # TODO: supposed to be ordereddict than Any pylint: disable=fixme repo_dir = team_config_git_repo.get_full_file_path(".") applist = { name for name in os.listdir(repo_dir) if os.path.isdir(os.path.join(repo_dir, name)) and not name.startswith(".") } - # TODO: Create YAML() object without writing template strings + # TODO: Create YAML() object without writing template strings pylint: disable=fixme # Currently this is the easiest method, although it can be better template_yaml = """ config: @@ -55,10 +43,11 @@ def generate_config_from_team_repo( data["config"]["applications"][app].insert(1, "customAppConfig", customconfig) return data - # TODO: method should contain all aps, not only one, requires rewriting of merging during root repo init - def get_custom_config(self, appname, team_config_git_repo ) -> str | None: - team_config_git_repo = team_config_git_repo - + # TODO: method should contain all aps, not only one, requires rewriting of merging during root repo init pylint: disable=fixme + @staticmethod + def get_custom_config( + appname: str, team_config_git_repo: GitRepo + ) -> Any | None: # TODO: supposed to be ordereddict instead of Any from ruamel pylint: disable=fixme # try: custom_config_file = team_config_git_repo.get_full_file_path(f"{appname}/app_value_file.yaml") # except Exception as ex: @@ -66,8 +55,8 @@ def get_custom_config(self, appname, team_config_git_repo ) -> str | None: # handle broken file/nod adhering to allowed # return ex # sanitize - # TODO: how to keep whole content with comments - # TODO: handling generic values for all apps + # TODO: how to keep whole content with comments pylint: disable=fixme + # TODO: handling generic values for all apps pylint: disable=fixme if os.path.exists(custom_config_file): custom_config_content = yaml_file_load(custom_config_file) return custom_config_content @@ -81,33 +70,35 @@ def create( file_path: str = None, file_name: str = None, config_source_repository: GitRepo = None, - found_apps_path: str = None - ): - if config_type == "root": + found_apps_path: str = None, + ) -> "AppTenantConfig": + if config_type == "root": # pylint: disable=no-else-return return AppTenantConfig(config_type, data, name, file_path, file_name, found_apps_path=found_apps_path) - elif config_type == "team": config_source_repository.clone() data = self.generate_config_from_team_repo(config_source_repository) - return AppTenantConfig(config_type, data, name, config_source_repository=config_source_repository.get_clone_url()) + return AppTenantConfig( + config_type, data, name, config_source_repository=config_source_repository.get_clone_url() + ) + raise GitOpsException("wrong config_type called") @dataclass class AppTenantConfig: config_type: str # is instance initialized as config located in root/team repo - data: Any # TODO: supposed to be ordereddict from ruamel + data: Any # TODO: supposed to be ordereddict from ruamel pylint: disable=fixme name: str # tenant name - file_path: str = None - file_name: str = None - config_source_repository: str = None # team tenant repository url - config_api_version: tuple[Any, ...] = None - found_apps_path: str = None + file_path: Optional[str] = None + file_name: Optional[str] = None + config_source_repository: Optional[str] = None # team tenant repository url + config_api_version: Optional[tuple[Any, ...]] = None + found_apps_path: Optional[str] = None - def __post_init__(self): + def __post_init__(self) -> None: self.config_api_version = self.__get_config_api_version() def __get_config_api_version(self) -> tuple[Any, ...]: - #NOT WORKING AS SHOULD, SHOILD REPLACE MANUAL FOUND_APPS_PATH maybe count the keys? + # NOT WORKING AS SHOULD, SHOILD REPLACE MANUAL FOUND_APPS_PATH maybe count the keys? if "config" in self.data.keys(): return ("v2", ("config", "applications")) return ("v1", ("applications",)) @@ -129,7 +120,8 @@ def delete_app(self) -> None: class RootRepoFactory: - def __get_bootstrap_entries(self, bootstrap_values_file: str) -> Any: + @staticmethod + def __get_bootstrap_entries(bootstrap_values_file: str) -> Any: try: bootstrap_yaml = yaml_file_load(bootstrap_values_file) except FileNotFoundError as ex: @@ -141,7 +133,10 @@ def __get_bootstrap_entries(self, bootstrap_values_file: str) -> Any: raise GitOpsException("Every bootstrap entry must have a 'name' property.") return bootstrap_yaml["bootstrap"] - def __generate_tenant_app_dict_from_root_repo(self, root_config_git_repo: GitRepo, bootstrap: Any): + @staticmethod + def __generate_tenant_app_dict_from_root_repo( + root_config_git_repo: GitRepo, bootstrap: Any + ) -> dict[str, "AppTenantConfig"]: tenant_app_dict = {} for bootstrap_entry in bootstrap: tenant_apps_config_file_name = "apps/" + bootstrap_entry["name"] + ".yaml" @@ -151,27 +146,28 @@ def __generate_tenant_app_dict_from_root_repo(self, root_config_git_repo: GitRep tenant_apps_config_content = yaml_file_load(tenant_apps_config_file) except FileNotFoundError as ex: raise GitOpsException(f"File '{tenant_apps_config_file_name}' not found in root repository.") from ex - # TODO exception handling for malformed yaml + # TODO exception handling for malformed yaml pylint: disable=fixme found_apps_path = "applications" if "config" in tenant_apps_config_content: tenant_apps_config_content = tenant_apps_config_content["config"] found_apps_path = "config.applications" if "repository" not in tenant_apps_config_content: raise GitOpsException(f"Cannot find key 'repository' in '{tenant_apps_config_file_name}'") - logging.info("adding {}".format(bootstrap_entry["name"])) + logging.info("adding %s", (bootstrap_entry["name"])) atc = AppTenantConfigFactory().create( data=tenant_apps_config_content, name=bootstrap_entry["name"], config_type="root", file_path=tenant_apps_config_file, file_name=tenant_apps_config_file_name, - found_apps_path=found_apps_path + found_apps_path=found_apps_path, ) tenant_app_dict.update({bootstrap_entry["name"]: atc}) return tenant_app_dict - # TODO SHOULD THIS FULL METHOD INSTEAD OF POPULATING - def __get_all_apps_list(self, tenant_dict: Any): + # TODO SHOULD THIS FULL METHOD INSTEAD OF POPULATING pylint: disable=fixme + @staticmethod + def __get_all_apps_list(tenant_dict: Any) -> dict[str, list]: all_apps_list = dict() for tenant in tenant_dict: value = traverse_config(tenant_dict[tenant].data, tenant_dict[tenant].config_api_version) @@ -184,7 +180,6 @@ def create(self, root_repo: GitRepo): bootstrap_values_file = root_repo.get_full_file_path("bootstrap/values.yaml") bootstrap = self.__get_bootstrap_entries(bootstrap_values_file) tenant_dict = self.__generate_tenant_app_dict_from_root_repo(root_repo, bootstrap) - all_app_list = self.__get_all_apps_list(tenant_dict) return RootRepo(name, tenant_dict, bootstrap, all_app_list) @@ -192,17 +187,19 @@ def create(self, root_repo: GitRepo): @dataclass class RootRepo: name: str # root repository name - tenant_dict: dict # TODO of AppTenantConfig #list of the tenant configs in the root repository (in apps folder) - bootstrap: set # list of tenants to be bootstrapped, derived form values.yaml in bootstrap root repo dict - all_app_list: set # list of apps without custormer separation + tenant_dict: dict[ + str, "AppTenantConfig" + ] # TODO of AppTenantConfig #list of the tenant configs in the root repository (in apps folder) pylint: disable=fixme + bootstrap: set[Any] # list of tenants to be bootstrapped, derived form values.yaml in bootstrap root repo dict + all_app_list: set[str] # list of apps without custormer separation -def traverse_config(data, configver): - path = configver[1] - lookup = data - for key in path: - lookup = lookup[key] - return lookup +# def traverse_config(data, configver): +# path = configver[1] +# lookup = data +# for key in path: +# lookup = lookup[key] +# return lookup ################################################################################################# @@ -245,7 +242,7 @@ def __sync_apps(team_config_git_repo: GitRepo, root_config_git_repo: GitRepo, gi # dict conversion causes YAML object to be unordered tenant_config_repo_apps = dict(tenant_config_team_repo.list_apps()) - + if not team_config_app_name in list(root_repo.tenant_dict.keys()): raise GitOpsException("Couldn't find config file for apps repository in root repository's 'apps/' directory") current_repo_apps = dict(root_repo.tenant_dict[team_config_app_name].list_apps()) @@ -267,7 +264,7 @@ def __sync_apps(team_config_git_repo: GitRepo, root_config_git_repo: GitRepo, gi # removing all keys not being current app repo in order to compare app lists # excluding keys added by root repo administrator, - # TODO: figure out how to handle that better + # TODO: figure out how to handle that better pylint: disable=fixme for app in list(current_repo_apps.keys()): if current_repo_apps.get(app, dict()) is not None: for key in list(current_repo_apps.get(app, dict())): diff --git a/tests/commands/test_sync_apps.py b/tests/commands/test_sync_apps.py index 27dbbd9d..6f3c6c54 100644 --- a/tests/commands/test_sync_apps.py +++ b/tests/commands/test_sync_apps.py @@ -104,7 +104,6 @@ def test_sync_apps_happy_flow(self): call.logging.info("Team config repository: %s", "https://repository.url/team/team-non-prod.git"), call.GitRepo_root.get_clone_url(), call.logging.info("Root config repository: %s", "https://repository.url/root/root-config.git"), - call.GitRepo_team.get_clone_url(), call.GitRepo_root.get_clone_url(), call.GitRepo_root.clone(), call.GitRepo_root.get_full_file_path("bootstrap/values.yaml"), @@ -112,11 +111,12 @@ def test_sync_apps_happy_flow(self): call.logging.info("Analyzing %s in root repository", "apps/team-non-prod.yaml"), call.GitRepo_root.get_full_file_path("apps/team-non-prod.yaml"), call.yaml_file_load("/tmp/root-config-repo/apps/team-non-prod.yaml"), - call.logging.info("adding team-non-prod"), + call.logging.info("adding %s", "team-non-prod"), call.logging.info("Analyzing %s in root repository", "apps/other-team-non-prod.yaml"), call.GitRepo_root.get_full_file_path("apps/other-team-non-prod.yaml"), call.yaml_file_load("/tmp/root-config-repo/apps/other-team-non-prod.yaml"), - call.logging.info("adding other-team-non-prod"), + call.logging.info("adding %s", "other-team-non-prod"), + call.GitRepo_team.get_clone_url(), call.GitRepo_team.clone(), call.GitRepo_team.get_full_file_path("."), call.os.listdir("/tmp/team-config-repo/."), @@ -125,6 +125,7 @@ def test_sync_apps_happy_flow(self): call.GitRepo_team.get_clone_url(), call.GitRepo_team.get_full_file_path("my-app/app_value_file.yaml"), call.os.path.exists("/tmp/team-config-repo/my-app/app_value_file.yaml"), + call.GitRepo_team.get_clone_url(), call.logging.info("Found %s app(s) in apps repository: %s", 1, "my-app"), call.logging.info("Searching apps repository in root repository's 'apps/' directory..."), call.logging.info("Sync applications in root repository's %s.", "apps/team-non-prod.yaml"), @@ -163,7 +164,6 @@ def test_sync_apps_already_up_to_date(self): call.logging.info("Team config repository: %s", "https://repository.url/team/team-non-prod.git"), call.GitRepo_root.get_clone_url(), call.logging.info("Root config repository: %s", "https://repository.url/root/root-config.git"), - call.GitRepo_team.get_clone_url(), call.GitRepo_root.get_clone_url(), call.GitRepo_root.clone(), call.GitRepo_root.get_full_file_path("bootstrap/values.yaml"), @@ -171,11 +171,12 @@ def test_sync_apps_already_up_to_date(self): call.logging.info("Analyzing %s in root repository", "apps/team-non-prod.yaml"), call.GitRepo_root.get_full_file_path("apps/team-non-prod.yaml"), call.yaml_file_load("/tmp/root-config-repo/apps/team-non-prod.yaml"), - call.logging.info("adding team-non-prod"), + call.logging.info("adding %s", "team-non-prod"), call.logging.info("Analyzing %s in root repository", "apps/other-team-non-prod.yaml"), call.GitRepo_root.get_full_file_path("apps/other-team-non-prod.yaml"), call.yaml_file_load("/tmp/root-config-repo/apps/other-team-non-prod.yaml"), - call.logging.info("adding other-team-non-prod"), + call.logging.info("adding %s", "other-team-non-prod"), + call.GitRepo_team.get_clone_url(), call.GitRepo_team.clone(), call.GitRepo_team.get_full_file_path("."), call.os.listdir("/tmp/team-config-repo/."), @@ -184,6 +185,7 @@ def test_sync_apps_already_up_to_date(self): call.GitRepo_team.get_clone_url(), call.GitRepo_team.get_full_file_path("my-app/app_value_file.yaml"), call.os.path.exists("/tmp/team-config-repo/my-app/app_value_file.yaml"), + call.GitRepo_team.get_clone_url(), call.logging.info("Found %s app(s) in apps repository: %s", 1, "my-app"), call.logging.info("Searching apps repository in root repository's 'apps/' directory..."), call.logging.info("Root repository already up-to-date. I'm done here."), From ae294e579ef4cef8b4a7c53b101d6a0e0d3784e8 Mon Sep 17 00:00:00 2001 From: makrelas Date: Mon, 28 Nov 2022 16:39:05 +0100 Subject: [PATCH 07/17] tst --- gitopscli/commands/sync_apps.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/gitopscli/commands/sync_apps.py b/gitopscli/commands/sync_apps.py index 6163a3b2..473d5ab0 100644 --- a/gitopscli/commands/sync_apps.py +++ b/gitopscli/commands/sync_apps.py @@ -2,6 +2,7 @@ import os from typing import Any, Optional from dataclasses import dataclass +from ruamel.yaml.comments import CommentedMap from gitopscli.git_api import GitApiConfig, GitRepo, GitRepoApiFactory from gitopscli.io_api.yaml_util import merge_yaml_element from gitopscli.gitops_exception import GitOpsException @@ -13,7 +14,7 @@ class AppTenantConfigFactory: def generate_config_from_team_repo( self, team_config_git_repo: GitRepo - ) -> Any: # TODO: supposed to be ordereddict than Any pylint: disable=fixme + ) -> Any: # TODO: supposed to be ordereddic00t than Any pylint: disable=fixme repo_dir = team_config_git_repo.get_full_file_path(".") applist = { name @@ -66,14 +67,14 @@ def create( self, config_type: str, name: str, - data: str = None, - file_path: str = None, - file_name: str = None, - config_source_repository: GitRepo = None, - found_apps_path: str = None, + data: CommentedMap | dict[Any, Any], + config_source_repository: GitRepo | None, + file_path: Optional[str] | None, + file_name: Optional[str] | None, + found_apps_path: Optional[str] | None, ) -> "AppTenantConfig": if config_type == "root": # pylint: disable=no-else-return - return AppTenantConfig(config_type, data, name, file_path, file_name, found_apps_path=found_apps_path) + return AppTenantConfig(config_type, data, name, file_path, file_name, None, found_apps_path=found_apps_path) elif config_type == "team": config_source_repository.clone() data = self.generate_config_from_team_repo(config_source_repository) @@ -86,11 +87,11 @@ def create( @dataclass class AppTenantConfig: config_type: str # is instance initialized as config located in root/team repo - data: Any # TODO: supposed to be ordereddict from ruamel pylint: disable=fixme + data: CommentedMap | dict # TODO: supposed to be ordereddict from ruamel pylint: disable=fixme name: str # tenant name + config_source_repository: str | None file_path: Optional[str] = None - file_name: Optional[str] = None - config_source_repository: Optional[str] = None # team tenant repository url + file_name: Optional[str] = None # team tenant repository url config_api_version: Optional[tuple[Any, ...]] = None found_apps_path: Optional[str] = None @@ -103,7 +104,7 @@ def __get_config_api_version(self) -> tuple[Any, ...]: return ("v2", ("config", "applications")) return ("v1", ("applications",)) - def list_apps(self) -> None: + def list_apps(self) -> CommentedMap | dict: return traverse_config(self.data, self.config_api_version) def add_app(self) -> None: @@ -161,6 +162,7 @@ def __generate_tenant_app_dict_from_root_repo( file_path=tenant_apps_config_file, file_name=tenant_apps_config_file_name, found_apps_path=found_apps_path, + config_source_repository=None ) tenant_app_dict.update({bootstrap_entry["name"]: atc}) return tenant_app_dict @@ -174,7 +176,7 @@ def __get_all_apps_list(tenant_dict: Any) -> dict[str, list]: all_apps_list.update({tenant: list((dict(value).keys()))}) return all_apps_list - def create(self, root_repo: GitRepo): + def create(self, root_repo: GitRepo) -> 'RootRepo': name = root_repo.get_clone_url().split("/")[-1].removesuffix(".git") root_repo.clone() bootstrap_values_file = root_repo.get_full_file_path("bootstrap/values.yaml") @@ -237,7 +239,7 @@ def __sync_apps(team_config_git_repo: GitRepo, root_config_git_repo: GitRepo, gi root_repo = RootRepoFactory().create(root_repo=root_config_git_repo) team_config_app_name = team_config_git_repo.get_clone_url().split("/")[-1].removesuffix(".git") tenant_config_team_repo = AppTenantConfigFactory().create( - config_type="team", name=team_config_app_name, config_source_repository=team_config_git_repo + config_type="team", data=dict(), name=team_config_app_name, config_source_repository=team_config_git_repo ) # dict conversion causes YAML object to be unordered @@ -290,7 +292,7 @@ def __sync_apps(team_config_git_repo: GitRepo, root_config_git_repo: GitRepo, gi def __commit_and_push( - team_config_git_repo: GitRepo, root_config_git_repo: GitRepo, git_user: str, git_email: str, app_file_name: str + team_config_git_repo: GitRepo, root_config_git_repo: GitRepo, git_user: str, git_email: str, app_file_name: optional[str] ) -> None: author = team_config_git_repo.get_author_from_last_commit() root_config_git_repo.commit(git_user, git_email, f"{author} updated " + app_file_name) From 1e211d608e101a498f5c705e0606f841736d714d Mon Sep 17 00:00:00 2001 From: makrelas Date: Thu, 1 Dec 2022 14:35:39 +0100 Subject: [PATCH 08/17] refactor: prettier version of the custom config validator --- gitopscli/commands/sync_apps.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/gitopscli/commands/sync_apps.py b/gitopscli/commands/sync_apps.py index f6503b64..26d90cf8 100644 --- a/gitopscli/commands/sync_apps.py +++ b/gitopscli/commands/sync_apps.py @@ -247,14 +247,11 @@ def __sync_apps(team_config_git_repo: GitRepo, root_config_git_repo: GitRepo, gi ) # dict conversion causes YAML object to be unordered - tenant_config_repo_apps = dict(tenant_config_team_repo.list_apps()) - if not team_config_app_name in list(root_repo.tenant_dict.keys()): raise GitOpsException("Couldn't find config file for apps repository in root repository's 'apps/' directory") - current_repo_apps = dict(root_repo.tenant_dict[team_config_app_name].list_apps()) - apps_from_other_repos = root_repo.all_app_list.copy() apps_from_other_repos.pop(team_config_app_name) + tenant_config_repo_apps = dict(tenant_config_team_repo.list_apps()) for app in list(tenant_config_repo_apps.keys()): for tenant in apps_from_other_repos.values(): if app in tenant: @@ -268,14 +265,14 @@ def __sync_apps(team_config_git_repo: GitRepo, root_config_git_repo: GitRepo, gi apps_config_file = root_repo.tenant_dict[team_config_app_name].file_path apps_config_file_name = root_repo.tenant_dict[team_config_app_name].file_name - # removing all keys not being current app repo in order to compare app lists - # excluding keys added by root repo administrator, - # TODO: figure out how to handle that better pylint: disable=fixme + current_repo_apps = dict(root_repo.tenant_dict[team_config_app_name].list_apps()) for app in list(current_repo_apps.keys()): - if current_repo_apps.get(app, dict()) is not None: - for key in list(current_repo_apps.get(app, dict())): - if key != "customAppConfig": - del current_repo_apps[app][key] + if current_repo_apps.get(app) is not None: + app_properties = current_repo_apps[app] + custom_app_config_item = app_properties.get("customAppConfig", None) + current_repo_apps[app].clear() + current_repo_apps[app].insert(0, "customAppConfig", custom_app_config_item) + if current_repo_apps == tenant_config_repo_apps: logging.info("Root repository already up-to-date. I'm done here.") return From f123228b1b688313708a741608732cf8090a67e0 Mon Sep 17 00:00:00 2001 From: makrelas Date: Thu, 1 Dec 2022 15:58:36 +0100 Subject: [PATCH 09/17] refactor: moved app list verification outside sync_apps function --- gitopscli/commands/sync_apps.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/gitopscli/commands/sync_apps.py b/gitopscli/commands/sync_apps.py index 26d90cf8..d4fc3de0 100644 --- a/gitopscli/commands/sync_apps.py +++ b/gitopscli/commands/sync_apps.py @@ -236,6 +236,16 @@ def _sync_apps_command(args: SyncAppsCommand.Args) -> None: __sync_apps(team_config_git_repo, root_config_git_repo, args.git_user, args.git_email) +def __check_app_other_tenant( + apps_from_other_repos: dict[Any, Any], team_config_app_name: str, tenant_config_repo_apps: Any +) -> None: + apps_from_other_repos.pop(team_config_app_name) + for app in list(tenant_config_repo_apps.keys()): + for tenant in apps_from_other_repos.values(): + if app in tenant: + raise GitOpsException(f"Application '{app}' already exists in a different repository") + + def __sync_apps(team_config_git_repo: GitRepo, root_config_git_repo: GitRepo, git_user: str, git_email: str) -> None: logging.info("Team config repository: %s", team_config_git_repo.get_clone_url()) logging.info("Root config repository: %s", root_config_git_repo.get_clone_url()) @@ -249,13 +259,8 @@ def __sync_apps(team_config_git_repo: GitRepo, root_config_git_repo: GitRepo, gi # dict conversion causes YAML object to be unordered if not team_config_app_name in list(root_repo.tenant_dict.keys()): raise GitOpsException("Couldn't find config file for apps repository in root repository's 'apps/' directory") - apps_from_other_repos = root_repo.all_app_list.copy() - apps_from_other_repos.pop(team_config_app_name) tenant_config_repo_apps = dict(tenant_config_team_repo.list_apps()) - for app in list(tenant_config_repo_apps.keys()): - for tenant in apps_from_other_repos.values(): - if app in tenant: - raise GitOpsException(f"Application '{app}' already exists in a different repository") + __check_app_other_tenant(root_repo.all_app_list.copy(), team_config_app_name, tenant_config_repo_apps) logging.info( "Found %s app(s) in apps repository: %s", len(tenant_config_repo_apps), ", ".join(tenant_config_repo_apps) From 55e8670b936a3502cffe06c86804cf3d4c06bee2 Mon Sep 17 00:00:00 2001 From: makrelas Date: Fri, 9 Dec 2022 01:23:09 +0100 Subject: [PATCH 10/17] refactor: rearranging of existing code, do not add customAppConfig value file does not exist --- docs/commands/sync-apps.md | 16 ++++- gitopscli/commands/sync_apps.py | 100 +++++++++++++++++-------------- tests/commands/test_sync_apps.py | 4 +- 3 files changed, 72 insertions(+), 48 deletions(-) diff --git a/docs/commands/sync-apps.md b/docs/commands/sync-apps.md index d6a38703..b365afa6 100644 --- a/docs/commands/sync-apps.md +++ b/docs/commands/sync-apps.md @@ -29,7 +29,21 @@ root-config-repo/ └── values.yaml ``` ### app specific values -app specific values may be set using a values.yaml file directly in the app directory. gitopscli will process these values and remove key that would be blacklisted for security purpose and then store them in the result files under app key. +app specific values may be set using a app_value_file.yaml file directly in the app directory. gitopscli will process these values and add them under customAppConfig parameter of application +**tenantrepo.git/app1/app_value_file.yaml** +```yaml +customvalue: test +``` +**rootrepo.git/apps/tenantrepo.yaml** +```yaml +config: + repository: https://tenantrepo.git + applications: + app1: + customAppConfig: + customvalue: test + app2: {} +``` **bootstrap/values.yaml** ```yaml diff --git a/gitopscli/commands/sync_apps.py b/gitopscli/commands/sync_apps.py index d4fc3de0..1c4f67ae 100644 --- a/gitopscli/commands/sync_apps.py +++ b/gitopscli/commands/sync_apps.py @@ -14,7 +14,7 @@ class AppTenantConfigFactory: def generate_config_from_team_repo( self, team_config_git_repo: GitRepo - ) -> Any: # TODO: supposed to be ordereddic00t than Any pylint: disable=fixme + ) -> Any: # TODO: supposed to be ruamel object than Any pylint: disable=fixme repo_dir = team_config_git_repo.get_full_file_path(".") applist = { name @@ -22,7 +22,6 @@ def generate_config_from_team_repo( if os.path.isdir(os.path.join(repo_dir, name)) and not name.startswith(".") } # TODO: Create YAML() object without writing template strings pylint: disable=fixme - # Currently this is the easiest method, although it can be better template_yaml = """ config: repository: {} @@ -41,7 +40,8 @@ def generate_config_from_team_repo( app_as_yaml_object = yaml_load(template_yaml) # dict path hardcoded as object generated will always be in v2 or later data["config"]["applications"].update(app_as_yaml_object) - data["config"]["applications"][app].insert(1, "customAppConfig", customconfig) + if customconfig: + data["config"]["applications"][app].insert(1, "customAppConfig", customconfig) return data # TODO: method should contain all aps, not only one, requires rewriting of merging during root repo init pylint: disable=fixme @@ -63,32 +63,34 @@ def get_custom_config( return custom_config_content return None - def create( - self, - config_type: str, + @staticmethod + def create_root_repo_tenant_config( name: str, data: CommentedMap | dict[Any, Any], - config_source_repository: Any, - file_path: str = "", - file_name: str = "", + file_name: str, + file_path: str, found_apps_path: str = "config.applications", ) -> "AppTenantConfig": - if config_type == "root": # pylint: disable=no-else-return - return AppTenantConfig( - config_type=config_type, - data=data, - name=name, - file_name=file_name, - file_path=file_path, - found_apps_path=found_apps_path, - ) - elif config_type == "team": - config_source_repository.clone() - data = self.generate_config_from_team_repo(config_source_repository) - return AppTenantConfig( - config_type, data, name, config_source_repository.get_clone_url(), file_name, file_path - ) - raise GitOpsException("wrong config_type called") + return AppTenantConfig( + config_type="root", + data=data, + name=name, + found_apps_path=found_apps_path, + file_name=file_name, + file_path=file_path, + ) + + def create_team_repo_tenant_config( + self, + name: str, + config_source_repository: Any, + # found_apps_path: str = "config.applications", + ) -> "AppTenantConfig": + config_source_repository.clone() + data = self.generate_config_from_team_repo(config_source_repository) + return AppTenantConfig( + config_type="team", data=data, name=name, config_source_repository=config_source_repository.get_clone_url() + ) @dataclass @@ -112,7 +114,7 @@ def __get_config_api_version(self) -> tuple[Any, ...]: return ("v1", ("applications",)) def list_apps(self) -> Any: - return traverse_config(self.data, self.config_api_version) + return dict(traverse_config(self.data, self.config_api_version)) def add_app(self) -> None: # adds app to the app tenant config @@ -126,6 +128,10 @@ def delete_app(self) -> None: # deletes app from tenant config pass + # def get_sanitized_app_tenant_config(self): + # self.data + # pass + class RootRepoFactory: @staticmethod @@ -168,14 +174,12 @@ def __generate_tenant_app_dict_from_root_repo( if "repository" not in tenant_apps_config_content: raise GitOpsException(f"Cannot find key 'repository' in '{tenant_apps_config_file_name}'") logging.info("adding %s", (bootstrap_entry["name"])) - atc = AppTenantConfigFactory().create( + atc = AppTenantConfigFactory().create_root_repo_tenant_config( data=tenant_apps_config_content, name=bootstrap_entry["name"], - config_type="root", file_path=tenant_apps_config_file, file_name=tenant_apps_config_file_name, found_apps_path=found_apps_path, - config_source_repository=None, ) tenant_app_dict.update({bootstrap_entry["name"]: atc}) return tenant_app_dict @@ -208,6 +212,9 @@ class RootRepo: bootstrap: list[Any] # list of tenants to be bootstrapped, derived form values.yaml in bootstrap root repo dict all_app_list: dict[str, list[Any]] # list of apps without custormer separation + def list_tenants(self) -> list[str]: + return list(self.tenant_dict.keys()) + class SyncAppsCommand(Command): @dataclass(frozen=True) @@ -236,7 +243,7 @@ def _sync_apps_command(args: SyncAppsCommand.Args) -> None: __sync_apps(team_config_git_repo, root_config_git_repo, args.git_user, args.git_email) -def __check_app_other_tenant( +def __validate_if_app_not_present( apps_from_other_repos: dict[Any, Any], team_config_app_name: str, tenant_config_repo_apps: Any ) -> None: apps_from_other_repos.pop(team_config_app_name) @@ -252,36 +259,39 @@ def __sync_apps(team_config_git_repo: GitRepo, root_config_git_repo: GitRepo, gi root_repo = RootRepoFactory().create(root_repo=root_config_git_repo) team_config_app_name = team_config_git_repo.get_clone_url().split("/")[-1].removesuffix(".git") - tenant_config_team_repo = AppTenantConfigFactory().create( - config_type="team", data=dict(), name=team_config_app_name, config_source_repository=team_config_git_repo + if not team_config_app_name in root_repo.list_tenants(): + raise GitOpsException("Couldn't find config file for apps repository in root repository's 'apps/' directory") + tenant_config_team_repo = AppTenantConfigFactory().create_team_repo_tenant_config( + name=team_config_app_name, config_source_repository=team_config_git_repo ) - # dict conversion causes YAML object to be unordered - if not team_config_app_name in list(root_repo.tenant_dict.keys()): - raise GitOpsException("Couldn't find config file for apps repository in root repository's 'apps/' directory") - tenant_config_repo_apps = dict(tenant_config_team_repo.list_apps()) - __check_app_other_tenant(root_repo.all_app_list.copy(), team_config_app_name, tenant_config_repo_apps) + tenant_config_repo_apps = tenant_config_team_repo.list_apps() + __validate_if_app_not_present(root_repo.all_app_list.copy(), team_config_app_name, tenant_config_repo_apps) logging.info( "Found %s app(s) in apps repository: %s", len(tenant_config_repo_apps), ", ".join(tenant_config_repo_apps) ) logging.info("Searching apps repository in root repository's 'apps/' directory...") - apps_config_file = root_repo.tenant_dict[team_config_app_name].file_path - apps_config_file_name = root_repo.tenant_dict[team_config_app_name].file_name - - current_repo_apps = dict(root_repo.tenant_dict[team_config_app_name].list_apps()) + current_repo_apps = root_repo.tenant_dict[team_config_app_name].list_apps() for app in list(current_repo_apps.keys()): - if current_repo_apps.get(app) is not None: - app_properties = current_repo_apps[app] - custom_app_config_item = app_properties.get("customAppConfig", None) + if ( + current_repo_apps.get(app) is not None + ): # None is returend when application does not have any additional parameters + # app_properties = current_repo_apps[app] + custom_app_config_item = current_repo_apps[app].get("customAppConfig") + # None is returend when application does not have custom configuration current_repo_apps[app].clear() - current_repo_apps[app].insert(0, "customAppConfig", custom_app_config_item) + if custom_app_config_item is not None: + current_repo_apps[app]["customAppConfig"] = custom_app_config_item + # current_repo_apps[app].insert(0, "customAppConfig", custom_app_config_item) if current_repo_apps == tenant_config_repo_apps: logging.info("Root repository already up-to-date. I'm done here.") return + apps_config_file = root_repo.tenant_dict[team_config_app_name].file_path + apps_config_file_name = root_repo.tenant_dict[team_config_app_name].file_name logging.info("Sync applications in root repository's %s.", apps_config_file_name) merge_yaml_element( apps_config_file, diff --git a/tests/commands/test_sync_apps.py b/tests/commands/test_sync_apps.py index 02c552ef..c77a86a7 100644 --- a/tests/commands/test_sync_apps.py +++ b/tests/commands/test_sync_apps.py @@ -130,7 +130,7 @@ def test_sync_apps_happy_flow(self): call.merge_yaml_element( "/tmp/root-config-repo/apps/team-non-prod.yaml", "config.applications", - {"my-app": ordereddict([("customAppConfig", None)])}, + {"my-app": {}}, ), call.GitRepo_team.get_author_from_last_commit(), call.GitRepo_root.commit("GIT_USER", "GIT_EMAIL", "author updated apps/team-non-prod.yaml"), @@ -144,7 +144,7 @@ def test_sync_apps_already_up_to_date(self): }, "/tmp/root-config-repo/apps/team-non-prod.yaml": { "repository": "https://repository.url/team/team-non-prod.git", - "applications": {"my-app": ordereddict([("customAppConfig", None)])}, # my-app already exists + "applications": {"my-app": {}}, # my-app already exists }, "/tmp/root-config-repo/apps/other-team-non-prod.yaml": { "repository": "https://repository.url/other-team/other-team-non-prod.git", From d1d412abda6e42fd7220c9e46c8f31780ff8f3e9 Mon Sep 17 00:00:00 2001 From: makrelas Date: Fri, 9 Dec 2022 15:54:14 +0100 Subject: [PATCH 11/17] refactor: added apiVersion class for storing yaml property infirmation --- gitopscli/appconfig_api/traverse_config.py | 2 +- gitopscli/commands/sync_apps.py | 34 ++++++++++++---------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/gitopscli/appconfig_api/traverse_config.py b/gitopscli/appconfig_api/traverse_config.py index 8fe3f5f8..aa6dc55b 100644 --- a/gitopscli/appconfig_api/traverse_config.py +++ b/gitopscli/appconfig_api/traverse_config.py @@ -2,7 +2,7 @@ def traverse_config(data: Any, configver: Any) -> Any: - path = configver[1] + path = configver lookup = data for key in path: lookup = lookup[key] diff --git a/gitopscli/commands/sync_apps.py b/gitopscli/commands/sync_apps.py index 1c4f67ae..0ab4d66e 100644 --- a/gitopscli/commands/sync_apps.py +++ b/gitopscli/commands/sync_apps.py @@ -1,6 +1,6 @@ import logging import os -from typing import Any, Optional +from typing import Any from dataclasses import dataclass from ruamel.yaml.comments import CommentedMap from gitopscli.git_api import GitApiConfig, GitRepo, GitRepoApiFactory @@ -11,6 +11,12 @@ from .command import Command +@dataclass +class AppTenantConfigApiVersion: + version: str + path: tuple[str, ...] + + class AppTenantConfigFactory: def generate_config_from_team_repo( self, team_config_git_repo: GitRepo @@ -98,23 +104,22 @@ class AppTenantConfig: config_type: str # is instance initialized as config located in root/team repo data: CommentedMap | dict[Any, Any] # TODO: supposed to be ordereddict from ruamel pylint: disable=fixme name: str # tenant name + config_api_version: AppTenantConfigApiVersion = AppTenantConfigApiVersion("v2", ("config", "applications")) config_source_repository: str = "" file_name: str = "" found_apps_path: str = "" file_path: str = "" # team tenant repository url - config_api_version: Optional[tuple[Any, ...]] = None def __post_init__(self) -> None: self.config_api_version = self.__get_config_api_version() - def __get_config_api_version(self) -> tuple[Any, ...]: - # NOT WORKING AS SHOULD, SHOILD REPLACE MANUAL FOUND_APPS_PATH maybe count the keys? + def __get_config_api_version(self) -> AppTenantConfigApiVersion: if "config" in self.data.keys(): - return ("v2", ("config", "applications")) - return ("v1", ("applications",)) + return AppTenantConfigApiVersion("v2", ("config", "applications")) + return AppTenantConfigApiVersion("v1", ("applications",)) def list_apps(self) -> Any: - return dict(traverse_config(self.data, self.config_api_version)) + return dict(traverse_config(self.data, self.config_api_version.path)) def add_app(self) -> None: # adds app to the app tenant config @@ -189,7 +194,7 @@ def __generate_tenant_app_dict_from_root_repo( def __get_all_apps_list(tenant_dict: Any) -> dict[str, list[Any]]: all_apps_list = dict() for tenant in tenant_dict: - value = traverse_config(tenant_dict[tenant].data, tenant_dict[tenant].config_api_version) + value = traverse_config(tenant_dict[tenant].data, tenant_dict[tenant].config_api_version.path) all_apps_list.update({tenant: list((dict(value).keys()))}) return all_apps_list @@ -278,13 +283,12 @@ def __sync_apps(team_config_git_repo: GitRepo, root_config_git_repo: GitRepo, gi if ( current_repo_apps.get(app) is not None ): # None is returend when application does not have any additional parameters - # app_properties = current_repo_apps[app] custom_app_config_item = current_repo_apps[app].get("customAppConfig") - # None is returend when application does not have custom configuration current_repo_apps[app].clear() - if custom_app_config_item is not None: + if ( + custom_app_config_item is not None + ): # None is returend when application does not have custom configuration current_repo_apps[app]["customAppConfig"] = custom_app_config_item - # current_repo_apps[app].insert(0, "customAppConfig", custom_app_config_item) if current_repo_apps == tenant_config_repo_apps: logging.info("Root repository already up-to-date. I'm done here.") @@ -297,9 +301,9 @@ def __sync_apps(team_config_git_repo: GitRepo, root_config_git_repo: GitRepo, gi apps_config_file, root_repo.tenant_dict[team_config_app_name].found_apps_path, { - repo_app: traverse_config(tenant_config_team_repo.data, tenant_config_team_repo.config_api_version).get( - repo_app, "{}" - ) + repo_app: traverse_config( + tenant_config_team_repo.data, tenant_config_team_repo.config_api_version.path + ).get(repo_app, "{}") for repo_app in tenant_config_repo_apps }, ) From 4e46a2db313e1d411c60897b9e38992ba1f8c688 Mon Sep 17 00:00:00 2001 From: Nikolas Philips Date: Fri, 9 Dec 2022 20:53:40 +0100 Subject: [PATCH 12/17] Complete refactoring --- gitopscli/appconfig_api/traverse_config.py | 9 - gitopscli/commands/sync_apps.py | 378 +++++++++------------ 2 files changed, 154 insertions(+), 233 deletions(-) delete mode 100644 gitopscli/appconfig_api/traverse_config.py diff --git a/gitopscli/appconfig_api/traverse_config.py b/gitopscli/appconfig_api/traverse_config.py deleted file mode 100644 index aa6dc55b..00000000 --- a/gitopscli/appconfig_api/traverse_config.py +++ /dev/null @@ -1,9 +0,0 @@ -from typing import Any - - -def traverse_config(data: Any, configver: Any) -> Any: - path = configver - lookup = data - for key in path: - lookup = lookup[key] - return lookup diff --git a/gitopscli/commands/sync_apps.py b/gitopscli/commands/sync_apps.py index 0ab4d66e..25c6e773 100644 --- a/gitopscli/commands/sync_apps.py +++ b/gitopscli/commands/sync_apps.py @@ -2,224 +2,201 @@ import os from typing import Any from dataclasses import dataclass +from ruamel.yaml import YAML from ruamel.yaml.comments import CommentedMap from gitopscli.git_api import GitApiConfig, GitRepo, GitRepoApiFactory -from gitopscli.io_api.yaml_util import merge_yaml_element from gitopscli.gitops_exception import GitOpsException -from gitopscli.appconfig_api.traverse_config import traverse_config from gitopscli.io_api.yaml_util import yaml_file_load, yaml_load from .command import Command -@dataclass -class AppTenantConfigApiVersion: - version: str - path: tuple[str, ...] - - -class AppTenantConfigFactory: - def generate_config_from_team_repo( - self, team_config_git_repo: GitRepo +class TenantAppTenantConfigFactory: + def __generate_config_from_tenant_repo( + self, tenant_repo: GitRepo ) -> Any: # TODO: supposed to be ruamel object than Any pylint: disable=fixme - repo_dir = team_config_git_repo.get_full_file_path(".") - applist = { - name - for name in os.listdir(repo_dir) - if os.path.isdir(os.path.join(repo_dir, name)) and not name.startswith(".") - } - # TODO: Create YAML() object without writing template strings pylint: disable=fixme - template_yaml = """ + tenant_app_dirs = self.__get_all_tenant_applications(tenant_repo) + tenant_config_template = """ config: repository: {} applications: {{}} """.format( - team_config_git_repo.get_clone_url() + tenant_repo.get_clone_url() ) - data = yaml_load(template_yaml) - for app in applist: - template_yaml = """ + yaml = yaml_load(tenant_config_template) + for app_dir in tenant_app_dirs: + tenant_application_template = """ {}: {{}} """.format( - app + app_dir ) - customconfig = self.get_custom_config(app, team_config_git_repo) - app_as_yaml_object = yaml_load(template_yaml) + tenant_applications_yaml = yaml_load(tenant_application_template) # dict path hardcoded as object generated will always be in v2 or later - data["config"]["applications"].update(app_as_yaml_object) - if customconfig: - data["config"]["applications"][app].insert(1, "customAppConfig", customconfig) - return data + yaml["config"]["applications"].update(tenant_applications_yaml) + custom_app_config = self.__get_custom_config(app_dir, tenant_repo) + if custom_app_config: + yaml["config"]["applications"][app_dir]["customAppConfig"] = custom_app_config + return yaml + + def __get_all_tenant_applications(self, tenant_repo): + repo_dir = tenant_repo.get_full_file_path(".") + applist = { + name + for name in os.listdir(repo_dir) + if os.path.isdir(os.path.join(repo_dir, name)) and not name.startswith(".") + } + return applist - # TODO: method should contain all aps, not only one, requires rewriting of merging during root repo init pylint: disable=fixme @staticmethod - def get_custom_config( - appname: str, team_config_git_repo: GitRepo - ) -> Any | None: # TODO: supposed to be ordereddict instead of Any from ruamel pylint: disable=fixme - # try: - custom_config_file = team_config_git_repo.get_full_file_path(f"{appname}/app_value_file.yaml") - # except Exception as ex: - # handle missing file - # handle broken file/nod adhering to allowed - # return ex - # sanitize - # TODO: how to keep whole content with comments pylint: disable=fixme - # TODO: handling generic values for all apps pylint: disable=fixme - if os.path.exists(custom_config_file): - custom_config_content = yaml_file_load(custom_config_file) + def __get_custom_config(appname: str, tenant_config_git_repo: GitRepo) -> Any: + custom_config_path = tenant_config_git_repo.get_full_file_path(f"{appname}/.config.yaml") + if os.path.exists(custom_config_path): + custom_config_content = yaml_file_load(custom_config_path) return custom_config_content - return None - - @staticmethod - def create_root_repo_tenant_config( - name: str, - data: CommentedMap | dict[Any, Any], - file_name: str, - file_path: str, - found_apps_path: str = "config.applications", - ) -> "AppTenantConfig": - return AppTenantConfig( - config_type="root", - data=data, - name=name, - found_apps_path=found_apps_path, - file_name=file_name, - file_path=file_path, - ) + return dict() - def create_team_repo_tenant_config( + def create( self, - name: str, - config_source_repository: Any, - # found_apps_path: str = "config.applications", + tenant_repo: GitRepo, ) -> "AppTenantConfig": - config_source_repository.clone() - data = self.generate_config_from_team_repo(config_source_repository) - return AppTenantConfig( - config_type="team", data=data, name=name, config_source_repository=config_source_repository.get_clone_url() - ) + tenant_repo.clone() + tenant_config_yaml = self.__generate_config_from_tenant_repo(tenant_repo) + return AppTenantConfig(yaml=tenant_config_yaml) @dataclass class AppTenantConfig: - config_type: str # is instance initialized as config located in root/team repo - data: CommentedMap | dict[Any, Any] # TODO: supposed to be ordereddict from ruamel pylint: disable=fixme - name: str # tenant name - config_api_version: AppTenantConfigApiVersion = AppTenantConfigApiVersion("v2", ("config", "applications")) - config_source_repository: str = "" - file_name: str = "" - found_apps_path: str = "" - file_path: str = "" # team tenant repository url + yaml: CommentedMap | dict[Any, Any] # TODO: supposed to be ordereddict from ruamel pylint: disable=fixme + tenant_config: CommentedMap = None | dict[Any, Any] + repo_url: str = "" + file_path: str = "" + dirty: bool = False + ruamel_yaml: YAML = YAML() def __post_init__(self) -> None: - self.config_api_version = self.__get_config_api_version() - - def __get_config_api_version(self) -> AppTenantConfigApiVersion: - if "config" in self.data.keys(): - return AppTenantConfigApiVersion("v2", ("config", "applications")) - return AppTenantConfigApiVersion("v1", ("applications",)) - - def list_apps(self) -> Any: - return dict(traverse_config(self.data, self.config_api_version.path)) - - def add_app(self) -> None: - # adds app to the app tenant config - pass - - def modify_app(self) -> None: - # modifies existing app in tenant config - pass - - def delete_app(self) -> None: - # deletes app from tenant config - pass - - # def get_sanitized_app_tenant_config(self): - # self.data - # pass + if "config" in self.yaml: + self.tenant_config = self.yaml["config"] + else: + self.tenant_config = self.yaml + self.repo_url = self.tenant_config["repository"] + + def list_apps(self) -> dict[dict[Any]]: + return dict(self.tenant_config["applications"]) + + def merge_applications(self, desired_tenant_config: "AppTenantConfig") -> None: + desired_apps = desired_tenant_config.list_apps() + self.__delete_removed_applications(desired_apps) + self.__add_new_applications(desired_apps) + self.__update_custom_app_config(desired_apps) + + def __update_custom_app_config(self, desired_apps): + for desired_app_name, desired_app_value in desired_apps.items(): + if desired_app_name in self.list_apps().keys(): + existing_application_value = self.list_apps().get(desired_app_name) + if "customAppConfig" not in desired_app_value: + if "customAppConfig" in existing_application_value: + del existing_application_value["customAppConfig"] + self.__set_dirty() + else: + if "customAppConfig" not in existing_application_value or existing_application_value["customAppConfig"] != desired_app_value["customAppConfig"]: + existing_application_value["customAppConfig"] = desired_app_value["customAppConfig"] + self.__set_dirty() + + def __add_new_applications(self, desired_apps): + for desired_app_name, desired_app_value in desired_apps.items(): + if desired_app_name not in self.list_apps().keys(): + self.tenant_config["applications"][desired_app_name] = desired_app_value + self.__set_dirty() + + def __delete_removed_applications(self, desired_apps): + for current_app in self.list_apps().keys(): + if current_app not in desired_apps.keys(): + del self.tenant_config["applications"][current_app] + self.__set_dirty() + + def __set_dirty(self) -> None: + self.dirty = True + + def dump(self) -> None: + with open(self.file_path, "w+") as stream: + self.ruamel_yaml.dump(self.yaml, stream) class RootRepoFactory: @staticmethod - def __get_bootstrap_entries(bootstrap_values_file: str) -> list[Any]: + def __load_tenants_from_bootstrap_values(root_repo: GitRepo) -> dict[AppTenantConfig]: + boostrap_tenant_list = RootRepoFactory.__get_bootstrap_tenant_list(root_repo) + tenants = dict() + for bootstrap_tenant in boostrap_tenant_list: + try: + tenant_name = bootstrap_tenant["name"] + absolute_tenant_file_path = root_repo.get_full_file_path("apps/" + tenant_name + ".yaml") + yaml = yaml_file_load(absolute_tenant_file_path) + tenants[tenant_name] = AppTenantConfig( + yaml=yaml, + file_path=absolute_tenant_file_path, + ) + except FileNotFoundError as ex: + raise GitOpsException(f"File '{absolute_tenant_file_path}' not found in root repository.") from ex + return tenants + + @staticmethod + def __get_bootstrap_tenant_list(root_repo) -> list[str]: + root_repo.clone() try: - bootstrap_yaml = yaml_file_load(bootstrap_values_file) + boostrap_values_path = root_repo.get_full_file_path("bootstrap/values.yaml") + bootstrap_yaml = yaml_file_load(boostrap_values_path) except FileNotFoundError as ex: raise GitOpsException("File 'bootstrap/values.yaml' not found in root repository.") from ex + bootstrap_tenants = None if "bootstrap" in bootstrap_yaml: - return RootRepoFactory.bootstrap_name_validator(bootstrap_yaml["bootstrap"]) + bootstrap_tenants = bootstrap_yaml["bootstrap"] if "config" in bootstrap_yaml and "bootstrap" in bootstrap_yaml["config"]: - return RootRepoFactory.bootstrap_name_validator(bootstrap_yaml["config"]["bootstrap"]) - raise GitOpsException("Cannot find key 'bootstrap' or 'config.bootstrap' in 'bootstrap/values.yaml'") + bootstrap_tenants = bootstrap_yaml["config"]["bootstrap"] + RootRepoFactory.validate_bootstrap_tenants(bootstrap_tenants) + return bootstrap_tenants @staticmethod - def bootstrap_name_validator(bootstrap_entries: list[Any]) -> list[Any]: + def validate_bootstrap_tenants(bootstrap_entries: list[Any]) -> list[Any]: + if bootstrap_entries is None: + raise GitOpsException("Cannot find key 'bootstrap' or 'config.bootstrap' in 'bootstrap/values.yaml'") for bootstrap_entry in bootstrap_entries: if "name" not in bootstrap_entry: raise GitOpsException("Every bootstrap entry must have a 'name' property.") return bootstrap_entries - @staticmethod - def __generate_tenant_app_dict_from_root_repo( - root_config_git_repo: GitRepo, bootstrap: Any - ) -> dict[str, "AppTenantConfig"]: - tenant_app_dict = {} - for bootstrap_entry in bootstrap: - tenant_apps_config_file_name = "apps/" + bootstrap_entry["name"] + ".yaml" - logging.info("Analyzing %s in root repository", tenant_apps_config_file_name) - tenant_apps_config_file = root_config_git_repo.get_full_file_path(tenant_apps_config_file_name) - try: - tenant_apps_config_content = yaml_file_load(tenant_apps_config_file) - except FileNotFoundError as ex: - raise GitOpsException(f"File '{tenant_apps_config_file_name}' not found in root repository.") from ex - # TODO exception handling for malformed yaml pylint: disable=fixme - found_apps_path = "applications" - if "config" in tenant_apps_config_content: - tenant_apps_config_content = tenant_apps_config_content["config"] - found_apps_path = "config.applications" - if "repository" not in tenant_apps_config_content: - raise GitOpsException(f"Cannot find key 'repository' in '{tenant_apps_config_file_name}'") - logging.info("adding %s", (bootstrap_entry["name"])) - atc = AppTenantConfigFactory().create_root_repo_tenant_config( - data=tenant_apps_config_content, - name=bootstrap_entry["name"], - file_path=tenant_apps_config_file, - file_name=tenant_apps_config_file_name, - found_apps_path=found_apps_path, - ) - tenant_app_dict.update({bootstrap_entry["name"]: atc}) - return tenant_app_dict - - # TODO SHOULD THIS FULL METHOD INSTEAD OF POPULATING pylint: disable=fixme - @staticmethod - def __get_all_apps_list(tenant_dict: Any) -> dict[str, list[Any]]: - all_apps_list = dict() - for tenant in tenant_dict: - value = traverse_config(tenant_dict[tenant].data, tenant_dict[tenant].config_api_version.path) - all_apps_list.update({tenant: list((dict(value).keys()))}) - return all_apps_list - def create(self, root_repo: GitRepo) -> "RootRepo": - name = root_repo.get_clone_url().split("/")[-1].removesuffix(".git") - root_repo.clone() - bootstrap_values_file = root_repo.get_full_file_path("bootstrap/values.yaml") - bootstrap = self.__get_bootstrap_entries(bootstrap_values_file) - tenant_dict = self.__generate_tenant_app_dict_from_root_repo(root_repo, bootstrap) - all_app_list = self.__get_all_apps_list(tenant_dict) - return RootRepo(name, tenant_dict, bootstrap, all_app_list) + root_repo_tenants = self.__load_tenants_from_bootstrap_values(root_repo) + return RootRepo(root_repo_tenants) @dataclass class RootRepo: - name: str # root repository name - tenant_dict: dict[ - str, "AppTenantConfig" - ] # TODO of AppTenantConfig #list of the tenant configs in the root repository (in apps folder) pylint: disable=fixme - bootstrap: list[Any] # list of tenants to be bootstrapped, derived form values.yaml in bootstrap root repo dict - all_app_list: dict[str, list[Any]] # list of apps without custormer separation + tenants: dict[AppTenantConfig] def list_tenants(self) -> list[str]: return list(self.tenant_dict.keys()) + def get_tenant_by_repo_url(self, repo_url: str) -> AppTenantConfig: + for tenant in self.tenants.values(): + if tenant.repo_url == repo_url: + return tenant + return None + + def get_all_applications(self) -> list[str]: + apps = list() + for tenant in self.tenants: + apps.extend(tenant.tenant_config["applications"].keys()) + return apps + + def validate_tenant(self, tenant_config): + apps_from_other_tenants = list() + for tenant in self.tenants.values(): + if tenant.repo_url != tenant_config.repo_url: + apps_from_other_tenants.extend(tenant.list_apps().keys()) + for app_name in tenant_config.list_apps().keys(): + if app_name in apps_from_other_tenants: + raise GitOpsException(f"Application '{app_name}' already exists in a different repository") + class SyncAppsCommand(Command): @dataclass(frozen=True) @@ -248,67 +225,20 @@ def _sync_apps_command(args: SyncAppsCommand.Args) -> None: __sync_apps(team_config_git_repo, root_config_git_repo, args.git_user, args.git_email) -def __validate_if_app_not_present( - apps_from_other_repos: dict[Any, Any], team_config_app_name: str, tenant_config_repo_apps: Any -) -> None: - apps_from_other_repos.pop(team_config_app_name) - for app in list(tenant_config_repo_apps.keys()): - for tenant in apps_from_other_repos.values(): - if app in tenant: - raise GitOpsException(f"Application '{app}' already exists in a different repository") - - -def __sync_apps(team_config_git_repo: GitRepo, root_config_git_repo: GitRepo, git_user: str, git_email: str) -> None: - logging.info("Team config repository: %s", team_config_git_repo.get_clone_url()) - logging.info("Root config repository: %s", root_config_git_repo.get_clone_url()) - - root_repo = RootRepoFactory().create(root_repo=root_config_git_repo) - team_config_app_name = team_config_git_repo.get_clone_url().split("/")[-1].removesuffix(".git") - if not team_config_app_name in root_repo.list_tenants(): +# TODO: BETTER NAMES FOR STUFF HERE +def __sync_apps(tenant_git_repo: GitRepo, root_git_repo: GitRepo, git_user: str, git_email: str) -> None: + logging.info("Team config repository: %s", tenant_git_repo.get_clone_url()) + logging.info("Root config repository: %s", root_git_repo.get_clone_url()) + root_repo = RootRepoFactory().create(root_repo=root_git_repo) + root_repo_tenant = root_repo.get_tenant_by_repo_url(tenant_git_repo.get_clone_url()) + if root_repo_tenant is None: raise GitOpsException("Couldn't find config file for apps repository in root repository's 'apps/' directory") - tenant_config_team_repo = AppTenantConfigFactory().create_team_repo_tenant_config( - name=team_config_app_name, config_source_repository=team_config_git_repo - ) - - tenant_config_repo_apps = tenant_config_team_repo.list_apps() - __validate_if_app_not_present(root_repo.all_app_list.copy(), team_config_app_name, tenant_config_repo_apps) - - logging.info( - "Found %s app(s) in apps repository: %s", len(tenant_config_repo_apps), ", ".join(tenant_config_repo_apps) - ) - logging.info("Searching apps repository in root repository's 'apps/' directory...") - - current_repo_apps = root_repo.tenant_dict[team_config_app_name].list_apps() - for app in list(current_repo_apps.keys()): - if ( - current_repo_apps.get(app) is not None - ): # None is returend when application does not have any additional parameters - custom_app_config_item = current_repo_apps[app].get("customAppConfig") - current_repo_apps[app].clear() - if ( - custom_app_config_item is not None - ): # None is returend when application does not have custom configuration - current_repo_apps[app]["customAppConfig"] = custom_app_config_item - - if current_repo_apps == tenant_config_repo_apps: - logging.info("Root repository already up-to-date. I'm done here.") - return - - apps_config_file = root_repo.tenant_dict[team_config_app_name].file_path - apps_config_file_name = root_repo.tenant_dict[team_config_app_name].file_name - logging.info("Sync applications in root repository's %s.", apps_config_file_name) - merge_yaml_element( - apps_config_file, - root_repo.tenant_dict[team_config_app_name].found_apps_path, - { - repo_app: traverse_config( - tenant_config_team_repo.data, tenant_config_team_repo.config_api_version.path - ).get(repo_app, "{}") - for repo_app in tenant_config_repo_apps - }, - ) - - __commit_and_push(team_config_git_repo, root_config_git_repo, git_user, git_email, apps_config_file_name) + tenant_from_repo = TenantAppTenantConfigFactory().create(tenant_repo=tenant_git_repo) + root_repo.validate_tenant(tenant_from_repo) + root_repo_tenant.merge_applications(tenant_from_repo) + if root_repo_tenant.dirty: + root_repo_tenant.dump() + __commit_and_push(tenant_git_repo, root_git_repo, git_user, git_email, root_repo_tenant.file_path) def __commit_and_push( From 46b543908cc9da0eb9c6e64eb28d66b096533343 Mon Sep 17 00:00:00 2001 From: Nikolas Philips Date: Fri, 9 Dec 2022 20:55:55 +0100 Subject: [PATCH 13/17] Complete refactoring --- gitopscli/commands/sync_apps.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/gitopscli/commands/sync_apps.py b/gitopscli/commands/sync_apps.py index 25c6e773..4e62a9b3 100644 --- a/gitopscli/commands/sync_apps.py +++ b/gitopscli/commands/sync_apps.py @@ -37,7 +37,8 @@ def __generate_config_from_tenant_repo( yaml["config"]["applications"][app_dir]["customAppConfig"] = custom_app_config return yaml - def __get_all_tenant_applications(self, tenant_repo): + @staticmethod + def __get_all_tenant_applications(tenant_repo): repo_dir = tenant_repo.get_full_file_path(".") applist = { name @@ -97,7 +98,10 @@ def __update_custom_app_config(self, desired_apps): del existing_application_value["customAppConfig"] self.__set_dirty() else: - if "customAppConfig" not in existing_application_value or existing_application_value["customAppConfig"] != desired_app_value["customAppConfig"]: + if ( + "customAppConfig" not in existing_application_value + or existing_application_value["customAppConfig"] != desired_app_value["customAppConfig"] + ): existing_application_value["customAppConfig"] = desired_app_value["customAppConfig"] self.__set_dirty() @@ -174,7 +178,7 @@ class RootRepo: tenants: dict[AppTenantConfig] def list_tenants(self) -> list[str]: - return list(self.tenant_dict.keys()) + return list(self.tenants.keys()) def get_tenant_by_repo_url(self, repo_url: str) -> AppTenantConfig: for tenant in self.tenants.values(): @@ -225,7 +229,7 @@ def _sync_apps_command(args: SyncAppsCommand.Args) -> None: __sync_apps(team_config_git_repo, root_config_git_repo, args.git_user, args.git_email) -# TODO: BETTER NAMES FOR STUFF HERE +# TODO: BETTER NAMES FOR STUFF HERE pylint: disable=fixme def __sync_apps(tenant_git_repo: GitRepo, root_git_repo: GitRepo, git_user: str, git_email: str) -> None: logging.info("Team config repository: %s", tenant_git_repo.get_clone_url()) logging.info("Root config repository: %s", root_git_repo.get_clone_url()) From 06b1a60a9f6e541a8804d86ad2cdbb381ae13a17 Mon Sep 17 00:00:00 2001 From: Nikolas Philips Date: Sun, 11 Dec 2022 23:37:14 +0100 Subject: [PATCH 14/17] Fix tests & mypy --- gitopscli/commands/sync_apps.py | 91 +++++++++++++++++++------------- tests/commands/test_sync_apps.py | 66 +++++++++++------------ 2 files changed, 86 insertions(+), 71 deletions(-) diff --git a/gitopscli/commands/sync_apps.py b/gitopscli/commands/sync_apps.py index 4e62a9b3..7b8e31e9 100644 --- a/gitopscli/commands/sync_apps.py +++ b/gitopscli/commands/sync_apps.py @@ -1,12 +1,10 @@ import logging import os -from typing import Any -from dataclasses import dataclass -from ruamel.yaml import YAML -from ruamel.yaml.comments import CommentedMap +from typing import Any, Optional, List +from dataclasses import dataclass, field from gitopscli.git_api import GitApiConfig, GitRepo, GitRepoApiFactory from gitopscli.gitops_exception import GitOpsException -from gitopscli.io_api.yaml_util import yaml_file_load, yaml_load +from gitopscli.io_api.yaml_util import yaml_file_load, yaml_load, yaml_file_dump from .command import Command @@ -14,7 +12,7 @@ class TenantAppTenantConfigFactory: def __generate_config_from_tenant_repo( self, tenant_repo: GitRepo ) -> Any: # TODO: supposed to be ruamel object than Any pylint: disable=fixme - tenant_app_dirs = self.__get_all_tenant_applications(tenant_repo) + tenant_app_dirs = self.__get_all_tenant_applications_dirs(tenant_repo) tenant_config_template = """ config: repository: {} @@ -38,7 +36,7 @@ def __generate_config_from_tenant_repo( return yaml @staticmethod - def __get_all_tenant_applications(tenant_repo): + def __get_all_tenant_applications_dirs(tenant_repo: GitRepo) -> set[str]: repo_dir = tenant_repo.get_full_file_path(".") applist = { name @@ -66,21 +64,22 @@ def create( @dataclass class AppTenantConfig: - yaml: CommentedMap | dict[Any, Any] # TODO: supposed to be ordereddict from ruamel pylint: disable=fixme - tenant_config: CommentedMap = None | dict[Any, Any] + yaml: dict[str, dict[str, Any]] + tenant_config: dict[str, dict[str, Any]] = field(default_factory=dict) repo_url: str = "" file_path: str = "" dirty: bool = False - ruamel_yaml: YAML = YAML() def __post_init__(self) -> None: if "config" in self.yaml: self.tenant_config = self.yaml["config"] else: self.tenant_config = self.yaml - self.repo_url = self.tenant_config["repository"] + if "repository" not in self.tenant_config: + raise GitOpsException("Cannot find key 'repository' in " + self.file_path) + self.repo_url = str(self.tenant_config["repository"]) - def list_apps(self) -> dict[dict[Any]]: + def list_apps(self) -> dict[str, dict[str, Any]]: return dict(self.tenant_config["applications"]) def merge_applications(self, desired_tenant_config: "AppTenantConfig") -> None: @@ -89,12 +88,17 @@ def merge_applications(self, desired_tenant_config: "AppTenantConfig") -> None: self.__add_new_applications(desired_apps) self.__update_custom_app_config(desired_apps) - def __update_custom_app_config(self, desired_apps): + def __update_custom_app_config(self, desired_apps: dict[str, dict[str, Any]]) -> None: for desired_app_name, desired_app_value in desired_apps.items(): - if desired_app_name in self.list_apps().keys(): - existing_application_value = self.list_apps().get(desired_app_name) + if desired_app_name in self.list_apps(): + existing_application_value = self.list_apps()[desired_app_name] if "customAppConfig" not in desired_app_value: - if "customAppConfig" in existing_application_value: + if existing_application_value and "customAppConfig" in existing_application_value: + logging.info( + "Removing customAppConfig in for %s in %s applications", + existing_application_value, + self.file_path, + ) del existing_application_value["customAppConfig"] self.__set_dirty() else: @@ -102,32 +106,35 @@ def __update_custom_app_config(self, desired_apps): "customAppConfig" not in existing_application_value or existing_application_value["customAppConfig"] != desired_app_value["customAppConfig"] ): + logging.info( + "Updating customAppConfig in for %s in %s applications", + existing_application_value, + self.file_path, + ) existing_application_value["customAppConfig"] = desired_app_value["customAppConfig"] self.__set_dirty() - def __add_new_applications(self, desired_apps): + def __add_new_applications(self, desired_apps: dict[str, Any]) -> None: for desired_app_name, desired_app_value in desired_apps.items(): if desired_app_name not in self.list_apps().keys(): + logging.info("Adding % in %s applications", desired_app_name, self.file_path) self.tenant_config["applications"][desired_app_name] = desired_app_value self.__set_dirty() - def __delete_removed_applications(self, desired_apps): + def __delete_removed_applications(self, desired_apps: dict[str, Any]) -> None: for current_app in self.list_apps().keys(): if current_app not in desired_apps.keys(): + logging.info("Removing % from %s applications", current_app, self.file_path) del self.tenant_config["applications"][current_app] self.__set_dirty() def __set_dirty(self) -> None: self.dirty = True - def dump(self) -> None: - with open(self.file_path, "w+") as stream: - self.ruamel_yaml.dump(self.yaml, stream) - class RootRepoFactory: @staticmethod - def __load_tenants_from_bootstrap_values(root_repo: GitRepo) -> dict[AppTenantConfig]: + def __load_tenants_from_bootstrap_values(root_repo: GitRepo) -> dict[str, AppTenantConfig]: boostrap_tenant_list = RootRepoFactory.__get_bootstrap_tenant_list(root_repo) tenants = dict() for bootstrap_tenant in boostrap_tenant_list: @@ -144,29 +151,28 @@ def __load_tenants_from_bootstrap_values(root_repo: GitRepo) -> dict[AppTenantCo return tenants @staticmethod - def __get_bootstrap_tenant_list(root_repo) -> list[str]: + def __get_bootstrap_tenant_list(root_repo: GitRepo) -> List[Any]: root_repo.clone() try: boostrap_values_path = root_repo.get_full_file_path("bootstrap/values.yaml") bootstrap_yaml = yaml_file_load(boostrap_values_path) except FileNotFoundError as ex: raise GitOpsException("File 'bootstrap/values.yaml' not found in root repository.") from ex - bootstrap_tenants = None + bootstrap_tenants = [] if "bootstrap" in bootstrap_yaml: - bootstrap_tenants = bootstrap_yaml["bootstrap"] + bootstrap_tenants = list(bootstrap_yaml["bootstrap"]) if "config" in bootstrap_yaml and "bootstrap" in bootstrap_yaml["config"]: - bootstrap_tenants = bootstrap_yaml["config"]["bootstrap"] + bootstrap_tenants = list(bootstrap_yaml["config"]["bootstrap"]) RootRepoFactory.validate_bootstrap_tenants(bootstrap_tenants) return bootstrap_tenants @staticmethod - def validate_bootstrap_tenants(bootstrap_entries: list[Any]) -> list[Any]: - if bootstrap_entries is None: + def validate_bootstrap_tenants(bootstrap_entries: Optional[List[Any]]) -> None: + if not bootstrap_entries: raise GitOpsException("Cannot find key 'bootstrap' or 'config.bootstrap' in 'bootstrap/values.yaml'") for bootstrap_entry in bootstrap_entries: if "name" not in bootstrap_entry: raise GitOpsException("Every bootstrap entry must have a 'name' property.") - return bootstrap_entries def create(self, root_repo: GitRepo) -> "RootRepo": root_repo_tenants = self.__load_tenants_from_bootstrap_values(root_repo) @@ -175,25 +181,25 @@ def create(self, root_repo: GitRepo) -> "RootRepo": @dataclass class RootRepo: - tenants: dict[AppTenantConfig] + tenants: dict[str, AppTenantConfig] def list_tenants(self) -> list[str]: return list(self.tenants.keys()) - def get_tenant_by_repo_url(self, repo_url: str) -> AppTenantConfig: + def get_tenant_by_repo_url(self, repo_url: str) -> Optional[AppTenantConfig]: for tenant in self.tenants.values(): if tenant.repo_url == repo_url: return tenant return None def get_all_applications(self) -> list[str]: - apps = list() - for tenant in self.tenants: - apps.extend(tenant.tenant_config["applications"].keys()) + apps: list[str] = list() + for tenant in self.tenants.values(): + apps.extend(tenant.list_apps().keys()) return apps - def validate_tenant(self, tenant_config): - apps_from_other_tenants = list() + def validate_tenant(self, tenant_config: AppTenantConfig) -> None: + apps_from_other_tenants: list[str] = list() for tenant in self.tenants.values(): if tenant.repo_url != tenant_config.repo_url: apps_from_other_tenants.extend(tenant.list_apps().keys()) @@ -238,11 +244,20 @@ def __sync_apps(tenant_git_repo: GitRepo, root_git_repo: GitRepo, git_user: str, if root_repo_tenant is None: raise GitOpsException("Couldn't find config file for apps repository in root repository's 'apps/' directory") tenant_from_repo = TenantAppTenantConfigFactory().create(tenant_repo=tenant_git_repo) + logging.info( + "Found %s app(s) in apps repository: %s", + len(tenant_from_repo.list_apps().keys()), + ", ".join(tenant_from_repo.list_apps().keys()), + ) root_repo.validate_tenant(tenant_from_repo) root_repo_tenant.merge_applications(tenant_from_repo) if root_repo_tenant.dirty: - root_repo_tenant.dump() + logging.info("Appling changes to: %s", root_repo_tenant.file_path) + yaml_file_dump(root_repo_tenant.yaml, root_repo_tenant.file_path) + logging.info("Commiting and pushing changes to %s", root_git_repo.get_clone_url()) __commit_and_push(tenant_git_repo, root_git_repo, git_user, git_email, root_repo_tenant.file_path) + else: + logging.info("No changes applied to %s", root_repo_tenant.file_path) def __commit_and_push( diff --git a/tests/commands/test_sync_apps.py b/tests/commands/test_sync_apps.py index c77a86a7..25c39b5f 100644 --- a/tests/commands/test_sync_apps.py +++ b/tests/commands/test_sync_apps.py @@ -4,8 +4,8 @@ import unittest from unittest.mock import call from gitopscli.git_api import GitProvider, GitRepo, GitRepoApi, GitRepoApiFactory -from gitopscli.commands.sync_apps import SyncAppsCommand -from gitopscli.io_api.yaml_util import merge_yaml_element, yaml_file_load +from gitopscli.commands.sync_apps import SyncAppsCommand, AppTenantConfig +from gitopscli.io_api.yaml_util import merge_yaml_element, yaml_file_load, yaml_file_dump from gitopscli.gitops_exception import GitOpsException from ruamel.yaml.compat import ordereddict from .mock_mixin import MockMixin @@ -86,8 +86,8 @@ def setUp(self): }, }[file_path] - self.merge_yaml_element_mock = self.monkey_patch(merge_yaml_element) - self.merge_yaml_element_mock.return_value = None + self.yaml_file_dump_mock = self.monkey_patch(yaml_file_dump) + self.yaml_file_dump_mock.return_value = None self.seal_mocks() @@ -102,18 +102,13 @@ def test_sync_apps_happy_flow(self): call.logging.info("Team config repository: %s", "https://repository.url/team/team-non-prod.git"), call.GitRepo_root.get_clone_url(), call.logging.info("Root config repository: %s", "https://repository.url/root/root-config.git"), - call.GitRepo_root.get_clone_url(), call.GitRepo_root.clone(), call.GitRepo_root.get_full_file_path("bootstrap/values.yaml"), call.yaml_file_load("/tmp/root-config-repo/bootstrap/values.yaml"), - call.logging.info("Analyzing %s in root repository", "apps/team-non-prod.yaml"), call.GitRepo_root.get_full_file_path("apps/team-non-prod.yaml"), call.yaml_file_load("/tmp/root-config-repo/apps/team-non-prod.yaml"), - call.logging.info("adding %s", "team-non-prod"), - call.logging.info("Analyzing %s in root repository", "apps/other-team-non-prod.yaml"), call.GitRepo_root.get_full_file_path("apps/other-team-non-prod.yaml"), call.yaml_file_load("/tmp/root-config-repo/apps/other-team-non-prod.yaml"), - call.logging.info("adding %s", "other-team-non-prod"), call.GitRepo_team.get_clone_url(), call.GitRepo_team.clone(), call.GitRepo_team.get_full_file_path("."), @@ -121,19 +116,29 @@ def test_sync_apps_happy_flow(self): call.os.path.join("/tmp/team-config-repo/.", "my-app"), call.os.path.isdir("/tmp/team-config-repo/./my-app"), call.GitRepo_team.get_clone_url(), - call.GitRepo_team.get_full_file_path("my-app/app_value_file.yaml"), - call.os.path.exists("/tmp/team-config-repo/my-app/app_value_file.yaml"), - call.GitRepo_team.get_clone_url(), + call.GitRepo_team.get_full_file_path("my-app/.config.yaml"), + call.os.path.exists("/tmp/team-config-repo/my-app/.config.yaml"), call.logging.info("Found %s app(s) in apps repository: %s", 1, "my-app"), - call.logging.info("Searching apps repository in root repository's 'apps/' directory..."), - call.logging.info("Sync applications in root repository's %s.", "apps/team-non-prod.yaml"), - call.merge_yaml_element( + call.logging.info( + "Removing % from %s applications", "some-other-app-1", "/tmp/root-config-repo/apps/team-non-prod.yaml" + ), + call.logging.info("Adding % in %s applications", "my-app", "/tmp/root-config-repo/apps/team-non-prod.yaml"), + call.logging.info("Appling changes to: %s", "/tmp/root-config-repo/apps/team-non-prod.yaml"), + call.yaml_file_dump( + { + "config": { + "repository": "https://repository.url/team/team-non-prod.git", + "applications": {"my-app": ordereddict()}, + } + }, "/tmp/root-config-repo/apps/team-non-prod.yaml", - "config.applications", - {"my-app": {}}, ), + call.GitRepo_root.get_clone_url(), + call.logging.info("Commiting and pushing changes to %s", "https://repository.url/root/root-config.git"), call.GitRepo_team.get_author_from_last_commit(), - call.GitRepo_root.commit("GIT_USER", "GIT_EMAIL", "author updated apps/team-non-prod.yaml"), + call.GitRepo_root.commit( + "GIT_USER", "GIT_EMAIL", "author updated /tmp/root-config-repo/apps/team-non-prod.yaml" + ), call.GitRepo_root.push(), ] @@ -162,18 +167,13 @@ def test_sync_apps_already_up_to_date(self): call.logging.info("Team config repository: %s", "https://repository.url/team/team-non-prod.git"), call.GitRepo_root.get_clone_url(), call.logging.info("Root config repository: %s", "https://repository.url/root/root-config.git"), - call.GitRepo_root.get_clone_url(), call.GitRepo_root.clone(), call.GitRepo_root.get_full_file_path("bootstrap/values.yaml"), call.yaml_file_load("/tmp/root-config-repo/bootstrap/values.yaml"), - call.logging.info("Analyzing %s in root repository", "apps/team-non-prod.yaml"), call.GitRepo_root.get_full_file_path("apps/team-non-prod.yaml"), call.yaml_file_load("/tmp/root-config-repo/apps/team-non-prod.yaml"), - call.logging.info("adding %s", "team-non-prod"), - call.logging.info("Analyzing %s in root repository", "apps/other-team-non-prod.yaml"), call.GitRepo_root.get_full_file_path("apps/other-team-non-prod.yaml"), call.yaml_file_load("/tmp/root-config-repo/apps/other-team-non-prod.yaml"), - call.logging.info("adding %s", "other-team-non-prod"), call.GitRepo_team.get_clone_url(), call.GitRepo_team.clone(), call.GitRepo_team.get_full_file_path("."), @@ -181,12 +181,10 @@ def test_sync_apps_already_up_to_date(self): call.os.path.join("/tmp/team-config-repo/.", "my-app"), call.os.path.isdir("/tmp/team-config-repo/./my-app"), call.GitRepo_team.get_clone_url(), - call.GitRepo_team.get_full_file_path("my-app/app_value_file.yaml"), - call.os.path.exists("/tmp/team-config-repo/my-app/app_value_file.yaml"), - call.GitRepo_team.get_clone_url(), + call.GitRepo_team.get_full_file_path("my-app/.config.yaml"), + call.os.path.exists("/tmp/team-config-repo/my-app/.config.yaml"), call.logging.info("Found %s app(s) in apps repository: %s", 1, "my-app"), - call.logging.info("Searching apps repository in root repository's 'apps/' directory..."), - call.logging.info("Root repository already up-to-date. I'm done here."), + call.logging.info("No changes applied to %s", "/tmp/root-config-repo/apps/team-non-prod.yaml"), ] def test_sync_apps_bootstrap_chart(self): @@ -197,17 +195,17 @@ def test_sync_apps_bootstrap_chart(self): } }, "/tmp/root-config-repo/apps/team-non-prod.yaml": { - "repository": "https://team.config.repo.git", + "repository": "https://repository.url/team/team-non-prod.git", "applications": {"my-app": None}, # my-app already exists }, "/tmp/root-config-repo/apps/other-team-non-prod.yaml": { - "repository": "https://other-team.config.repo.git", + "repository": "https://repository.url/team/other-team-non-prod.git", "applications": {}, }, }[file_path] try: SyncAppsCommand(ARGS).execute() - except GitOpsException: + except GitOpsException as ex: self.fail("'config.bootstrap' should be read correctly'") def test_sync_apps_bootstrap_yaml_not_found(self): @@ -271,7 +269,9 @@ def file_load_mock_side_effect(file_path): SyncAppsCommand(ARGS).execute() self.fail() except GitOpsException as ex: - self.assertEqual("File 'apps/team-non-prod.yaml' not found in root repository.", str(ex)) + self.assertEqual( + "File '/tmp/root-config-repo/apps/team-non-prod.yaml' not found in root repository.", str(ex) + ) def test_sync_apps_missing_repository_element_in_team_yaml(self): self.yaml_file_load_mock.side_effect = lambda file_path: { @@ -286,7 +286,7 @@ def test_sync_apps_missing_repository_element_in_team_yaml(self): SyncAppsCommand(ARGS).execute() self.fail() except GitOpsException as ex: - self.assertEqual("Cannot find key 'repository' in 'apps/team-non-prod.yaml'", str(ex)) + self.assertEqual("Cannot find key 'repository' in /tmp/root-config-repo/apps/team-non-prod.yaml", str(ex)) def test_sync_apps_undefined_team_repo(self): self.yaml_file_load_mock.side_effect = lambda file_path: { From 03e225146a1473ce7a7fc50c32ba2eb0773c4bc2 Mon Sep 17 00:00:00 2001 From: Nikolas Philips Date: Mon, 12 Dec 2022 11:22:09 +0100 Subject: [PATCH 15/17] Fix tests and move classes out of sync_apps.py --- gitopscli/appconfig_api/app_tenant_config.py | 132 ++++++++++++ gitopscli/appconfig_api/root_repo.py | 83 +++++++ gitopscli/commands/sync_apps.py | 214 +------------------ tests/commands/test_sync_apps.py | 34 +-- 4 files changed, 242 insertions(+), 221 deletions(-) create mode 100644 gitopscli/appconfig_api/app_tenant_config.py create mode 100644 gitopscli/appconfig_api/root_repo.py diff --git a/gitopscli/appconfig_api/app_tenant_config.py b/gitopscli/appconfig_api/app_tenant_config.py new file mode 100644 index 00000000..62984e80 --- /dev/null +++ b/gitopscli/appconfig_api/app_tenant_config.py @@ -0,0 +1,132 @@ +import logging +from dataclasses import dataclass, field +import os +from typing import Any + +from gitopscli.git_api import GitRepo +from gitopscli.io_api.yaml_util import yaml_load, yaml_file_load + +from gitopscli.gitops_exception import GitOpsException + + +@dataclass +class AppTenantConfig: + yaml: dict[str, dict[str, Any]] + tenant_config: dict[str, dict[str, Any]] = field(default_factory=dict) + repo_url: str = "" + file_path: str = "" + dirty: bool = False + + def __post_init__(self) -> None: + if "config" in self.yaml: + self.tenant_config = self.yaml["config"] + else: + self.tenant_config = self.yaml + if "repository" not in self.tenant_config: + raise GitOpsException("Cannot find key 'repository' in " + self.file_path) + self.repo_url = str(self.tenant_config["repository"]) + + def list_apps(self) -> dict[str, dict[str, Any]]: + return dict(self.tenant_config["applications"]) + + def merge_applications(self, desired_tenant_config: "AppTenantConfig") -> None: + desired_apps = desired_tenant_config.list_apps() + self.__delete_removed_applications(desired_apps) + self.__add_new_applications(desired_apps) + self.__update_custom_app_config(desired_apps) + + def __update_custom_app_config(self, desired_apps: dict[str, dict[str, Any]]) -> None: + for desired_app_name, desired_app_value in desired_apps.items(): + if desired_app_name in self.list_apps(): + existing_application_value = self.list_apps()[desired_app_name] + if "customAppConfig" not in desired_app_value: + if existing_application_value and "customAppConfig" in existing_application_value: + logging.info( + "Removing customAppConfig in for %s in %s applications", + existing_application_value, + self.file_path, + ) + del existing_application_value["customAppConfig"] + self.__set_dirty() + else: + if ( + "customAppConfig" not in existing_application_value + or existing_application_value["customAppConfig"] != desired_app_value["customAppConfig"] + ): + logging.info( + "Updating customAppConfig in for %s in %s applications", + existing_application_value, + self.file_path, + ) + existing_application_value["customAppConfig"] = desired_app_value["customAppConfig"] + self.__set_dirty() + + def __add_new_applications(self, desired_apps: dict[str, Any]) -> None: + for desired_app_name, desired_app_value in desired_apps.items(): + if desired_app_name not in self.list_apps().keys(): + logging.info("Adding % in %s applications", desired_app_name, self.file_path) + self.tenant_config["applications"][desired_app_name] = desired_app_value + self.__set_dirty() + + def __delete_removed_applications(self, desired_apps: dict[str, Any]) -> None: + for current_app in self.list_apps().keys(): + if current_app not in desired_apps.keys(): + logging.info("Removing % from %s applications", current_app, self.file_path) + del self.tenant_config["applications"][current_app] + self.__set_dirty() + + def __set_dirty(self) -> None: + self.dirty = True + + +def __generate_config_from_tenant_repo( + tenant_repo: GitRepo, +) -> Any: # TODO: supposed to be ruamel object than Any pylint: disable=fixme + tenant_app_dirs = __get_all_tenant_applications_dirs(tenant_repo) + tenant_config_template = """ + config: + repository: {} + applications: {{}} + """.format( + tenant_repo.get_clone_url() + ) + yaml = yaml_load(tenant_config_template) + for app_dir in tenant_app_dirs: + tenant_application_template = """ + {}: {{}} + """.format( + app_dir + ) + tenant_applications_yaml = yaml_load(tenant_application_template) + # dict path hardcoded as object generated will always be in v2 or later + yaml["config"]["applications"].update(tenant_applications_yaml) + custom_app_config = __get_custom_config(app_dir, tenant_repo) + if custom_app_config: + yaml["config"]["applications"][app_dir]["customAppConfig"] = custom_app_config + return yaml + + +def __get_all_tenant_applications_dirs(tenant_repo: GitRepo) -> set[str]: + repo_dir = tenant_repo.get_full_file_path(".") + applist = { + name + for name in os.listdir(repo_dir) + if os.path.isdir(os.path.join(repo_dir, name)) and not name.startswith(".") + } + return applist + + +def __get_custom_config(appname: str, tenant_config_git_repo: GitRepo) -> Any: + custom_config_path = tenant_config_git_repo.get_full_file_path(f"{appname}/.config.yaml") + if os.path.exists(custom_config_path): + custom_config_content = yaml_file_load(custom_config_path) + return custom_config_content + return dict() + + +def create_app_tenant_config_from_repo( + tenant_repo: GitRepo, +) -> "AppTenantConfig": + tenant_repo.clone() + tenant_config_yaml = __generate_config_from_tenant_repo(tenant_repo) + return AppTenantConfig(yaml=tenant_config_yaml) diff --git a/gitopscli/appconfig_api/root_repo.py b/gitopscli/appconfig_api/root_repo.py new file mode 100644 index 00000000..ae581965 --- /dev/null +++ b/gitopscli/appconfig_api/root_repo.py @@ -0,0 +1,83 @@ +from dataclasses import dataclass +from typing import List, Any, Optional + +from gitopscli.git_api import GitRepo +from gitopscli.io_api.yaml_util import yaml_file_load + +from gitopscli.appconfig_api.app_tenant_config import AppTenantConfig +from gitopscli.gitops_exception import GitOpsException + + +@dataclass +class RootRepo: + tenants: dict[str, AppTenantConfig] + + def list_tenants(self) -> list[str]: + return list(self.tenants.keys()) + + def get_tenant_by_repo_url(self, repo_url: str) -> Optional[AppTenantConfig]: + for tenant in self.tenants.values(): + if tenant.repo_url == repo_url: + return tenant + return None + + def get_all_applications(self) -> list[str]: + apps: list[str] = list() + for tenant in self.tenants.values(): + apps.extend(tenant.list_apps().keys()) + return apps + + def validate_tenant(self, tenant_config: AppTenantConfig) -> None: + apps_from_other_tenants: list[str] = list() + for tenant in self.tenants.values(): + if tenant.repo_url != tenant_config.repo_url: + apps_from_other_tenants.extend(tenant.list_apps().keys()) + for app_name in tenant_config.list_apps().keys(): + if app_name in apps_from_other_tenants: + raise GitOpsException(f"Application '{app_name}' already exists in a different repository") + + +def __load_tenants_from_bootstrap_values(root_repo: GitRepo) -> dict[str, AppTenantConfig]: + boostrap_tenant_list = __get_bootstrap_tenant_list(root_repo) + tenants = dict() + for bootstrap_tenant in boostrap_tenant_list: + try: + tenant_name = bootstrap_tenant["name"] + absolute_tenant_file_path = root_repo.get_full_file_path("apps/" + tenant_name + ".yaml") + yaml = yaml_file_load(absolute_tenant_file_path) + tenants[tenant_name] = AppTenantConfig( + yaml=yaml, + file_path=absolute_tenant_file_path, + ) + except FileNotFoundError as ex: + raise GitOpsException(f"File '{absolute_tenant_file_path}' not found in root repository.") from ex + return tenants + + +def __get_bootstrap_tenant_list(root_repo: GitRepo) -> List[Any]: + root_repo.clone() + try: + boostrap_values_path = root_repo.get_full_file_path("bootstrap/values.yaml") + bootstrap_yaml = yaml_file_load(boostrap_values_path) + except FileNotFoundError as ex: + raise GitOpsException("File 'bootstrap/values.yaml' not found in root repository.") from ex + bootstrap_tenants = [] + if "bootstrap" in bootstrap_yaml: + bootstrap_tenants = list(bootstrap_yaml["bootstrap"]) + if "config" in bootstrap_yaml and "bootstrap" in bootstrap_yaml["config"]: + bootstrap_tenants = list(bootstrap_yaml["config"]["bootstrap"]) + __validate_bootstrap_tenants(bootstrap_tenants) + return bootstrap_tenants + + +def __validate_bootstrap_tenants(bootstrap_entries: Optional[List[Any]]) -> None: + if not bootstrap_entries: + raise GitOpsException("Cannot find key 'bootstrap' or 'config.bootstrap' in 'bootstrap/values.yaml'") + for bootstrap_entry in bootstrap_entries: + if "name" not in bootstrap_entry: + raise GitOpsException("Every bootstrap entry must have a 'name' property.") + + +def create_root_repo(root_repo: GitRepo) -> "RootRepo": + root_repo_tenants = __load_tenants_from_bootstrap_values(root_repo) + return RootRepo(root_repo_tenants) diff --git a/gitopscli/commands/sync_apps.py b/gitopscli/commands/sync_apps.py index 7b8e31e9..5b63b852 100644 --- a/gitopscli/commands/sync_apps.py +++ b/gitopscli/commands/sync_apps.py @@ -1,211 +1,11 @@ import logging -import os -from typing import Any, Optional, List -from dataclasses import dataclass, field +from dataclasses import dataclass from gitopscli.git_api import GitApiConfig, GitRepo, GitRepoApiFactory from gitopscli.gitops_exception import GitOpsException -from gitopscli.io_api.yaml_util import yaml_file_load, yaml_load, yaml_file_dump -from .command import Command - - -class TenantAppTenantConfigFactory: - def __generate_config_from_tenant_repo( - self, tenant_repo: GitRepo - ) -> Any: # TODO: supposed to be ruamel object than Any pylint: disable=fixme - tenant_app_dirs = self.__get_all_tenant_applications_dirs(tenant_repo) - tenant_config_template = """ - config: - repository: {} - applications: {{}} - """.format( - tenant_repo.get_clone_url() - ) - yaml = yaml_load(tenant_config_template) - for app_dir in tenant_app_dirs: - tenant_application_template = """ - {}: {{}} - """.format( - app_dir - ) - tenant_applications_yaml = yaml_load(tenant_application_template) - # dict path hardcoded as object generated will always be in v2 or later - yaml["config"]["applications"].update(tenant_applications_yaml) - custom_app_config = self.__get_custom_config(app_dir, tenant_repo) - if custom_app_config: - yaml["config"]["applications"][app_dir]["customAppConfig"] = custom_app_config - return yaml - - @staticmethod - def __get_all_tenant_applications_dirs(tenant_repo: GitRepo) -> set[str]: - repo_dir = tenant_repo.get_full_file_path(".") - applist = { - name - for name in os.listdir(repo_dir) - if os.path.isdir(os.path.join(repo_dir, name)) and not name.startswith(".") - } - return applist - - @staticmethod - def __get_custom_config(appname: str, tenant_config_git_repo: GitRepo) -> Any: - custom_config_path = tenant_config_git_repo.get_full_file_path(f"{appname}/.config.yaml") - if os.path.exists(custom_config_path): - custom_config_content = yaml_file_load(custom_config_path) - return custom_config_content - return dict() - - def create( - self, - tenant_repo: GitRepo, - ) -> "AppTenantConfig": - tenant_repo.clone() - tenant_config_yaml = self.__generate_config_from_tenant_repo(tenant_repo) - return AppTenantConfig(yaml=tenant_config_yaml) - - -@dataclass -class AppTenantConfig: - yaml: dict[str, dict[str, Any]] - tenant_config: dict[str, dict[str, Any]] = field(default_factory=dict) - repo_url: str = "" - file_path: str = "" - dirty: bool = False - - def __post_init__(self) -> None: - if "config" in self.yaml: - self.tenant_config = self.yaml["config"] - else: - self.tenant_config = self.yaml - if "repository" not in self.tenant_config: - raise GitOpsException("Cannot find key 'repository' in " + self.file_path) - self.repo_url = str(self.tenant_config["repository"]) - - def list_apps(self) -> dict[str, dict[str, Any]]: - return dict(self.tenant_config["applications"]) - - def merge_applications(self, desired_tenant_config: "AppTenantConfig") -> None: - desired_apps = desired_tenant_config.list_apps() - self.__delete_removed_applications(desired_apps) - self.__add_new_applications(desired_apps) - self.__update_custom_app_config(desired_apps) - - def __update_custom_app_config(self, desired_apps: dict[str, dict[str, Any]]) -> None: - for desired_app_name, desired_app_value in desired_apps.items(): - if desired_app_name in self.list_apps(): - existing_application_value = self.list_apps()[desired_app_name] - if "customAppConfig" not in desired_app_value: - if existing_application_value and "customAppConfig" in existing_application_value: - logging.info( - "Removing customAppConfig in for %s in %s applications", - existing_application_value, - self.file_path, - ) - del existing_application_value["customAppConfig"] - self.__set_dirty() - else: - if ( - "customAppConfig" not in existing_application_value - or existing_application_value["customAppConfig"] != desired_app_value["customAppConfig"] - ): - logging.info( - "Updating customAppConfig in for %s in %s applications", - existing_application_value, - self.file_path, - ) - existing_application_value["customAppConfig"] = desired_app_value["customAppConfig"] - self.__set_dirty() - - def __add_new_applications(self, desired_apps: dict[str, Any]) -> None: - for desired_app_name, desired_app_value in desired_apps.items(): - if desired_app_name not in self.list_apps().keys(): - logging.info("Adding % in %s applications", desired_app_name, self.file_path) - self.tenant_config["applications"][desired_app_name] = desired_app_value - self.__set_dirty() - - def __delete_removed_applications(self, desired_apps: dict[str, Any]) -> None: - for current_app in self.list_apps().keys(): - if current_app not in desired_apps.keys(): - logging.info("Removing % from %s applications", current_app, self.file_path) - del self.tenant_config["applications"][current_app] - self.__set_dirty() - - def __set_dirty(self) -> None: - self.dirty = True - - -class RootRepoFactory: - @staticmethod - def __load_tenants_from_bootstrap_values(root_repo: GitRepo) -> dict[str, AppTenantConfig]: - boostrap_tenant_list = RootRepoFactory.__get_bootstrap_tenant_list(root_repo) - tenants = dict() - for bootstrap_tenant in boostrap_tenant_list: - try: - tenant_name = bootstrap_tenant["name"] - absolute_tenant_file_path = root_repo.get_full_file_path("apps/" + tenant_name + ".yaml") - yaml = yaml_file_load(absolute_tenant_file_path) - tenants[tenant_name] = AppTenantConfig( - yaml=yaml, - file_path=absolute_tenant_file_path, - ) - except FileNotFoundError as ex: - raise GitOpsException(f"File '{absolute_tenant_file_path}' not found in root repository.") from ex - return tenants - - @staticmethod - def __get_bootstrap_tenant_list(root_repo: GitRepo) -> List[Any]: - root_repo.clone() - try: - boostrap_values_path = root_repo.get_full_file_path("bootstrap/values.yaml") - bootstrap_yaml = yaml_file_load(boostrap_values_path) - except FileNotFoundError as ex: - raise GitOpsException("File 'bootstrap/values.yaml' not found in root repository.") from ex - bootstrap_tenants = [] - if "bootstrap" in bootstrap_yaml: - bootstrap_tenants = list(bootstrap_yaml["bootstrap"]) - if "config" in bootstrap_yaml and "bootstrap" in bootstrap_yaml["config"]: - bootstrap_tenants = list(bootstrap_yaml["config"]["bootstrap"]) - RootRepoFactory.validate_bootstrap_tenants(bootstrap_tenants) - return bootstrap_tenants - - @staticmethod - def validate_bootstrap_tenants(bootstrap_entries: Optional[List[Any]]) -> None: - if not bootstrap_entries: - raise GitOpsException("Cannot find key 'bootstrap' or 'config.bootstrap' in 'bootstrap/values.yaml'") - for bootstrap_entry in bootstrap_entries: - if "name" not in bootstrap_entry: - raise GitOpsException("Every bootstrap entry must have a 'name' property.") - - def create(self, root_repo: GitRepo) -> "RootRepo": - root_repo_tenants = self.__load_tenants_from_bootstrap_values(root_repo) - return RootRepo(root_repo_tenants) - - -@dataclass -class RootRepo: - tenants: dict[str, AppTenantConfig] - - def list_tenants(self) -> list[str]: - return list(self.tenants.keys()) - - def get_tenant_by_repo_url(self, repo_url: str) -> Optional[AppTenantConfig]: - for tenant in self.tenants.values(): - if tenant.repo_url == repo_url: - return tenant - return None - - def get_all_applications(self) -> list[str]: - apps: list[str] = list() - for tenant in self.tenants.values(): - apps.extend(tenant.list_apps().keys()) - return apps - - def validate_tenant(self, tenant_config: AppTenantConfig) -> None: - apps_from_other_tenants: list[str] = list() - for tenant in self.tenants.values(): - if tenant.repo_url != tenant_config.repo_url: - apps_from_other_tenants.extend(tenant.list_apps().keys()) - for app_name in tenant_config.list_apps().keys(): - if app_name in apps_from_other_tenants: - raise GitOpsException(f"Application '{app_name}' already exists in a different repository") +from gitopscli.io_api.yaml_util import yaml_file_dump +from gitopscli.commands.command import Command +from gitopscli.appconfig_api.app_tenant_config import create_app_tenant_config_from_repo +from gitopscli.appconfig_api.root_repo import create_root_repo class SyncAppsCommand(Command): @@ -239,11 +39,11 @@ def _sync_apps_command(args: SyncAppsCommand.Args) -> None: def __sync_apps(tenant_git_repo: GitRepo, root_git_repo: GitRepo, git_user: str, git_email: str) -> None: logging.info("Team config repository: %s", tenant_git_repo.get_clone_url()) logging.info("Root config repository: %s", root_git_repo.get_clone_url()) - root_repo = RootRepoFactory().create(root_repo=root_git_repo) + root_repo = create_root_repo(root_repo=root_git_repo) root_repo_tenant = root_repo.get_tenant_by_repo_url(tenant_git_repo.get_clone_url()) if root_repo_tenant is None: raise GitOpsException("Couldn't find config file for apps repository in root repository's 'apps/' directory") - tenant_from_repo = TenantAppTenantConfigFactory().create(tenant_repo=tenant_git_repo) + tenant_from_repo = create_app_tenant_config_from_repo(tenant_repo=tenant_git_repo) logging.info( "Found %s app(s) in apps repository: %s", len(tenant_from_repo.list_apps().keys()), diff --git a/tests/commands/test_sync_apps.py b/tests/commands/test_sync_apps.py index 25c39b5f..38e67545 100644 --- a/tests/commands/test_sync_apps.py +++ b/tests/commands/test_sync_apps.py @@ -2,10 +2,11 @@ import logging import os import unittest -from unittest.mock import call +from unittest.mock import call, patch + from gitopscli.git_api import GitProvider, GitRepo, GitRepoApi, GitRepoApiFactory -from gitopscli.commands.sync_apps import SyncAppsCommand, AppTenantConfig -from gitopscli.io_api.yaml_util import merge_yaml_element, yaml_file_load, yaml_file_dump +from gitopscli.commands.sync_apps import SyncAppsCommand +from gitopscli.io_api.yaml_util import yaml_file_load, yaml_file_dump from gitopscli.gitops_exception import GitOpsException from ruamel.yaml.compat import ordereddict from .mock_mixin import MockMixin @@ -28,7 +29,11 @@ class SyncAppsCommandTest(MockMixin, unittest.TestCase): def setUp(self): self.init_mock_manager(SyncAppsCommand) - self.os_mock = self.monkey_patch(os) + patcher = patch("gitopscli.appconfig_api.app_tenant_config.os", spec_set=os) + self.addCleanup(patcher.stop) + self.os_mock = patcher.start() + self.mock_manager.attach_mock(self.os_mock, "os") + self.os_mock.path.isdir.return_value = True self.os_mock.path.join.side_effect = posixpath.join # tests are designed to emulate posix env self.os_mock.listdir.return_value = ["my-app"] @@ -69,7 +74,12 @@ def setUp(self): id(self.root_config_git_repo_api_mock): self.root_config_git_repo_mock, }[id(api)] - self.yaml_file_load_mock = self.monkey_patch(yaml_file_load) + patcher = patch("gitopscli.appconfig_api.root_repo.yaml_file_load", spec_set=yaml_file_load) + self.addCleanup(patcher.stop) + self.yaml_file_load_mock = patcher.start() + self.mock_manager.attach_mock(self.yaml_file_load_mock, "yaml_file_load") + + # self.yaml_file_load_mock = self.monkey_patch(yaml_file_load) self.yaml_file_load_mock.side_effect = lambda file_path: { "/tmp/root-config-repo/bootstrap/values.yaml": { "bootstrap": [{"name": "team-non-prod"}, {"name": "other-team-non-prod"}], @@ -77,12 +87,12 @@ def setUp(self): "/tmp/root-config-repo/apps/team-non-prod.yaml": { "config": { "repository": "https://repository.url/team/team-non-prod.git", - "applications": {"some-other-app-1": None}, + "applications": {"some-other-app-1": {}}, } }, "/tmp/root-config-repo/apps/other-team-non-prod.yaml": { "repository": "https://repository.url/other-team/other-team-non-prod.git", - "applications": {"some-other-app-2": None}, + "applications": {"some-other-app-2": {}}, }, }[file_path] @@ -119,10 +129,6 @@ def test_sync_apps_happy_flow(self): call.GitRepo_team.get_full_file_path("my-app/.config.yaml"), call.os.path.exists("/tmp/team-config-repo/my-app/.config.yaml"), call.logging.info("Found %s app(s) in apps repository: %s", 1, "my-app"), - call.logging.info( - "Removing % from %s applications", "some-other-app-1", "/tmp/root-config-repo/apps/team-non-prod.yaml" - ), - call.logging.info("Adding % in %s applications", "my-app", "/tmp/root-config-repo/apps/team-non-prod.yaml"), call.logging.info("Appling changes to: %s", "/tmp/root-config-repo/apps/team-non-prod.yaml"), call.yaml_file_dump( { @@ -196,7 +202,7 @@ def test_sync_apps_bootstrap_chart(self): }, "/tmp/root-config-repo/apps/team-non-prod.yaml": { "repository": "https://repository.url/team/team-non-prod.git", - "applications": {"my-app": None}, # my-app already exists + "applications": {"my-app": {}}, # my-app already exists }, "/tmp/root-config-repo/apps/other-team-non-prod.yaml": { "repository": "https://repository.url/team/other-team-non-prod.git", @@ -312,11 +318,11 @@ def test_sync_apps_app_name_collission(self): }, "/tmp/root-config-repo/apps/team-non-prod.yaml": { "repository": "https://repository.url/team/team-non-prod.git", - "applications": {"some-other-app-1": None}, + "applications": {"some-other-app-1": {}}, }, "/tmp/root-config-repo/apps/other-team-non-prod.yaml": { "repository": "https://repository.url/other-team/other-team-non-prod.git", - "applications": {"my-app": None}, # the other-team already has an app named "my-app" + "applications": {"my-app": {}}, # the other-team already has an app named "my-app" }, }[file_path] From d2993ae7c302fbc9afe2d14fb874193e609095cb Mon Sep 17 00:00:00 2001 From: Nikolas Philips Date: Mon, 12 Dec 2022 11:37:51 +0100 Subject: [PATCH 16/17] Fix missing %s --- gitopscli/appconfig_api/app_tenant_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitopscli/appconfig_api/app_tenant_config.py b/gitopscli/appconfig_api/app_tenant_config.py index 62984e80..f3a16edb 100644 --- a/gitopscli/appconfig_api/app_tenant_config.py +++ b/gitopscli/appconfig_api/app_tenant_config.py @@ -71,7 +71,7 @@ def __add_new_applications(self, desired_apps: dict[str, Any]) -> None: def __delete_removed_applications(self, desired_apps: dict[str, Any]) -> None: for current_app in self.list_apps().keys(): if current_app not in desired_apps.keys(): - logging.info("Removing % from %s applications", current_app, self.file_path) + logging.info("Removing %s from %s applications", current_app, self.file_path) del self.tenant_config["applications"][current_app] self.__set_dirty() From 52a2211c0b85a937f67bfe619f281043467d0d1f Mon Sep 17 00:00:00 2001 From: makrelas <35281518+makrelas@users.noreply.github.com> Date: Tue, 13 Dec 2022 13:36:20 +0100 Subject: [PATCH 17/17] docs: fixed typo --- docs/commands/sync-apps.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/commands/sync-apps.md b/docs/commands/sync-apps.md index b365afa6..2e57ff21 100644 --- a/docs/commands/sync-apps.md +++ b/docs/commands/sync-apps.md @@ -29,7 +29,7 @@ root-config-repo/ └── values.yaml ``` ### app specific values -app specific values may be set using a app_value_file.yaml file directly in the app directory. gitopscli will process these values and add them under customAppConfig parameter of application +app specific values may be set using a .config.yaml file directly in the app directory. gitopscli will process these values and add them under customAppConfig parameter of application **tenantrepo.git/app1/app_value_file.yaml** ```yaml customvalue: test