GONRG-5320: domain is not set for istio policy

bootstrap_infra_on_prem/minio/minio_bootstrap.py

-import tempfile
-from string import Template
-import subprocess
-import os
-import sys
-import json
-import logging as log
-
-log.basicConfig(format='[%(levelname)s]: %(message)s', level=log.DEBUG)
-
-RECREATE_EXISTING = os.getenv('RECREATE_EXISTING', True)
-PARTITION_NAME = os.getenv('MINIO_PARTITION_NAME')
-
-
-## Helping functions
-
-def login_to_mc_admin():
-    """
-    Logins into mc client using subprocess command. 
-    Gets connection details from env vars: MINIO_HOST, MINIO_PORT, MINIO_ACCESS_KEY, MINIO_SECRET_KEY
-    Requires mc pre-installed and on PATH
-    """
-    minio_host = os.getenv('MINIO_HOST', 'http://127.0.0.1')
-    minio_port = os.getenv('MINIO_PORT', "9000")
-    minio_access_key = os.getenv('MINIO_ACCESS_KEY', 'minioadmin')
-    minio_secret_key = os.getenv('MINIO_SECRET_KEY', 'minioadmin')
-    code, out, err = execute_command(["mc", "alias", "set", "minio", f"{minio_host}:{minio_port}", minio_access_key, minio_secret_key, "--api", "S3v4"])
-    if (code==0):
-        log.info(f"Logged into mc client: {out}")
-    else:
-        log.error(f"[{code}] Failed to log into mc client: '{minio_host}:{minio_port}' with user '{minio_access_key}': {err}")
-
-def execute_command(arg_list):
-    """
-    Wrapper for command execution that uses subprocess
-    Returns tuple with (command_return_code, command_stdout_output, command_stderr_output)
-    """
-    command = subprocess.Popen(arg_list, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
-    out, err = command.communicate()
-    return (command.returncode, out.rstrip(), err.rstrip())
-
-def get_sorted_template_files(template_dir_path):
-    """
-    Returns list of alphabetically sorted template files from
-    'template_dir_path' which end with .tpl.json
-    """
-    template_files = [os.path.join(template_dir_path, f) for f in os.listdir(template_dir_path) if os.path.isfile(os.path.join(template_dir_path, f)) and f.endswith('.tpl.json')]
-    template_files.sort()
-    return template_files
-## Template functions
-
-def render_minio_update_json_template(filename: str, substitute_dict) -> str:
-    """
-    Reads json 'filename' file. 
-    Finds template values in it (specified as ${..} or {...}), as in Python's Template class
-    Evaluates whether all values can be substituted using provided dictionary 'substitute_dict'
-    In case of failure - prints list of missing variables and exits script completely with code 1
-    Otherwise, substitutes template variables with values and returns rendered json converted into dictionary
-    """
-    with open(filename, 'r') as f:
-        content = f.read()
-        template = Template(content)
-        required_vars = get_required_template_variables(content)
-        for required_var in required_vars:
-            if required_var not in substitute_dict:
-                log.error(f"Cannot substitute value for '{required_var}' in file '{filename}'.")
-                sys.exit(1)
-        evaluated_minio_json = template.substitute(substitute_dict)
-        return json.loads(evaluated_minio_json)
-
-def get_required_template_variables(template_str: str):
-    """
-    Returns list of template values names found in provided string
-    Template values are as specified in Template class (ex. ${..} or {..})
-    """
-    return [s[1] or s[2] for s in Template.pattern.findall(template_str)]
-
-## User
-
-def create_user(username: str, password: str):
-    """
-    Creates user using logged in mc client
-    In case of failure - script exits with code 1
-    """
-    code, out, err = execute_command(["mc", "admin", "user", "add", "minio", username, password])
-    if (code==0):
-        log.info(out)
-    else:
-        log.error(f"[{code}] Failed to create user '{username}': {err}")
-        sys.exit(1)
-
-def exists_user(username: str) -> bool:
-    """
-    Verifies if user exists using logged in mc client
-    True - exists, False - doesn't exist
-    """
-    code, _, _ = execute_command(["mc", "admin", "user", "info", "minio", username])
-    return code == 0
-
-def remove_user(username: str):
-    """
-    Removes user using logged in mc client
-    Assumes user with provided username exists
-    In case of failure - script exits with code 1
-    """
-    code, out, err = execute_command(["mc", "admin", "user", "remove", "minio", username])
-    if (code==0):
-        log.info(out)
-    else:
-        log.error(f"[{code}] Failed to remove user '{username}': {err}")
-        sys.exit(1)
-
-def update_user(username: str, password: str):
-    """
-    Updates user with provided username and password
-    In case user doesn't exist - user is created
-    In case user exists and RECREATE_EXISTING=True:
-        saves names of policies applied to old user, deletes old user, 
-        creates new user with provided name and password
-        and applies saved policies to him
-    If RECREATE_EXISTING=False - does nothing
-    """
-    if not exists_user(username):
-        create_user(username, password)
-    elif RECREATE_EXISTING:
-        log.info(f"User '{username}' already exists. Re-creating user with new password and re-applying current policies")
-        policy_list = get_user_policies(username)
-        # fixme Change user re-create logic, when found an API to modify existing user
-        remove_user(username)
-        create_user(username, password)
-        for policy in policy_list:
-            apply_policy(policy, username)
-    else:
-        log.info(f"User '{username}' already exists")
-
-## Bucket
-
-def create_bucket(bucket: str):
-    """
-    Creates bucket with provided name using logged in mc client
-    In case bucket already exists - does nothing
-    In case of failure - script exits with code 1
-    """
-    code, out, err = execute_command(["mc", "mb", f"minio/{bucket}", "--ignore-existing"])
-    if (code==0):
-        log.info(out)
-    else:
-        log.error(f"[{code}] Failed to create bucket '{bucket}': {err}")
-        sys.exit(1)
-
-## Policy
-
-def get_default_policy(bucket_list):
-    """
-    Returns dictionary containing default policy for accessing provided buckets
-    Uses default_policy.json file as template for default policy.
-    Dynamically adds bucket names to it from provided list
-    Assumes correct json structure in file and existence of file
-    Policy won't be correct for further usage in case empty bucket list is provided 
-    """
-    with open('default_policy.json') as f:
-        default_policy = json.load(f)
-    default_policy['Statement'][0]['Resource']=[f"arn:aws:s3:::{bucket_name}/*" for bucket_name in bucket_list]
-    return json.dumps(default_policy)
-
-def create_policy(policy_name: str, policy_file_content: str):
-    """
-    Creates new policy using logged in mc client
-    In case policy with the same name exists - it's content will be overriden
-    Users will be automatically transfered to new policy
-    Creates temporary file for policy in /tmp folder that is deleted in case of success
-    In case of failure - script exits with code 1
-    """
-    with tempfile.NamedTemporaryFile(mode='w+t', delete=False) as temp:
-        temp.write(policy_file_content)
-    code, out, err = execute_command(["mc", "admin", "policy", "add", "minio", policy_name, temp.name])
-    if (code==0):
-        log.info(out)
-        os.remove(temp.name)
-    else:
-        log.error(f"[{code}] Failed to create policy '{policy_name}': {err}")
-        sys.exit(1)
-
-def apply_policy(policy_name: str, username: str):
-    """
-    Applies provided policy to provided user using logged in mc client
-    Assumes existence of both policy and user
-    In case of failure - script exits with code 1
-    """
-    if policy_name not in get_user_policies(username):
-        code, out, err = execute_command(["mc", "admin", "policy", "update", "minio", policy_name, f"user={username}"])
-        if (code==0):
-            log.info(out)
-        else:
-            log.error(f"[{code}] Failed to apply policy '{policy_name} to user '{username}': {err}")
-            sys.exit(1)
-    else:
-        log.info(f"User '{username}' already has policy '{policy_name}', skipping policy update")
-
-def get_user_policies(username: str):
-    """
-    Returns list with names of policies applied to provided user using logged in mc client
-    Assumes existence of user
-    Can return empty list
-    """
-    user_info = json.loads(subprocess.check_output(["mc", "admin", "user", "info", "minio", username, "--json"]))
-    if 'policyName' in user_info:
-        return user_info['policyName'].split(',')
-    else:
-        return []
-
-## Main cycle
-
-def perform_minio_update(update_list):
-    """
-    Expects list of update objects
-    Assumes all substitutions were already performed
-    Assumes valid structure of dictionary
-    Updates user from 'user' and 'password'
-    Creates buckets from 'buckets'
-    If finds 'policy' - applies it, otherwise creates & applies default policy
-    N.B. Policies are created with the folllowing name: {PARTITION_NAME}_{username}_policy
-    """
-    for update in update_list:
-        if 'user' in update and 'password' in update:
-            # update user
-            username = update['user']
-            update_user(username, update['password'])
-            # update buckets
-            if 'buckets' in update:
-                buckets = update['buckets']
-                for bucket in buckets:
-                    create_bucket(bucket)
-            # update policy
-            if 'policy' in update:
-                policy_content = json.dumps(update['policy'])
-            else:
-                policy_content = get_default_policy(buckets)
-            policy_name= f"{PARTITION_NAME}_{username}_policy"
-            create_policy(policy_name, policy_content)
-            apply_policy(policy_name, username)
-        else:
-            log.error(f'No info about user in update: {update}') 
-            sys.exit(1)
-
-## Entrypoint
-
-# Iterates over *.tpl.json files in /opt/templates folder in alphabetical order
-# Substitutes values in update files and applies them
-if __name__ == "__main__":
-    login_to_mc_admin()
-    env_vars = dict(os.environ)
-    template_files = get_sorted_template_files('/opt/templates')
-    for template_file in template_files:
-        update_list = render_minio_update_json_template(template_file, env_vars)
-        perform_minio_update(update_list)
+import tempfile
+from string import Template
+import subprocess
+import os
+import sys
+import json
+import logging as log
+
+log.basicConfig(format='[%(levelname)s]: %(message)s', level=log.DEBUG)
+
+RECREATE_EXISTING = os.getenv('RECREATE_EXISTING', True)
+PARTITION_NAME = os.getenv('MINIO_PARTITION_NAME')
+
+
+## Helping functions
+
+def login_to_mc_admin():
+    """
+    Logins into mc client using subprocess command. 
+    Gets connection details from env vars: MINIO_HOST, MINIO_PORT, MINIO_ACCESS_KEY, MINIO_SECRET_KEY
+    Requires mc pre-installed and on PATH
+    """
+    minio_host = os.getenv('MINIO_HOST', 'http://127.0.0.1')
+    minio_port = os.getenv('MINIO_PORT', "9000")
+    minio_access_key = os.getenv('MINIO_ACCESS_KEY', 'minioadmin')
+    minio_secret_key = os.getenv('MINIO_SECRET_KEY', 'minioadmin')
+    code, out, err = execute_command(["mc", "alias", "set", "minio", f"{minio_host}:{minio_port}", minio_access_key, minio_secret_key, "--api", "S3v4"])
+    if (code==0):
+        log.info(f"Logged into mc client: {out}")
+    else:
+        log.error(f"[{code}] Failed to log into mc client: '{minio_host}:{minio_port}' with user '{minio_access_key}': {err}")
+
+def execute_command(arg_list):
+    """
+    Wrapper for command execution that uses subprocess
+    Returns tuple with (command_return_code, command_stdout_output, command_stderr_output)
+    """
+    command = subprocess.Popen(arg_list, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+    out, err = command.communicate()
+    return (command.returncode, out.rstrip(), err.rstrip())
+
+def get_sorted_template_files(template_dir_path):
+    """
+    Returns list of alphabetically sorted template files from
+    'template_dir_path' which end with .tpl.json
+    """
+    template_files = [os.path.join(template_dir_path, f) for f in os.listdir(template_dir_path) if os.path.isfile(os.path.join(template_dir_path, f)) and f.endswith('.tpl.json')]
+    template_files.sort()
+    return template_files
+## Template functions
+
+def render_minio_update_json_template(filename: str, substitute_dict) -> str:
+    """
+    Reads json 'filename' file. 
+    Finds template values in it (specified as ${..} or {...}), as in Python's Template class
+    Evaluates whether all values can be substituted using provided dictionary 'substitute_dict'
+    In case of failure - prints list of missing variables and exits script completely with code 1
+    Otherwise, substitutes template variables with values and returns rendered json converted into dictionary
+    """
+    with open(filename, 'r') as f:
+        content = f.read()
+        template = Template(content)
+        required_vars = get_required_template_variables(content)
+        for required_var in required_vars:
+            if required_var not in substitute_dict:
+                log.error(f"Cannot substitute value for '{required_var}' in file '{filename}'.")
+                sys.exit(1)
+        evaluated_minio_json = template.substitute(substitute_dict)
+        return json.loads(evaluated_minio_json)
+
+def get_required_template_variables(template_str: str):
+    """
+    Returns list of template values names found in provided string
+    Template values are as specified in Template class (ex. ${..} or {..})
+    """
+    return [s[1] or s[2] for s in Template.pattern.findall(template_str)]
+
+## User
+
+def create_user(username: str, password: str):
+    """
+    Creates user using logged in mc client
+    In case of failure - script exits with code 1
+    """
+    code, out, err = execute_command(["mc", "admin", "user", "add", "minio", username, password])
+    if (code==0):
+        log.info(out)
+    else:
+        log.error(f"[{code}] Failed to create user '{username}': {err}")
+        sys.exit(1)
+
+def exists_user(username: str) -> bool:
+    """
+    Verifies if user exists using logged in mc client
+    True - exists, False - doesn't exist
+    """
+    code, _, _ = execute_command(["mc", "admin", "user", "info", "minio", username])
+    return code == 0
+
+def remove_user(username: str):
+    """
+    Removes user using logged in mc client
+    Assumes user with provided username exists
+    In case of failure - script exits with code 1
+    """
+    code, out, err = execute_command(["mc", "admin", "user", "remove", "minio", username])
+    if (code==0):
+        log.info(out)
+    else:
+        log.error(f"[{code}] Failed to remove user '{username}': {err}")
+        sys.exit(1)
+
+def update_user(username: str, password: str):
+    """
+    Updates user with provided username and password
+    In case user doesn't exist - user is created
+    In case user exists and RECREATE_EXISTING=True:
+        saves names of policies applied to old user, deletes old user, 
+        creates new user with provided name and password
+        and applies saved policies to him
+    If RECREATE_EXISTING=False - does nothing
+    """
+    if not exists_user(username):
+        create_user(username, password)
+    elif RECREATE_EXISTING:
+        log.info(f"User '{username}' already exists. Re-creating user with new password and re-applying current policies")
+        policy_list = get_user_policies(username)
+        # fixme Change user re-create logic, when found an API to modify existing user
+        remove_user(username)
+        create_user(username, password)
+        for policy in policy_list:
+            apply_policy(policy, username)
+    else:
+        log.info(f"User '{username}' already exists")
+
+## Bucket
+
+def create_bucket(bucket: str):
+    """
+    Creates bucket with provided name using logged in mc client
+    In case bucket already exists - does nothing
+    In case of failure - script exits with code 1
+    """
+    code, out, err = execute_command(["mc", "mb", f"minio/{bucket}", "--ignore-existing"])
+    if (code==0):
+        log.info(out)
+    else:
+        log.error(f"[{code}] Failed to create bucket '{bucket}': {err}")
+        sys.exit(1)
+
+## Policy
+
+def get_default_policy(bucket_list):
+    """
+    Returns dictionary containing default policy for accessing provided buckets
+    Uses default_policy.json file as template for default policy.
+    Dynamically adds bucket names to it from provided list
+    Assumes correct json structure in file and existence of file
+    Policy won't be correct for further usage in case empty bucket list is provided 
+    """
+    with open('default_policy.json') as f:
+        default_policy = json.load(f)
+    default_policy['Statement'][0]['Resource']=[f"arn:aws:s3:::{bucket_name}/*" for bucket_name in bucket_list]
+    return json.dumps(default_policy)
+
+def create_policy(policy_name: str, policy_file_content: str):
+    """
+    Creates new policy using logged in mc client
+    In case policy with the same name exists - it's content will be overriden
+    Users will be automatically transfered to new policy
+    Creates temporary file for policy in /tmp folder that is deleted in case of success
+    In case of failure - script exits with code 1
+    """
+    with tempfile.NamedTemporaryFile(mode='w+t', delete=False) as temp:
+        temp.write(policy_file_content)
+    code, out, err = execute_command(["mc", "admin", "policy", "add", "minio", policy_name, temp.name])
+    if (code==0):
+        log.info(out)
+        os.remove(temp.name)
+    else:
+        log.error(f"[{code}] Failed to create policy '{policy_name}': {err}")
+        sys.exit(1)
+
+def apply_policy(policy_name: str, username: str):
+    """
+    Applies provided policy to provided user using logged in mc client
+    Assumes existence of both policy and user
+    In case of failure - script exits with code 1
+    """
+    if policy_name not in get_user_policies(username):
+        code, out, err = execute_command(["mc", "admin", "policy", "update", "minio", policy_name, f"user={username}"])
+        if (code==0):
+            log.info(out)
+        else:
+            log.error(f"[{code}] Failed to apply policy '{policy_name} to user '{username}': {err}")
+            sys.exit(1)
+    else:
+        log.info(f"User '{username}' already has policy '{policy_name}', skipping policy update")
+
+def get_user_policies(username: str):
+    """
+    Returns list with names of policies applied to provided user using logged in mc client
+    Assumes existence of user
+    Can return empty list
+    """
+    user_info = json.loads(subprocess.check_output(["mc", "admin", "user", "info", "minio", username, "--json"]))
+    if 'policyName' in user_info:
+        return user_info['policyName'].split(',')
+    else:
+        return []
+
+## Main cycle
+
+def perform_minio_update(update_list):
+    """
+    Expects list of update objects
+    Assumes all substitutions were already performed
+    Assumes valid structure of dictionary
+    Updates user from 'user' and 'password'
+    Creates buckets from 'buckets'
+    If finds 'policy' - applies it, otherwise creates & applies default policy
+    N.B. Policies are created with the folllowing name: {PARTITION_NAME}_{username}_policy
+    """
+    for update in update_list:
+        if 'user' in update and 'password' in update:
+            # update user
+            username = update['user']
+            update_user(username, update['password'])
+            # update buckets
+            if 'buckets' in update:
+                buckets = update['buckets']
+                for bucket in buckets:
+                    create_bucket(bucket)
+            # update policy
+            if 'policy' in update:
+                policy_content = json.dumps(update['policy'])
+            else:
+                policy_content = get_default_policy(buckets)
+            policy_name= f"{PARTITION_NAME}_{username}_policy"
+            create_policy(policy_name, policy_content)
+            apply_policy(policy_name, username)
+        else:
+            log.error(f'No info about user in update: {update}') 
+            sys.exit(1)
+
+## Entrypoint
+
+# Iterates over *.tpl.json files in /opt/templates folder in alphabetical order
+# Substitutes values in update files and applies them
+if __name__ == "__main__":
+    login_to_mc_admin()
+    env_vars = dict(os.environ)
+    template_files = get_sorted_template_files('/opt/templates')
+    for template_file in template_files:
+        update_list = render_minio_update_json_template(template_file, env_vars)
+        perform_minio_update(update_list)