Skip to content
Snippets Groups Projects

GONRG-5320: domain is not set for istio policy

Merged GONRG-5320: domain is not set for istio policy
GONRG-5320-domain-is-not-set-for-istio-policy into master
All threads resolved!
Merged Krisztian Molnar (EPAM) requested to merge GONRG-5320-domain-is-not-set-for-istio-policy into master
All threads resolved!
Compare
and
5 files
+ 327
− 323
Compare changes
  • Side-by-side
  • Inline
Files
5
bootstrap_infra_on_prem/minio/minio_bootstrap.py
+ 256
− 256
import tempfile
from string import Template
import subprocess
import os
import sys
import json
import logging as log
log.basicConfig(format='[%(levelname)s]: %(message)s', level=log.DEBUG)
RECREATE_EXISTING = os.getenv('RECREATE_EXISTING', True)
PARTITION_NAME = os.getenv('MINIO_PARTITION_NAME')
## Helping functions
def login_to_mc_admin():
"""
Logins into mc client using subprocess command.
Gets connection details from env vars: MINIO_HOST, MINIO_PORT, MINIO_ACCESS_KEY, MINIO_SECRET_KEY
Requires mc pre-installed and on PATH
"""
minio_host = os.getenv('MINIO_HOST', 'http://127.0.0.1')
minio_port = os.getenv('MINIO_PORT', "9000")
minio_access_key = os.getenv('MINIO_ACCESS_KEY', 'minioadmin')
minio_secret_key = os.getenv('MINIO_SECRET_KEY', 'minioadmin')
code, out, err = execute_command(["mc", "alias", "set", "minio", f"{minio_host}:{minio_port}", minio_access_key, minio_secret_key, "--api", "S3v4"])
if (code==0):
log.info(f"Logged into mc client: {out}")
else:
log.error(f"[{code}] Failed to log into mc client: '{minio_host}:{minio_port}' with user '{minio_access_key}': {err}")
def execute_command(arg_list):
"""
Wrapper for command execution that uses subprocess
Returns tuple with (command_return_code, command_stdout_output, command_stderr_output)
"""
command = subprocess.Popen(arg_list, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
out, err = command.communicate()
return (command.returncode, out.rstrip(), err.rstrip())
def get_sorted_template_files(template_dir_path):
"""
Returns list of alphabetically sorted template files from
'template_dir_path' which end with .tpl.json
"""
template_files = [os.path.join(template_dir_path, f) for f in os.listdir(template_dir_path) if os.path.isfile(os.path.join(template_dir_path, f)) and f.endswith('.tpl.json')]
template_files.sort()
return template_files
## Template functions
def render_minio_update_json_template(filename: str, substitute_dict) -> str:
"""
Reads json 'filename' file.
Finds template values in it (specified as ${..} or {...}), as in Python's Template class
Evaluates whether all values can be substituted using provided dictionary 'substitute_dict'
In case of failure - prints list of missing variables and exits script completely with code 1
Otherwise, substitutes template variables with values and returns rendered json converted into dictionary
"""
with open(filename, 'r') as f:
content = f.read()
template = Template(content)
required_vars = get_required_template_variables(content)
for required_var in required_vars:
if required_var not in substitute_dict:
log.error(f"Cannot substitute value for '{required_var}' in file '{filename}'.")
sys.exit(1)
evaluated_minio_json = template.substitute(substitute_dict)
return json.loads(evaluated_minio_json)
def get_required_template_variables(template_str: str):
"""
Returns list of template values names found in provided string
Template values are as specified in Template class (ex. ${..} or {..})
"""
return [s[1] or s[2] for s in Template.pattern.findall(template_str)]
## User
def create_user(username: str, password: str):
"""
Creates user using logged in mc client
In case of failure - script exits with code 1
"""
code, out, err = execute_command(["mc", "admin", "user", "add", "minio", username, password])
if (code==0):
log.info(out)
else:
log.error(f"[{code}] Failed to create user '{username}': {err}")
sys.exit(1)
def exists_user(username: str) -> bool:
"""
Verifies if user exists using logged in mc client
True - exists, False - doesn't exist
"""
code, _, _ = execute_command(["mc", "admin", "user", "info", "minio", username])
return code == 0
def remove_user(username: str):
"""
Removes user using logged in mc client
Assumes user with provided username exists
In case of failure - script exits with code 1
"""
code, out, err = execute_command(["mc", "admin", "user", "remove", "minio", username])
if (code==0):
log.info(out)
else:
log.error(f"[{code}] Failed to remove user '{username}': {err}")
sys.exit(1)
def update_user(username: str, password: str):
"""
Updates user with provided username and password
In case user doesn't exist - user is created
In case user exists and RECREATE_EXISTING=True:
saves names of policies applied to old user, deletes old user,
creates new user with provided name and password
and applies saved policies to him
If RECREATE_EXISTING=False - does nothing
"""
if not exists_user(username):
create_user(username, password)
elif RECREATE_EXISTING:
log.info(f"User '{username}' already exists. Re-creating user with new password and re-applying current policies")
policy_list = get_user_policies(username)
# fixme Change user re-create logic, when found an API to modify existing user
remove_user(username)
create_user(username, password)
for policy in policy_list:
apply_policy(policy, username)
else:
log.info(f"User '{username}' already exists")
## Bucket
def create_bucket(bucket: str):
"""
Creates bucket with provided name using logged in mc client
In case bucket already exists - does nothing
In case of failure - script exits with code 1
"""
code, out, err = execute_command(["mc", "mb", f"minio/{bucket}", "--ignore-existing"])
if (code==0):
log.info(out)
else:
log.error(f"[{code}] Failed to create bucket '{bucket}': {err}")
sys.exit(1)
## Policy
def get_default_policy(bucket_list):
"""
Returns dictionary containing default policy for accessing provided buckets
Uses default_policy.json file as template for default policy.
Dynamically adds bucket names to it from provided list
Assumes correct json structure in file and existence of file
Policy won't be correct for further usage in case empty bucket list is provided
"""
with open('default_policy.json') as f:
default_policy = json.load(f)
default_policy['Statement'][0]['Resource']=[f"arn:aws:s3:::{bucket_name}/*" for bucket_name in bucket_list]
return json.dumps(default_policy)
def create_policy(policy_name: str, policy_file_content: str):
"""
Creates new policy using logged in mc client
In case policy with the same name exists - it's content will be overriden
Users will be automatically transfered to new policy
Creates temporary file for policy in /tmp folder that is deleted in case of success
In case of failure - script exits with code 1
"""
with tempfile.NamedTemporaryFile(mode='w+t', delete=False) as temp:
temp.write(policy_file_content)
code, out, err = execute_command(["mc", "admin", "policy", "add", "minio", policy_name, temp.name])
if (code==0):
log.info(out)
os.remove(temp.name)
else:
log.error(f"[{code}] Failed to create policy '{policy_name}': {err}")
sys.exit(1)
def apply_policy(policy_name: str, username: str):
"""
Applies provided policy to provided user using logged in mc client
Assumes existence of both policy and user
In case of failure - script exits with code 1
"""
if policy_name not in get_user_policies(username):
code, out, err = execute_command(["mc", "admin", "policy", "update", "minio", policy_name, f"user={username}"])
if (code==0):
log.info(out)
else:
log.error(f"[{code}] Failed to apply policy '{policy_name} to user '{username}': {err}")
sys.exit(1)
else:
log.info(f"User '{username}' already has policy '{policy_name}', skipping policy update")
def get_user_policies(username: str):
"""
Returns list with names of policies applied to provided user using logged in mc client
Assumes existence of user
Can return empty list
"""
user_info = json.loads(subprocess.check_output(["mc", "admin", "user", "info", "minio", username, "--json"]))
if 'policyName' in user_info:
return user_info['policyName'].split(',')
else:
return []
## Main cycle
def perform_minio_update(update_list):
"""
Expects list of update objects
Assumes all substitutions were already performed
Assumes valid structure of dictionary
Updates user from 'user' and 'password'
Creates buckets from 'buckets'
If finds 'policy' - applies it, otherwise creates & applies default policy
N.B. Policies are created with the folllowing name: {PARTITION_NAME}_{username}_policy
"""
for update in update_list:
if 'user' in update and 'password' in update:
# update user
username = update['user']
update_user(username, update['password'])
# update buckets
if 'buckets' in update:
buckets = update['buckets']
for bucket in buckets:
create_bucket(bucket)
# update policy
if 'policy' in update:
policy_content = json.dumps(update['policy'])
else:
policy_content = get_default_policy(buckets)
policy_name= f"{PARTITION_NAME}_{username}_policy"
create_policy(policy_name, policy_content)
apply_policy(policy_name, username)
else:
log.error(f'No info about user in update: {update}')
sys.exit(1)
## Entrypoint
# Iterates over *.tpl.json files in /opt/templates folder in alphabetical order
# Substitutes values in update files and applies them
if __name__ == "__main__":
login_to_mc_admin()
env_vars = dict(os.environ)
template_files = get_sorted_template_files('/opt/templates')
for template_file in template_files:
update_list = render_minio_update_json_template(template_file, env_vars)
perform_minio_update(update_list)
import tempfile
from string import Template
import subprocess
import os
import sys
import json
import logging as log
log.basicConfig(format='[%(levelname)s]: %(message)s', level=log.DEBUG)
RECREATE_EXISTING = os.getenv('RECREATE_EXISTING', True)
PARTITION_NAME = os.getenv('MINIO_PARTITION_NAME')
## Helping functions
def login_to_mc_admin():
"""
Logins into mc client using subprocess command.
Gets connection details from env vars: MINIO_HOST, MINIO_PORT, MINIO_ACCESS_KEY, MINIO_SECRET_KEY
Requires mc pre-installed and on PATH
"""
minio_host = os.getenv('MINIO_HOST', 'http://127.0.0.1')
minio_port = os.getenv('MINIO_PORT', "9000")
minio_access_key = os.getenv('MINIO_ACCESS_KEY', 'minioadmin')
minio_secret_key = os.getenv('MINIO_SECRET_KEY', 'minioadmin')
code, out, err = execute_command(["mc", "alias", "set", "minio", f"{minio_host}:{minio_port}", minio_access_key, minio_secret_key, "--api", "S3v4"])
if (code==0):
log.info(f"Logged into mc client: {out}")
else:
log.error(f"[{code}] Failed to log into mc client: '{minio_host}:{minio_port}' with user '{minio_access_key}': {err}")
def execute_command(arg_list):
"""
Wrapper for command execution that uses subprocess
Returns tuple with (command_return_code, command_stdout_output, command_stderr_output)
"""
command = subprocess.Popen(arg_list, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
out, err = command.communicate()
return (command.returncode, out.rstrip(), err.rstrip())
def get_sorted_template_files(template_dir_path):
"""
Returns list of alphabetically sorted template files from
'template_dir_path' which end with .tpl.json
"""
template_files = [os.path.join(template_dir_path, f) for f in os.listdir(template_dir_path) if os.path.isfile(os.path.join(template_dir_path, f)) and f.endswith('.tpl.json')]
template_files.sort()
return template_files
## Template functions
def render_minio_update_json_template(filename: str, substitute_dict) -> str:
"""
Reads json 'filename' file.
Finds template values in it (specified as ${..} or {...}), as in Python's Template class
Evaluates whether all values can be substituted using provided dictionary 'substitute_dict'
In case of failure - prints list of missing variables and exits script completely with code 1
Otherwise, substitutes template variables with values and returns rendered json converted into dictionary
"""
with open(filename, 'r') as f:
content = f.read()
template = Template(content)
required_vars = get_required_template_variables(content)
for required_var in required_vars:
if required_var not in substitute_dict:
log.error(f"Cannot substitute value for '{required_var}' in file '{filename}'.")
sys.exit(1)
evaluated_minio_json = template.substitute(substitute_dict)
return json.loads(evaluated_minio_json)
def get_required_template_variables(template_str: str):
"""
Returns list of template values names found in provided string
Template values are as specified in Template class (ex. ${..} or {..})
"""
return [s[1] or s[2] for s in Template.pattern.findall(template_str)]
## User
def create_user(username: str, password: str):
"""
Creates user using logged in mc client
In case of failure - script exits with code 1
"""
code, out, err = execute_command(["mc", "admin", "user", "add", "minio", username, password])
if (code==0):
log.info(out)
else:
log.error(f"[{code}] Failed to create user '{username}': {err}")
sys.exit(1)
def exists_user(username: str) -> bool:
"""
Verifies if user exists using logged in mc client
True - exists, False - doesn't exist
"""
code, _, _ = execute_command(["mc", "admin", "user", "info", "minio", username])
return code == 0
def remove_user(username: str):
"""
Removes user using logged in mc client
Assumes user with provided username exists
In case of failure - script exits with code 1
"""
code, out, err = execute_command(["mc", "admin", "user", "remove", "minio", username])
if (code==0):
log.info(out)
else:
log.error(f"[{code}] Failed to remove user '{username}': {err}")
sys.exit(1)
def update_user(username: str, password: str):
"""
Updates user with provided username and password
In case user doesn't exist - user is created
In case user exists and RECREATE_EXISTING=True:
saves names of policies applied to old user, deletes old user,
creates new user with provided name and password
and applies saved policies to him
If RECREATE_EXISTING=False - does nothing
"""
if not exists_user(username):
create_user(username, password)
elif RECREATE_EXISTING:
log.info(f"User '{username}' already exists. Re-creating user with new password and re-applying current policies")
policy_list = get_user_policies(username)
# fixme Change user re-create logic, when found an API to modify existing user
remove_user(username)
create_user(username, password)
for policy in policy_list:
apply_policy(policy, username)
else:
log.info(f"User '{username}' already exists")
## Bucket
def create_bucket(bucket: str):
"""
Creates bucket with provided name using logged in mc client
In case bucket already exists - does nothing
In case of failure - script exits with code 1
"""
code, out, err = execute_command(["mc", "mb", f"minio/{bucket}", "--ignore-existing"])
if (code==0):
log.info(out)
else:
log.error(f"[{code}] Failed to create bucket '{bucket}': {err}")
sys.exit(1)
## Policy
def get_default_policy(bucket_list):
"""
Returns dictionary containing default policy for accessing provided buckets
Uses default_policy.json file as template for default policy.
Dynamically adds bucket names to it from provided list
Assumes correct json structure in file and existence of file
Policy won't be correct for further usage in case empty bucket list is provided
"""
with open('default_policy.json') as f:
default_policy = json.load(f)
default_policy['Statement'][0]['Resource']=[f"arn:aws:s3:::{bucket_name}/*" for bucket_name in bucket_list]
return json.dumps(default_policy)
def create_policy(policy_name: str, policy_file_content: str):
"""
Creates new policy using logged in mc client
In case policy with the same name exists - it's content will be overriden
Users will be automatically transfered to new policy
Creates temporary file for policy in /tmp folder that is deleted in case of success
In case of failure - script exits with code 1
"""
with tempfile.NamedTemporaryFile(mode='w+t', delete=False) as temp:
temp.write(policy_file_content)
code, out, err = execute_command(["mc", "admin", "policy", "add", "minio", policy_name, temp.name])
if (code==0):
log.info(out)
os.remove(temp.name)
else:
log.error(f"[{code}] Failed to create policy '{policy_name}': {err}")
sys.exit(1)
def apply_policy(policy_name: str, username: str):
"""
Applies provided policy to provided user using logged in mc client
Assumes existence of both policy and user
In case of failure - script exits with code 1
"""
if policy_name not in get_user_policies(username):
code, out, err = execute_command(["mc", "admin", "policy", "update", "minio", policy_name, f"user={username}"])
if (code==0):
log.info(out)
else:
log.error(f"[{code}] Failed to apply policy '{policy_name} to user '{username}': {err}")
sys.exit(1)
else:
log.info(f"User '{username}' already has policy '{policy_name}', skipping policy update")
def get_user_policies(username: str):
"""
Returns list with names of policies applied to provided user using logged in mc client
Assumes existence of user
Can return empty list
"""
user_info = json.loads(subprocess.check_output(["mc", "admin", "user", "info", "minio", username, "--json"]))
if 'policyName' in user_info:
return user_info['policyName'].split(',')
else:
return []
## Main cycle
def perform_minio_update(update_list):
"""
Expects list of update objects
Assumes all substitutions were already performed
Assumes valid structure of dictionary
Updates user from 'user' and 'password'
Creates buckets from 'buckets'
If finds 'policy' - applies it, otherwise creates & applies default policy
N.B. Policies are created with the folllowing name: {PARTITION_NAME}_{username}_policy
"""
for update in update_list:
if 'user' in update and 'password' in update:
# update user
username = update['user']
update_user(username, update['password'])
# update buckets
if 'buckets' in update:
buckets = update['buckets']
for bucket in buckets:
create_bucket(bucket)
# update policy
if 'policy' in update:
policy_content = json.dumps(update['policy'])
else:
policy_content = get_default_policy(buckets)
policy_name= f"{PARTITION_NAME}_{username}_policy"
create_policy(policy_name, policy_content)
apply_policy(policy_name, username)
else:
log.error(f'No info about user in update: {update}')
sys.exit(1)
## Entrypoint
# Iterates over *.tpl.json files in /opt/templates folder in alphabetical order
# Substitutes values in update files and applies them
if __name__ == "__main__":
login_to_mc_admin()
env_vars = dict(os.environ)
template_files = get_sorted_template_files('/opt/templates')
for template_file in template_files:
update_list = render_minio_update_json_template(template_file, env_vars)
perform_minio_update(update_list)