Skip to content

GitLab

There is a security vulnerability in SSH key-generation using GitKraken < v8.0.1. If you used this tool to create SSH keys, please update GitKraken and regenerate. If you need help with this, contact forum-support@opengroup.org

Commit 32bdc7e4 authored by harshit aggarwal's avatar harshit aggarwal
Browse files

Merge branch 'master' into airflow-add-variable

parents eeba6d75 4f196239
Pipeline #67692 passed with stages
in 1 minute and 41 seconds
Hide whitespace changes
Inline Side-by-side
charts/airflow/helm-config-dp.yaml
View file @ 32bdc7e4
......@@ -331,8 +331,9 @@ airflow:
"pyyaml==5.4.1",
"requests==2.25.1",
"tenacity==8.0.1",
"https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_api-0.11.0.tar.gz",
"https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_airflow-0.0.1.dev31+59e58330.tar.gz"
"https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_api-0.10.1.dev151+503e364a.tar.gz",
"https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_airflow-0.0.1.dev32+ea39f8bd.tar.gz"
]
extraVolumeMounts:
- name: azure-keyvault
......
charts/airflow/helm-config.yaml
View file @ 32bdc7e4
......@@ -346,8 +346,8 @@ airflow:
"pyyaml==5.4.1",
"requests==2.25.1",
"tenacity==8.0.1",
"https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_api-0.11.0.tar.gz",
"https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_airflow-0.0.1.dev31+59e58330.tar.gz"
"https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_api-0.10.1.dev151+503e364a.tar.gz",
"https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_airflow-0.0.1.dev32+ea39f8bd.tar.gz"
]
extraVolumeMounts:
- name: azure-keyvault
......
charts/osdu-common/templates/appgw-ingress.yaml
View file @ 32bdc7e4
......@@ -121,3 +121,7 @@ spec:
serviceName: dataset
servicePort: 80
path: /api/dataset/v1/*
- backend:
serviceName: seismic-dms-file-metadata-service
servicePort: 80
path: /seismic-file-metadata/api/v1/*
charts/osdu-istio/templates/cert-checker-cronjob.yaml
View file @ 32bdc7e4
......@@ -51,6 +51,11 @@ spec:
configMapKeyRef:
name: {{ .Values.global.job.configmap_name }}
key: ENV_CLUSTER_NAME
- name: ENV_APPGW_NAME
valueFrom:
configMapKeyRef:
name: {{ .Values.global.job.configmap_name }}
key: ENV_APPGW_NAME
command:
- /bin/sh
args:
......@@ -62,6 +67,7 @@ spec:
# Compare expire dates of certificates in Key Vault and in istio-system namespaces
function check_expire_date() {
echo "Compare expire dates of certificates in Key Vault and in istio-system namespaces"
az keyvault certificate download --vault-name ${ENV_KEYVAULT_NAME} -n ${KV_CERT_NAME} --file ${KV_CERT_NAME}.pem
KV_CERT_EXPIREDATE=$(openssl x509 -in ${KV_CERT_NAME}.pem -enddate -noout | cut -d '=' -f2)
KV_CERT_EXPIREDATE=$(date "+%Y-%m-%d" --date="${KV_CERT_EXPIREDATE}")
......@@ -84,7 +90,7 @@ spec:
# Cleanup function
cleanup() {
echo Clean all existing files
echo "Clean all existing files"
rm -f cert.crt cert.key osdu-certificate.pfx ${KV_CERT_NAME}.pem
curl -X POST "http://localhost:${SIDECAR_PORT}/quitquitquit"
}
......@@ -111,7 +117,7 @@ spec:
# Log In in Azure
az login --service-principal -u ${client_id} -p ${client_secret} --tenant ${tenant_id}
{{- if .Values.global.istio.enableIstioKeyvaultCert }}
{{- if .Values.global.istio.enableIstioKeyvaultCert }}
K8S_CERT_SECRET=istio-appgw-ssl-cert
K8S_NAMESPACE_NAME=istio-system
......@@ -120,8 +126,13 @@ spec:
check_expire_date
# Download BYOC certificate from keyvault
echo "Download BYOC certificate from keyvault"
az keyvault secret download --file ${KV_CERT_NAME}.pfx --vault-name ${ENV_KEYVAULT_NAME} --encoding base64 --name ${KV_CERT_NAME}
# Upload BYOC certificate to appgw
echo "Upload BYOC certificate to appgw"
az network application-gateway ssl-cert update -g ${ENV_SR_GROUP_NAME} --gateway-name ${ENV_APPGW_NAME} -n ${KV_CERT_NAME} --cert-file ${K8S_CERT_SECRET}.pfx --cert-password ""
# Extract key and crt
openssl pkcs12 -in ${K8S_CERT_SECRET}.pfx -out cert.pem -passin pass:"" -nodes -passout pass:""
openssl rsa -in cert.pem -out cert.key
......@@ -135,6 +146,9 @@ spec:
sleep 5
# Restart istio-ingressgateway pods
kubectl rollout restart -n ${K8S_ISTIONAMESPACE_NAME} deployment/istio-ingressgateway
# Check certificate expire date one more time
check_expire_date
......
charts/osdu-istio/templates/cert-init-job.yaml
View file @ 32bdc7e4
......@@ -43,6 +43,11 @@ spec:
configMapKeyRef:
name: {{ .Values.global.job.configmap_name }}
key: ENV_CLUSTER_NAME
- name: ENV_APPGW_NAME
valueFrom:
configMapKeyRef:
name: {{ .Values.global.job.configmap_name }}
key: ENV_APPGW_NAME
command:
- /bin/sh
args:
......@@ -82,14 +87,19 @@ spec:
az login --service-principal -u ${client_id} -p ${client_secret} --tenant ${tenant_id}
az aks get-credentials --resource-group ${ENV_SR_GROUP_NAME} --name ${ENV_CLUSTER_NAME}
{{- if .Values.global.istio.enableIstioKeyvaultCert }}
{{- if .Values.global.istio.enableIstioKeyvaultCert }}
K8S_CERT_SECRET=istio-appgw-ssl-cert
K8S_NAMESPACE_NAME=istio-system
# Download BYOC certificate from keyvault
echo "Download BYOC certificate from keyvault"
az keyvault secret download --file ${KV_CERT_NAME}.pfx --vault-name ${ENV_KEYVAULT_NAME} --encoding base64 --name ${KV_CERT_NAME}
# Upload BYOC certificate to appgw
echo "Upload BYOC certificate to appgw"
az network application-gateway ssl-cert create -g ${ENV_SR_GROUP_NAME} --gateway-name ${ENV_APPGW_NAME} -n ${KV_CERT_NAME} --cert-file ${K8S_CERT_SECRET}.pfx --cert-password ""
# Extract key and crt
openssl pkcs12 -in ${K8S_CERT_SECRET}.pfx -out cert.pem -passin pass:"" -nodes -passout pass:""
openssl rsa -in cert.pem -out cert.key
......@@ -101,6 +111,9 @@ spec:
--from-file=tls.crt=cert.crt --from-file=tls.key=cert.key \
-o yaml | kubectl apply -f -
# Restart istio-ingressgateway pods
kubectl rollout restart -n ${K8S_ISTIONAMESPACE_NAME} deployment/istio-ingressgateway
{{ else }}
K8S_CERT_SECRET=osdu-certificate
......
infra/templates/osdu-r3-mvp/service_resources/config_map.tf
View file @ 32bdc7e4
......@@ -66,6 +66,7 @@ resource "kubernetes_config_map" "appgw_configmap" {
ENV_SR_GROUP_NAME = azurerm_resource_group.main.name
ENV_KEYVAULT_NAME = data.terraform_remote_state.central_resources.outputs.keyvault_name
ENV_CLUSTER_NAME = module.aks.name
ENV_APPGW_NAME = module.istio_appgateway[count.index].name
}
depends_on = [kubernetes_namespace.osdu]
depends_on = [kubernetes_namespace.osdu, module.istio_appgateway]
}
infra/templates/osdu-r3-mvp/service_resources/main.tf
View file @ 32bdc7e4
......@@ -441,6 +441,39 @@ data "azurerm_resource_group" "aks_node_resource_group" {
name = module.aks.node_resource_group
}
// Give AD Principal Access rights to Change the Istio Application Gateway
resource "azurerm_role_assignment" "agic_istio_appgw_contributor" {
count = var.feature_flag.autoscaling ? 1 : 0
principal_id = data.terraform_remote_state.central_resources.outputs.osdu_service_principal_id
scope = module.istio_appgateway[count.index].id
role_definition_name = "Contributor"
depends_on = [module.istio_appgateway]
}
// Give AD Principal Access rights to Operate the Istio Application Gateway Identity
resource "azurerm_role_assignment" "agic_istio_app_gw_contributor_for_adsp" {
count = var.feature_flag.autoscaling ? 1 : 0
principal_id = data.terraform_remote_state.central_resources.outputs.osdu_service_principal_id
scope = module.istio_appgateway[count.index].managed_identity_resource_id
role_definition_name = "Managed Identity Operator"
depends_on = [module.istio_appgateway]
}
// Give AD Principal the rights to look at the Resource Group
resource "azurerm_role_assignment" "agic_istio_resourcegroup_reader" {
count = var.feature_flag.autoscaling ? 1 : 0
principal_id = data.terraform_remote_state.central_resources.outputs.osdu_service_principal_id
scope = azurerm_resource_group.main.id
role_definition_name = "Reader"
depends_on = [module.istio_appgateway]
}
// Give AKS Access rights to Operate the Node Resource Group
resource "azurerm_role_assignment" "all_mi_operator" {
principal_id = module.aks.kubelet_object_id
......
infra/templates/osdu-r3-mvp/service_resources/tests/unit/unit_test.go
View file @ 32bdc7e4
......@@ -36,8 +36,8 @@ var tfOptions = &terraform.Options{
}
var istioEnabled = os.Getenv("AUTOSCALING_ENABLED")
var istioResourses = 11
var totalResources = 138
var istioResourses = 14
var totalResources = 141
func TestTemplate(t *testing.T) {
expectedAppDevResourceGroup := asMap(t, `{
......
tools/helper_services_data_seeding_agent/Dockerfile 0 → 100644
View file @ 32bdc7e4
FROM mcr.microsoft.com/azure-cli
RUN apk update
RUN apk add ca-certificates wget && update-ca-certificates
RUN apk add libc6-compat
RUN apk add git
RUN apk add ca-certificates bash curl wget gettext jq bind-tools \
&& wget -q https://storage.googleapis.com/kubernetes-release/release/v1.21.2/bin/linux/amd64/kubectl -O /usr/local/bin/kubectl \
&& chmod +x /usr/local/bin/kubectl \
&& chmod g+rwx /root \
&& mkdir /config \
&& chmod g+rwx /config \
&& apk add --update coreutils && rm -rf /var/cache/apk/*
WORKDIR /usr/src/app
COPY transfer.sh ./
RUN chmod +x transfer.sh
ENTRYPOINT [ "./transfer.sh" ]
\ No newline at end of file
tools/helper_services_data_seeding_agent/docker-compose.yaml 0 → 100644
View file @ 32bdc7e4
version: '3'
services:
helper-services-data-seeding-agent:
build:
context: ../helper_services_data_seeding_agent
dockerfile: ./Dockerfile
image: msosdu.azurecr.io/helper-services-data-seeding-agent:$VERSION
environment:
OSDU_IDENTITY_ID: $OSDU_IDENTITY_ID
RESOURCE_GROUP_NAME: $RESOURCE_GROUP_NAME
CONFIG_MAP_NAME: $CONFIG_MAP_NAME
VERSION: $VERSION
\ No newline at end of file
tools/helper_services_data_seeding_agent/transfer.sh 0 → 100644
View file @ 32bdc7e4
#!/bin/bash
mkdir -p tmp
cd tmp
wget -O azcopy_v10.tar.gz https://aka.ms/downloadazcopy-v10-linux && tar -xf azcopy_v10.tar.gz --strip-components=1
cp ./azcopy /usr/bin/
cd ..
currentStatus=""
currentMessage=""
git clone https://community.opengroup.org/osdu/platform/system/reference/crs-conversion-service.git
if [[ $? -gt 0 ]]; then
currentMessage="failure"
currentMessage="${currentMessage}. Failed to clone crs-conversion-service"
fi
git clone https://community.opengroup.org/osdu/platform/system/reference/crs-catalog-service.git
if [[ $? -gt 0 ]]; then
currentMessage="failure"
currentMessage="${currentMessage}. Failed to clone crs-catalog-service"
fi
git clone https://community.opengroup.org/osdu/platform/system/reference/unit-service.git
if [[ $? -gt 0 ]]; then
currentMessage="failure"
currentMessage="${currentMessage}. Failed to clone unit-service"
fi
CRS_CONVERSION_SOURCE_FOLDER="crs-conversion-service/apachesis_setup"
CRS_CATALOG_SOURCE_FOLDER="crs-catalog-service/data/crs_catalog_v2.json"
UNIT_SOURCE_FOLDER="unit-service/data/unit_catalog_v2.json"
az login --identity --username $OSDU_IDENTITY_ID
ENV_VAULT=$(az keyvault list --resource-group $RESOURCE_GROUP_NAME --query [].name -otsv)
STORAGE_ACCOUNT_NAME=$(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/airflow-storage --query value -otsv)
if [ -z "$STORAGE_ACCOUNT_NAME" -a "$STORAGE_ACCOUNT_NAME" == " " ]; then
currentStatus="failure"
currentMessage="${currentMessage}. Storage Account Name Not Found. "
fi
STORAGE_ACCOUNT_KEY=$(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/airflow-storage-key --query value -otsv)
if [ -z "$STORAGE_ACCOUNT_KEY" -a "$STORAGE_ACCOUNT_KEY" == " " ]; then
currentStatus="failure"
currentMessage="${currentMessage}. Storage Account Key Not Found. "
else
EXPIRE=$(date -u -d "59 minutes" '+%Y-%m-%dT%H:%M:%SZ')
START=$(date -u -d "-1 minute" '+%Y-%m-%dT%H:%M:%SZ')
#Generating the SAS Token required for Authorization
AZURE_STORAGE_SAS_TOKEN=$(az storage account generate-sas --account-name $STORAGE_ACCOUNT_NAME --account-key $STORAGE_ACCOUNT_KEY --start $START --expiry $EXPIRE --https-only --resource-types sco --services f --permissions cwdlur -o tsv)
azcopy cp $CRS_CONVERSION_SOURCE_FOLDER "https://$STORAGE_ACCOUNT_NAME.file.core.windows.net/crs-conversion?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true
if [[ $? -gt 0 ]]; then
currentStatus="failure"
currentMessage="${currentMessage}. Failed to copy data to crs-conversion file share"
fi
azcopy cp $CRS_CATALOG_SOURCE_FOLDER "https://$STORAGE_ACCOUNT_NAME.file.core.windows.net/crs?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true
if [[ $? -gt 0 ]]; then
currentStatus="failure"
currentMessage="${currentMessage}. Failed to copy data to crs file share"
fi
azcopy cp $UNIT_SOURCE_FOLDER "https://$STORAGE_ACCOUNT_NAME.file.core.windows.net/unit?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true
if [[ $? -gt 0 ]]; then
currentStatus="failure"
currentMessage="${currentMessage}. Failed to copy data to unit file share"
fi
fi
if [ -z "$currentStatus" -a "$currentStatus"==" " ]; then
currentStatus="success"
fi
echo "Current Status: ${currentStatus}"
echo "Current Message: ${currentMessage}"
if [ ! -z "$CONFIG_MAP_NAME" -a "$CONFIG_MAP_NAME" != " " ]; then
az login --identity --username $OSDU_IDENTITY_ID
ENV_AKS=$(az aks list --resource-group $RESOURCE_GROUP_NAME --query [].name -otsv)
az aks get-credentials --resource-group $RESOURCE_GROUP_NAME --name $ENV_AKS
kubectl config set-context $RESOURCE_GROUP_NAME --cluster $ENV_AKS
Status=$(kubectl get configmap $CONFIG_MAP_NAME -o jsonpath='{.data.status}')
Message=$(kubectl get configmap $CONFIG_MAP_NAME -o jsonpath='{.data.message}')
Message="${Message}Helper Data Seeding Message: ${currentMessage}. "
## Update ConfigMap
kubectl create configmap $CONFIG_MAP_NAME \
--from-literal=status="$currentStatus" \
--from-literal=message="$Message" \
-o yaml --dry-run=client | kubectl replace -f -
fi
if [[ ${currentStatus} == "success" ]]; then
exit 0
else
exit 1
fi
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment