Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
Menu
Open sidebar
Open Subsurface Data Universe Software
Platform
Deployment and Operations
infra-azure-provisioning
Commits
32bdc7e4
Commit
32bdc7e4
authored
Sep 24, 2021
by
harshit aggarwal
Browse files
Merge branch 'master' into airflow-add-variable
parents
eeba6d75
4f196239
Pipeline
#67692
passed with stages
in 1 minute and 41 seconds
Changes
11
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
charts/airflow/helm-config-dp.yaml
View file @
32bdc7e4
...
...
@@ -331,8 +331,9 @@ airflow:
"
pyyaml==5.4.1"
,
"
requests==2.25.1"
,
"
tenacity==8.0.1"
,
"
https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_api-0.11.0.tar.gz"
,
"
https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_airflow-0.0.1.dev31+59e58330.tar.gz"
"
https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_api-0.10.1.dev151+503e364a.tar.gz"
,
"
https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_airflow-0.0.1.dev32+ea39f8bd.tar.gz"
]
extraVolumeMounts
:
-
name
:
azure-keyvault
...
...
charts/airflow/helm-config.yaml
View file @
32bdc7e4
...
...
@@ -346,8 +346,8 @@ airflow:
"
pyyaml==5.4.1"
,
"
requests==2.25.1"
,
"
tenacity==8.0.1"
,
"
https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_api-0.1
1.0
.tar.gz"
,
"
https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_airflow-0.0.1.dev3
1+59e58330
.tar.gz"
"
https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_api-0.1
0.1.dev151+503e364a
.tar.gz"
,
"
https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_airflow-0.0.1.dev3
2+ea39f8bd
.tar.gz"
]
extraVolumeMounts
:
-
name
:
azure-keyvault
...
...
charts/osdu-common/templates/appgw-ingress.yaml
View file @
32bdc7e4
...
...
@@ -121,3 +121,7 @@ spec:
serviceName
:
dataset
servicePort
:
80
path
:
/api/dataset/v1/*
-
backend
:
serviceName
:
seismic-dms-file-metadata-service
servicePort
:
80
path
:
/seismic-file-metadata/api/v1/*
charts/osdu-istio/templates/cert-checker-cronjob.yaml
View file @
32bdc7e4
...
...
@@ -51,6 +51,11 @@ spec:
configMapKeyRef
:
name
:
{{
.Values.global.job.configmap_name
}}
key
:
ENV_CLUSTER_NAME
-
name
:
ENV_APPGW_NAME
valueFrom
:
configMapKeyRef
:
name
:
{{
.Values.global.job.configmap_name
}}
key
:
ENV_APPGW_NAME
command
:
-
/bin/sh
args
:
...
...
@@ -62,6 +67,7 @@ spec:
# Compare expire dates of certificates in Key Vault and in istio-system namespaces
function check_expire_date() {
echo "Compare expire dates of certificates in Key Vault and in istio-system namespaces"
az keyvault certificate download --vault-name ${ENV_KEYVAULT_NAME} -n ${KV_CERT_NAME} --file ${KV_CERT_NAME}.pem
KV_CERT_EXPIREDATE=$(openssl x509 -in ${KV_CERT_NAME}.pem -enddate -noout | cut -d '=' -f2)
KV_CERT_EXPIREDATE=$(date "+%Y-%m-%d" --date="${KV_CERT_EXPIREDATE}")
...
...
@@ -84,7 +90,7 @@ spec:
# Cleanup function
cleanup() {
echo Clean all existing files
echo
"
Clean all existing files
"
rm -f cert.crt cert.key osdu-certificate.pfx ${KV_CERT_NAME}.pem
curl -X POST "http://localhost:${SIDECAR_PORT}/quitquitquit"
}
...
...
@@ -111,7 +117,7 @@ spec:
# Log In in Azure
az login --service-principal -u ${client_id} -p ${client_secret} --tenant ${tenant_id}
{{
- if .Values.global.istio.enableIstioKeyvaultCert
}}
{{
- if .Values.global.istio.enableIstioKeyvaultCert
}}
K8S_CERT_SECRET=istio-appgw-ssl-cert
K8S_NAMESPACE_NAME=istio-system
...
...
@@ -120,8 +126,13 @@ spec:
check_expire_date
# Download BYOC certificate from keyvault
echo "Download BYOC certificate from keyvault"
az keyvault secret download --file ${KV_CERT_NAME}.pfx --vault-name ${ENV_KEYVAULT_NAME} --encoding base64 --name ${KV_CERT_NAME}
# Upload BYOC certificate to appgw
echo "Upload BYOC certificate to appgw"
az network application-gateway ssl-cert update -g ${ENV_SR_GROUP_NAME} --gateway-name ${ENV_APPGW_NAME} -n ${KV_CERT_NAME} --cert-file ${K8S_CERT_SECRET}.pfx --cert-password ""
# Extract key and crt
openssl pkcs12 -in ${K8S_CERT_SECRET}.pfx -out cert.pem -passin pass:"" -nodes -passout pass:""
openssl rsa -in cert.pem -out cert.key
...
...
@@ -135,6 +146,9 @@ spec:
sleep
5
# Restart istio-ingressgateway pods
kubectl rollout restart -n ${K8S_ISTIONAMESPACE_NAME} deployment/istio-ingressgateway
# Check certificate expire date one more time
check_expire_date
...
...
charts/osdu-istio/templates/cert-init-job.yaml
View file @
32bdc7e4
...
...
@@ -43,6 +43,11 @@ spec:
configMapKeyRef
:
name
:
{{
.Values.global.job.configmap_name
}}
key
:
ENV_CLUSTER_NAME
-
name
:
ENV_APPGW_NAME
valueFrom
:
configMapKeyRef
:
name
:
{{
.Values.global.job.configmap_name
}}
key
:
ENV_APPGW_NAME
command
:
-
/bin/sh
args
:
...
...
@@ -82,14 +87,19 @@ spec:
az login --service-principal -u ${client_id} -p ${client_secret} --tenant ${tenant_id}
az aks get-credentials --resource-group ${ENV_SR_GROUP_NAME} --name ${ENV_CLUSTER_NAME}
{{
- if .Values.global.istio.enableIstioKeyvaultCert
}}
{{
- if .Values.global.istio.enableIstioKeyvaultCert
}}
K8S_CERT_SECRET=istio-appgw-ssl-cert
K8S_NAMESPACE_NAME=istio-system
# Download BYOC certificate from keyvault
echo "Download BYOC certificate from keyvault"
az keyvault secret download --file ${KV_CERT_NAME}.pfx --vault-name ${ENV_KEYVAULT_NAME} --encoding base64 --name ${KV_CERT_NAME}
# Upload BYOC certificate to appgw
echo "Upload BYOC certificate to appgw"
az network application-gateway ssl-cert create -g ${ENV_SR_GROUP_NAME} --gateway-name ${ENV_APPGW_NAME} -n ${KV_CERT_NAME} --cert-file ${K8S_CERT_SECRET}.pfx --cert-password ""
# Extract key and crt
openssl pkcs12 -in ${K8S_CERT_SECRET}.pfx -out cert.pem -passin pass:"" -nodes -passout pass:""
openssl rsa -in cert.pem -out cert.key
...
...
@@ -101,6 +111,9 @@ spec:
--from-file=tls.crt=cert.crt --from-file=tls.key=cert.key \
-o yaml | kubectl apply -f -
# Restart istio-ingressgateway pods
kubectl rollout restart -n ${K8S_ISTIONAMESPACE_NAME} deployment/istio-ingressgateway
{{
else
}}
K8S_CERT_SECRET=osdu-certificate
...
...
infra/templates/osdu-r3-mvp/service_resources/config_map.tf
View file @
32bdc7e4
...
...
@@ -66,6 +66,7 @@ resource "kubernetes_config_map" "appgw_configmap" {
ENV_SR_GROUP_NAME
=
azurerm_resource_group
.
main
.
name
ENV_KEYVAULT_NAME
=
data
.
terraform_remote_state
.
central_resources
.
outputs
.
keyvault_name
ENV_CLUSTER_NAME
=
module
.
aks
.
name
ENV_APPGW_NAME
=
module
.
istio_appgateway
[
count
.
index
].
name
}
depends_on
=
[
kubernetes_namespace
.
osdu
]
depends_on
=
[
kubernetes_namespace
.
osdu
,
module
.
istio_appgateway
]
}
infra/templates/osdu-r3-mvp/service_resources/main.tf
View file @
32bdc7e4
...
...
@@ -441,6 +441,39 @@ data "azurerm_resource_group" "aks_node_resource_group" {
name
=
module
.
aks
.
node_resource_group
}
// Give AD Principal Access rights to Change the Istio Application Gateway
resource
"azurerm_role_assignment"
"agic_istio_appgw_contributor"
{
count
=
var
.
feature_flag
.
autoscaling
?
1
:
0
principal_id
=
data
.
terraform_remote_state
.
central_resources
.
outputs
.
osdu_service_principal_id
scope
=
module
.
istio_appgateway
[
count
.
index
].
id
role_definition_name
=
"Contributor"
depends_on
=
[
module
.
istio_appgateway
]
}
// Give AD Principal Access rights to Operate the Istio Application Gateway Identity
resource
"azurerm_role_assignment"
"agic_istio_app_gw_contributor_for_adsp"
{
count
=
var
.
feature_flag
.
autoscaling
?
1
:
0
principal_id
=
data
.
terraform_remote_state
.
central_resources
.
outputs
.
osdu_service_principal_id
scope
=
module
.
istio_appgateway
[
count
.
index
].
managed_identity_resource_id
role_definition_name
=
"Managed Identity Operator"
depends_on
=
[
module
.
istio_appgateway
]
}
// Give AD Principal the rights to look at the Resource Group
resource
"azurerm_role_assignment"
"agic_istio_resourcegroup_reader"
{
count
=
var
.
feature_flag
.
autoscaling
?
1
:
0
principal_id
=
data
.
terraform_remote_state
.
central_resources
.
outputs
.
osdu_service_principal_id
scope
=
azurerm_resource_group
.
main
.
id
role_definition_name
=
"Reader"
depends_on
=
[
module
.
istio_appgateway
]
}
// Give AKS Access rights to Operate the Node Resource Group
resource
"azurerm_role_assignment"
"all_mi_operator"
{
principal_id
=
module
.
aks
.
kubelet_object_id
...
...
infra/templates/osdu-r3-mvp/service_resources/tests/unit/unit_test.go
View file @
32bdc7e4
...
...
@@ -36,8 +36,8 @@ var tfOptions = &terraform.Options{
}
var
istioEnabled
=
os
.
Getenv
(
"AUTOSCALING_ENABLED"
)
var
istioResourses
=
1
1
var
totalResources
=
1
38
var
istioResourses
=
1
4
var
totalResources
=
1
41
func
TestTemplate
(
t
*
testing
.
T
)
{
expectedAppDevResourceGroup
:=
asMap
(
t
,
`{
...
...
tools/helper_services_data_seeding_agent/Dockerfile
0 → 100644
View file @
32bdc7e4
FROM
mcr.microsoft.com/azure-cli
RUN
apk update
RUN
apk add ca-certificates wget
&&
update-ca-certificates
RUN
apk add libc6-compat
RUN
apk add git
RUN
apk add ca-certificates bash curl wget gettext jq bind-tools
\
&&
wget
-q
https://storage.googleapis.com/kubernetes-release/release/v1.21.2/bin/linux/amd64/kubectl
-O
/usr/local/bin/kubectl
\
&&
chmod
+x /usr/local/bin/kubectl
\
&&
chmod
g+rwx /root
\
&&
mkdir
/config
\
&&
chmod
g+rwx /config
\
&&
apk add
--update
coreutils
&&
rm
-rf
/var/cache/apk/
*
WORKDIR
/usr/src/app
COPY
transfer.sh ./
RUN
chmod
+x transfer.sh
ENTRYPOINT
[ "./transfer.sh" ]
\ No newline at end of file
tools/helper_services_data_seeding_agent/docker-compose.yaml
0 → 100644
View file @
32bdc7e4
version
:
'
3'
services
:
helper-services-data-seeding-agent
:
build
:
context
:
../helper_services_data_seeding_agent
dockerfile
:
./Dockerfile
image
:
msosdu.azurecr.io/helper-services-data-seeding-agent:$VERSION
environment
:
OSDU_IDENTITY_ID
:
$OSDU_IDENTITY_ID
RESOURCE_GROUP_NAME
:
$RESOURCE_GROUP_NAME
CONFIG_MAP_NAME
:
$CONFIG_MAP_NAME
VERSION
:
$VERSION
\ No newline at end of file
tools/helper_services_data_seeding_agent/transfer.sh
0 → 100644
View file @
32bdc7e4
#!/bin/bash
mkdir
-p
tmp
cd
tmp
wget
-O
azcopy_v10.tar.gz https://aka.ms/downloadazcopy-v10-linux
&&
tar
-xf
azcopy_v10.tar.gz
--strip-components
=
1
cp
./azcopy /usr/bin/
cd
..
currentStatus
=
""
currentMessage
=
""
git clone https://community.opengroup.org/osdu/platform/system/reference/crs-conversion-service.git
if
[[
$?
-gt
0
]]
;
then
currentMessage
=
"failure"
currentMessage
=
"
${
currentMessage
}
. Failed to clone crs-conversion-service"
fi
git clone https://community.opengroup.org/osdu/platform/system/reference/crs-catalog-service.git
if
[[
$?
-gt
0
]]
;
then
currentMessage
=
"failure"
currentMessage
=
"
${
currentMessage
}
. Failed to clone crs-catalog-service"
fi
git clone https://community.opengroup.org/osdu/platform/system/reference/unit-service.git
if
[[
$?
-gt
0
]]
;
then
currentMessage
=
"failure"
currentMessage
=
"
${
currentMessage
}
. Failed to clone unit-service"
fi
CRS_CONVERSION_SOURCE_FOLDER
=
"crs-conversion-service/apachesis_setup"
CRS_CATALOG_SOURCE_FOLDER
=
"crs-catalog-service/data/crs_catalog_v2.json"
UNIT_SOURCE_FOLDER
=
"unit-service/data/unit_catalog_v2.json"
az login
--identity
--username
$OSDU_IDENTITY_ID
ENV_VAULT
=
$(
az keyvault list
--resource-group
$RESOURCE_GROUP_NAME
--query
[]
.name
-otsv
)
STORAGE_ACCOUNT_NAME
=
$(
az keyvault secret show
--id
https://
${
ENV_VAULT
}
.vault.azure.net/secrets/airflow-storage
--query
value
-otsv
)
if
[
-z
"
$STORAGE_ACCOUNT_NAME
"
-a
"
$STORAGE_ACCOUNT_NAME
"
==
" "
]
;
then
currentStatus
=
"failure"
currentMessage
=
"
${
currentMessage
}
. Storage Account Name Not Found. "
fi
STORAGE_ACCOUNT_KEY
=
$(
az keyvault secret show
--id
https://
${
ENV_VAULT
}
.vault.azure.net/secrets/airflow-storage-key
--query
value
-otsv
)
if
[
-z
"
$STORAGE_ACCOUNT_KEY
"
-a
"
$STORAGE_ACCOUNT_KEY
"
==
" "
]
;
then
currentStatus
=
"failure"
currentMessage
=
"
${
currentMessage
}
. Storage Account Key Not Found. "
else
EXPIRE
=
$(
date
-u
-d
"59 minutes"
'+%Y-%m-%dT%H:%M:%SZ'
)
START
=
$(
date
-u
-d
"-1 minute"
'+%Y-%m-%dT%H:%M:%SZ'
)
#Generating the SAS Token required for Authorization
AZURE_STORAGE_SAS_TOKEN
=
$(
az storage account generate-sas
--account-name
$STORAGE_ACCOUNT_NAME
--account-key
$STORAGE_ACCOUNT_KEY
--start
$START
--expiry
$EXPIRE
--https-only
--resource-types
sco
--services
f
--permissions
cwdlur
-o
tsv
)
azcopy
cp
$CRS_CONVERSION_SOURCE_FOLDER
"https://
$STORAGE_ACCOUNT_NAME
.file.core.windows.net/crs-conversion?
${
AZURE_STORAGE_SAS_TOKEN
}
"
--recursive
=
true
if
[[
$?
-gt
0
]]
;
then
currentStatus
=
"failure"
currentMessage
=
"
${
currentMessage
}
. Failed to copy data to crs-conversion file share"
fi
azcopy
cp
$CRS_CATALOG_SOURCE_FOLDER
"https://
$STORAGE_ACCOUNT_NAME
.file.core.windows.net/crs?
${
AZURE_STORAGE_SAS_TOKEN
}
"
--recursive
=
true
if
[[
$?
-gt
0
]]
;
then
currentStatus
=
"failure"
currentMessage
=
"
${
currentMessage
}
. Failed to copy data to crs file share"
fi
azcopy
cp
$UNIT_SOURCE_FOLDER
"https://
$STORAGE_ACCOUNT_NAME
.file.core.windows.net/unit?
${
AZURE_STORAGE_SAS_TOKEN
}
"
--recursive
=
true
if
[[
$?
-gt
0
]]
;
then
currentStatus
=
"failure"
currentMessage
=
"
${
currentMessage
}
. Failed to copy data to unit file share"
fi
fi
if
[
-z
"
$currentStatus
"
-a
"
$currentStatus
"
==
" "
]
;
then
currentStatus
=
"success"
fi
echo
"Current Status:
${
currentStatus
}
"
echo
"Current Message:
${
currentMessage
}
"
if
[
!
-z
"
$CONFIG_MAP_NAME
"
-a
"
$CONFIG_MAP_NAME
"
!=
" "
]
;
then
az login
--identity
--username
$OSDU_IDENTITY_ID
ENV_AKS
=
$(
az aks list
--resource-group
$RESOURCE_GROUP_NAME
--query
[]
.name
-otsv
)
az aks get-credentials
--resource-group
$RESOURCE_GROUP_NAME
--name
$ENV_AKS
kubectl config set-context
$RESOURCE_GROUP_NAME
--cluster
$ENV_AKS
Status
=
$(
kubectl get configmap
$CONFIG_MAP_NAME
-o
jsonpath
=
'{.data.status}'
)
Message
=
$(
kubectl get configmap
$CONFIG_MAP_NAME
-o
jsonpath
=
'{.data.message}'
)
Message
=
"
${
Message
}
Helper Data Seeding Message:
${
currentMessage
}
. "
## Update ConfigMap
kubectl create configmap
$CONFIG_MAP_NAME
\
--from-literal
=
status
=
"
$currentStatus
"
\
--from-literal
=
message
=
"
$Message
"
\
-o
yaml
--dry-run
=
client | kubectl replace
-f
-
fi
if
[[
${
currentStatus
}
==
"success"
]]
;
then
exit
0
else
exit
1
fi
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment