Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
Menu
Open sidebar
Open Subsurface Data Universe Software
Platform
Deployment and Operations
infra-azure-provisioning
Commits
3e32f91b
Commit
3e32f91b
authored
Sep 21, 2021
by
testRepo
Browse files
Adding docker files for data services
parent
f9c5c1f8
Pipeline
#66923
passed with stages
in 13 minutes and 2 seconds
Changes
3
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
tools/helper_services_data_seeding_agent/Dockerfile
0 → 100644
View file @
3e32f91b
FROM
mcr.microsoft.com/azure-cli
RUN
apk update
RUN
apk add ca-certificates wget
&&
update-ca-certificates
RUN
apk add libc6-compat
RUN
apk add git
RUN
apk add ca-certificates bash curl wget gettext jq bind-tools
\
&&
wget
-q
https://storage.googleapis.com/kubernetes-release/release/v1.21.2/bin/linux/amd64/kubectl
-O
/usr/local/bin/kubectl
\
&&
chmod
+x /usr/local/bin/kubectl
\
&&
chmod
g+rwx /root
\
&&
mkdir
/config
\
&&
chmod
g+rwx /config
\
&&
apk add
--update
coreutils
&&
rm
-rf
/var/cache/apk/
*
WORKDIR
/usr/src/app
COPY
transfer.sh ./
RUN
chmod
+x transfer.sh
ENTRYPOINT
[ "./transfer.sh" ]
\ No newline at end of file
tools/helper_services_data_seeding_agent/docker-compose.yaml
0 → 100644
View file @
3e32f91b
version
:
'
3'
services
:
helper-services-data-seeding-agent
:
build
:
context
:
../helper_services_data_seeding_agent
dockerfile
:
./Dockerfile
image
:
msosdu.azurecr.io/helper-services-data-seeding-agent:$VERSION
environment
:
OSDU_IDENTITY_ID
:
$OSDU_IDENTITY_ID
RESOURCE_GROUP_NAME
:
$RESOURCE_GROUP_NAME
CONFIG_MAP_NAME
:
$CONFIG_MAP_NAME
VERSION
:
$VERSION
\ No newline at end of file
tools/helper_services_data_seeding_agent/transfer.sh
0 → 100644
View file @
3e32f91b
#!/bin/bash
mkdir
-p
tmp
cd
tmp
wget
-O
azcopy_v10.tar.gz https://aka.ms/downloadazcopy-v10-linux
&&
tar
-xf
azcopy_v10.tar.gz
--strip-components
=
1
cp
./azcopy /usr/bin/
cd
..
currentStatus
=
""
currentMessage
=
""
git clone https://community.opengroup.org/osdu/platform/system/reference/crs-conversion-service.git
if
[[
$?
-gt
0
]]
;
then
currentMessage
=
"failure"
currentMessage
=
"
${
currentMessage
}
. Failed to clone crs-conversion-service"
fi
git clone https://community.opengroup.org/osdu/platform/system/reference/crs-catalog-service.git
if
[[
$?
-gt
0
]]
;
then
currentMessage
=
"failure"
currentMessage
=
"
${
currentMessage
}
. Failed to clone crs-catalog-service"
fi
git clone https://community.opengroup.org/osdu/platform/system/reference/unit-service.git
if
[[
$?
-gt
0
]]
;
then
currentMessage
=
"failure"
currentMessage
=
"
${
currentMessage
}
. Failed to clone unit-service"
fi
CRS_CONVERSION_SOURCE_FOLDER
=
"crs-conversion-service/apachesis_setup"
CRS_CATALOG_SOURCE_FOLDER
=
"crs-catalog-service/data/crs_catalog_v2.json"
UNIT_SOURCE_FOLDER
=
"unit-service/data/unit_catalog_v2.json"
az login
--identity
--username
$OSDU_IDENTITY_ID
ENV_VAULT
=
$(
az keyvault list
--resource-group
$RESOURCE_GROUP_NAME
--query
[]
.name
-otsv
)
STORAGE_ACCOUNT_NAME
=
$(
az keyvault secret show
--id
https://
${
ENV_VAULT
}
.vault.azure.net/secrets/airflow-storage
--query
value
-otsv
)
if
[
-z
"
$STORAGE_ACCOUNT_NAME
"
-a
"
$STORAGE_ACCOUNT_NAME
"
==
" "
]
;
then
currentStatus
=
"failure"
currentMessage
=
"
${
currentMessage
}
. Storage Account Name Not Found. "
fi
STORAGE_ACCOUNT_KEY
=
$(
az keyvault secret show
--id
https://
${
ENV_VAULT
}
.vault.azure.net/secrets/airflow-storage-key
--query
value
-otsv
)
if
[
-z
"
$STORAGE_ACCOUNT_KEY
"
-a
"
$STORAGE_ACCOUNT_KEY
"
==
" "
]
;
then
currentStatus
=
"failure"
currentMessage
=
"
${
currentMessage
}
. Storage Account Key Not Found. "
else
EXPIRE
=
$(
date
-u
-d
"59 minutes"
'+%Y-%m-%dT%H:%M:%SZ'
)
START
=
$(
date
-u
-d
"-1 minute"
'+%Y-%m-%dT%H:%M:%SZ'
)
#Generating the SAS Token required for Authorization
AZURE_STORAGE_SAS_TOKEN
=
$(
az storage account generate-sas
--account-name
$STORAGE_ACCOUNT_NAME
--account-key
$STORAGE_ACCOUNT_KEY
--start
$START
--expiry
$EXPIRE
--https-only
--resource-types
sco
--services
f
--permissions
cwdlur
-o
tsv
)
azcopy
cp
$CRS_CONVERSION_SOURCE_FOLDER
"https://
$STORAGE_ACCOUNT_NAME
.file.core.windows.net/crs-conversion?
${
AZURE_STORAGE_SAS_TOKEN
}
"
--recursive
=
true
if
[[
$?
-gt
0
]]
;
then
currentStatus
=
"failure"
currentMessage
=
"
${
currentMessage
}
. Failed to copy data to crs-conversion file share"
fi
azcopy
cp
$CRS_CATALOG_SOURCE_FOLDER
"https://
$STORAGE_ACCOUNT_NAME
.file.core.windows.net/crs?
${
AZURE_STORAGE_SAS_TOKEN
}
"
--recursive
=
true
if
[[
$?
-gt
0
]]
;
then
currentStatus
=
"failure"
currentMessage
=
"
${
currentMessage
}
. Failed to copy data to crs file share"
fi
azcopy
cp
$UNIT_SOURCE_FOLDER
"https://
$STORAGE_ACCOUNT_NAME
.file.core.windows.net/unit?
${
AZURE_STORAGE_SAS_TOKEN
}
"
--recursive
=
true
if
[[
$?
-gt
0
]]
;
then
currentStatus
=
"failure"
currentMessage
=
"
${
currentMessage
}
. Failed to copy data to unit file share"
fi
fi
if
[
-z
"
$currentStatus
"
-a
"
$currentStatus
"
==
" "
]
;
then
currentStatus
=
"success"
fi
echo
"Current Status:
${
currentStatus
}
"
echo
"Current Message:
${
currentMessage
}
"
if
[
!
-z
"
$CONFIG_MAP_NAME
"
-a
"
$CONFIG_MAP_NAME
"
!=
" "
]
;
then
az login
--identity
--username
$OSDU_IDENTITY_ID
ENV_AKS
=
$(
az aks list
--resource-group
$RESOURCE_GROUP_NAME
--query
[]
.name
-otsv
)
az aks get-credentials
--resource-group
$RESOURCE_GROUP_NAME
--name
$ENV_AKS
kubectl config set-context
$RESOURCE_GROUP_NAME
--cluster
$ENV_AKS
Status
=
$(
kubectl get configmap
$CONFIG_MAP_NAME
-o
jsonpath
=
'{.data.status}'
)
Message
=
$(
kubectl get configmap
$CONFIG_MAP_NAME
-o
jsonpath
=
'{.data.message}'
)
Message
=
"
${
Message
}
Helper Data Seeding Message:
${
currentMessage
}
. "
## Update ConfigMap
kubectl create configmap
$CONFIG_MAP_NAME
\
--from-literal
=
status
=
"
$currentStatus
"
\
--from-literal
=
message
=
"
$Message
"
\
-o
yaml
--dry-run
=
client | kubectl replace
-f
-
fi
if
[[
${
currentStatus
}
==
"success"
]]
;
then
exit
0
else
exit
1
fi
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment