Commit 69a4e65e authored by Aryaan Singh's avatar Aryaan Singh
Browse files

Add docker container to data upload task

parent ef11d272
Pipeline #59492 passed with stages
in 1 minute and 51 seconds
FROM mcr.microsoft.com/azure-cli:2.15.1
RUN apk update
RUN apk add ca-certificates wget && update-ca-certificates
WORKDIR /usr/src/app
COPY data_seeding_agent/seed_data.sh ./
COPY test_data/Legal_COO.json ./test_data/
RUN chmod +x ./seed_data.sh
ENTRYPOINT [ "./seed_data.sh" ]
\ No newline at end of file
version: '3'
services:
data-seeding-agent:
build:
context: ../
dockerfile: ./data_seeding_agent/Dockerfile
image: msosdu.azurecr.io/data-seeding-agent:$VERSION
environment:
TENANT_ID: $TENANT_ID
PRINCIPAL_ID: $PRINCIPAL_ID
PRINCIPAL_SECRET: $PRINCIPAL_SECRET
PARTITION_NAME_ARRAY: $PARTITION_NAME_ARRAY # comma separated list of partitions
RESOURCE_GROUP_NAME: $RESOURCE_GROUP_NAME
VERSION: $VERSION
\ No newline at end of file
#!/bin/bash
# This logs the Azure CLI in using the configured service principal.
az login --service-principal -u $PRINCIPAL_ID -p $PRINCIPAL_SECRET --tenant $TENANT_ID
# The Legal_COO.json file needs to be loaded into the Data Partition Storage Account,
# in the container legal-service-azure-configuration.
ENV_VAULT=$(az keyvault list --resource-group $RESOURCE_GROUP_NAME --query [].name -otsv)
IFS=',' read -r -a partitions <<< ${PARTITION_NAME_ARRAY}
for index in "${!partitions[@]}"
do
echo "Ingesting file for partition: $index. ${partitions[index]}"
STORAGE_ACCOUNT_NAME=$(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/${partitions[index]}-storage --query value -otsv)
STORAGE_ACCOUNT_KEY=$(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/${partitions[index]}-storage-key --query value -otsv)
FILE_NAME=Legal_COO.json
if [ -z "$STORAGE_ACCOUNT_NAME" -a "$STORAGE_ACCOUNT_NAME"==" " ]; then
exit 1
fi
if [ -z "$STORAGE_ACCOUNT_KEY" -a "$STORAGE_ACCOUNT_KEY"==" " ]; then
exit 1
fi
az storage blob upload \
--account-name $STORAGE_ACCOUNT_NAME \
--account-key $STORAGE_ACCOUNT_KEY \
--file ./test_data/Legal_COO.json \
--container-name legal-service-azure-configuration \
--name $FILE_NAME
BLOB_LIST=$(az storage blob list \
--account-name $STORAGE_ACCOUNT_NAME \
--account-key $STORAGE_ACCOUNT_KEY \
--container-name legal-service-azure-configuration \
--query "[].{name:name}" -otsv)
if [[ ! " ${BLOB_LIST[@]} " =~ " ${FILE_NAME} " ]]; then
sleep 1m
az storage blob upload \
--account-name $STORAGE_ACCOUNT_NAME \
--account-key $STORAGE_ACCOUNT_KEY \
--file ./test_data/Legal_COO.json \
--container-name legal-service-azure-configuration \
--name $FILE_NAME
BLOB_LIST=$(az storage blob list \
--account-name $STORAGE_ACCOUNT_NAME \
--account-key $STORAGE_ACCOUNT_KEY \
--container-name legal-service-azure-configuration \
--query "[].{name:name}" -otsv)
if [[ ! " ${BLOB_LIST[@]} " =~ " ${FILE_NAME} " ]]; then
exit 1
fi
fi
echo "File ingested for partition: $index. ${partitions[index]}"
done
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment