Commit fe01afc0 authored by ethiraj krishnamanaidu's avatar ethiraj krishnamanaidu
Browse files

Merge branch 'master' into 'documenation-update'

# Conflicts:
#   README.md
parents 05658da9 00f32015
Pipeline #8903 passed with stage
in 6 seconds
# This pipeline config is for the ci-cd-pipelines project itself, and is not
# intended for inclusion in other projects. It generates an HTML view of the documentation
stages:
- document
- publish
# This job generates the HTML document, and copies image files to the output directory.
# We run this separately so that general documents from branches can be
# downloaded and reviewed.
generate-docs:
stage: document
image: opengroup/og-asciidoc-build-tools
artifacts:
paths:
- public
script:
- mkdir -p public
- cp doc/*.png public
- asciidoctor -v -D public -o index.html doc/_main.adoc
# This job only runs on master, and it pushes public documentation to the GitLab
# Pages server for easy access.
pages:
stage: publish
image: alpine
artifacts:
paths:
- public
# We need a script of some kind, so printing out what files are about to be
# included seems reasonable
script:
- find public
rules:
- if: $CI_COMMIT_BRANCH == 'master'
# OSDU CI/CD Pipeline Overview
Find the most recent documentation build [here](http://osdu.pages.community.opengroup.org/platform/ci-cd-pipelines/)
A common place for shared CI/CD Pipeline files, for use will all OSDU projects.
[The OSDU Platform System project](https://community.opengroup.org/osdu/platform/system) (including sub-projects) makes use of [GitLab's built-in CI/CD capabilities](https://docs.gitlab.com/ee/ci/introduction/) to provide a CI/CD pipeline that runs automatically on commits to the project. The stages in the pipeline include:
......
.maven:
image: maven:3.3.9-jdk-8
tags: ['docker-runner']
tags: ['osdu-medium']
variables:
MAVEN_REPO_PATH: "$CI_PROJECT_DIR/.m2/repository"
MAVEN_CLI_OPTS: "--batch-mode --settings=$CI_PROJECT_DIR/.mvn/community-maven.settings.xml"
......@@ -66,7 +66,9 @@
# --------------------------------------------------------------------------------
compile-and-unit-test:
extends: .maven
extends:
- .maven
- .skipForTriggeringMergeRequests
stage: build
script:
# First, build and deploy all the independent POM projects that we find
......
.aws_variables:
variables:
ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID
......@@ -21,6 +20,7 @@
DOMAIN: $AWS_TESTING_DOMAIN
LEGAL_URL: $AWS_LEGAL_URL
AWS_COGNITO_CLIENT_ID: $AWS_COGNITO_CLIENT_ID
AWS_COGNITO_USER_POOL_ID: $AWS_COGNITO_USER_POOL_ID
AWS_COGNITO_AUTH_FLOW: $AWS_COGNITO_AUTH_FLOW
AWS_COGNITO_AUTH_PARAMS_PASSWORD: $AWS_COGNITO_AUTH_PARAMS_PASSWORD
AWS_COGNITO_AUTH_PARAMS_USER: $AWS_COGNITO_AUTH_PARAMS_USER
......@@ -37,6 +37,7 @@
APPLICATION_PORT: $AWS_APPLICATION_PORT
HOST_URL: $AWS_LEGAL_URL
MY_TENANT: $AWS_TENANT_NAME
ENTITLEMENTS_TEST_TENANT: $AWS_ENTITLEMENTS_TEST_TENANT
AWS_S3_ENDPOINT: $AWS_S3_ENDPOINT
AWS_S3_REGION: $AWS_REGION
LOG_LEVEL: INFO
......@@ -44,6 +45,7 @@
S3_LEGAL_CONFIG_BUCKET: $AWS_S3_LEGAL_CONFIG_BUCKET
LEGAL_QUEUE: $AWS_LEGAL_QUEUE
TABLE_PREFIX: $AWS_TABLE_PREFIX
RESOURCE_PREFIX: $AWS_TABLE_PREFIX
DYNAMO_DB_REGION: $AWS_DYNAMO_DB_REGION
DYNAMO_DB_ENDPOINT: $AWS_DYNAMO_DB_ENDPOINT
DELIVERY_INT_TEST_BUCKET_NAME: $AWS_DELIVERY_INT_TEST_BUCKET_NAME
......@@ -59,12 +61,21 @@
DEFAULT_ELASTIC_PASSWORD: $AWS_REGION
ELASTIC_PORT: 443
INDEXER_HOST: $AWS_INDEXER_HOST
AWS_CLUSTER_NAME: $AWS_CLUSTER_NAME
ENTITLEMENTS_URL: $AWS_ENTITLEMENTS_URL
VIRTUAL_SERVICE_HOST_NAME: $AWS_API_GATEWAY_URL
.aws:
tags: ['docker-runner']
tags: ['osdu-medium']
image: divido2/aws-maven:v1.0
environment:
name: AWS
variables:
MAVEN_REPO_PATH: "$CI_PROJECT_DIR/.m2/repository"
MAVEN_CLI_OPTS: "--batch-mode --settings=$CI_PROJECT_DIR/.mvn/community-maven.settings.xml"
cache:
paths:
- $MAVEN_REPO_PATH
extends:
- .aws_variables
before_script:
......@@ -106,7 +117,7 @@ aws-update-ecs:
needs: ['aws-containerize']
script:
- ECS_SERVICE_NAME=$(aws ssm get-parameter --name ecs-$SERVICE_NAME --query Parameter.Value --output text --region $AWS_REGION)
- aws ecs update-service --cluster gitlab-core-cluster --service $ECS_SERVICE_NAME --region $AWS_REGION --force-new-deployment
- aws ecs update-service --cluster $AWS_CLUSTER_NAME --service $ECS_SERVICE_NAME --region $AWS_REGION --force-new-deployment
only:
variables:
- $AWS == 'true'
......@@ -120,7 +131,12 @@ aws-test:
script:
- ls -ltr
- cd $INTEGRATION_TEST_DIR
- mvn test -Dorg.slf4j.simpleLogger.defaultLogLevel=info
- mvn $MAVEN_CLI_OPTS -Dmaven.repo.local=$MAVEN_REPO_PATH test --update-snapshots -Dorg.slf4j.simpleLogger.defaultLogLevel=info
only:
variables:
- $AWS == 'true'
artifacts:
when: on_failure
paths:
- $INTEGRATION_TEST_DIR
expire_in: 1 week
--- # --------------------------------------------------------------------------------
# EXPECTED PIPELINE INHERITED GROUP VARIABLES
# --------------------------------------------------------------------------------
# AZURE (Protected Branch)
# AZURE_PRINCIPAL_ID (Protected Branch)
# AZURE_PRINCIPAL_SECRET (Protected Branch/Masked Variable)
# AZURE_REGISTRY (Protected Branch)
# JOBS
azure_containerize:
tags: ["osdu-medium"]
image: danielscholl/azure-build-image
stage: containerize
needs: ["compile-and-unit-test"]
variables:
SHA_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:${CI_COMMIT_SHA}
LATEST_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:latest
before_script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- az --version
- az login --service-principal -u $AZURE_PRINCIPAL_ID -p $AZURE_PRINCIPAL_SECRET --tenant $AZURE_TENANT_ID
script:
# Gitlab Container Registry
- docker build -f $AZURE_BUILD_SUBDIR/Dockerfile -t $CI_REGISTRY_IMAGE/$SHA_IMAGE .
- docker push ${CI_REGISTRY_IMAGE}/$SHA_IMAGE
# Azure Container Registry
- az acr login -n $AZURE_REGISTRY
- docker tag $CI_REGISTRY_IMAGE/$SHA_IMAGE ${AZURE_REGISTRY}.azurecr.io/$SHA_IMAGE
- docker push ${AZURE_REGISTRY}.azurecr.io/$SHA_IMAGE
- docker tag $CI_REGISTRY_IMAGE/$SHA_IMAGE ${AZURE_REGISTRY}.azurecr.io/$LATEST_IMAGE
- docker push ${AZURE_REGISTRY}.azurecr.io/$LATEST_IMAGE
only:
variables:
- $AZURE == 'true'
azure_deploy:
image: danielscholl/azure-build-image
tags: ["osdu-medium"]
stage: deploy
needs: ["azure_containerize"]
variables:
BRANCH: ${CI_COMMIT_REF_SLUG}
TAG: $CI_COMMIT_SHA
before_script:
- az login --service-principal -u $AZURE_PRINCIPAL_ID -p $AZURE_PRINCIPAL_SECRET --tenant $AZURE_TENANT_ID
- az aks get-credentials -g $AZURE_UNIQUE-rg -n $AZURE_UNIQUE-aks
script:
- cd devops/azure
- echo "--set image.branch=$BRANCH --set image.tag=$TAG"
# Install Service
- helm upgrade -i osdu-gitlab-$CI_PROJECT_NAME chart --set image.repository=${AZURE_REGISTRY}.azurecr.io --set image.branch=$BRANCH --set image.tag=$TAG
- pod=$(kubectl get pod -n osdu|grep $CI_PROJECT_NAME |tail -1 |awk '{print $1}')
- status=$(kubectl wait -n osdu --for=condition=Ready pod/$pod --timeout=100s)
- if [[ "$status" != *"met"* ]]; then echo "POD didn't start correctly" ; exit 1 ; fi
only:
variables:
- $AZURE == 'true'
......@@ -3,9 +3,10 @@
# AZURE (Protected Branch)
# AZURE_APP_ID (Protected Branch)
# AZURE_APP_ID_OTHER (Protected Branch)
# AZURE_APP_OID_OTHER (Protected Branch)
# AZURE_BASE (Protected Branch)
# AZURE_BASENAME (Protected Branch)
# AZURE_BASENAME_21 (Protected Branch)
# AZURE_DNS_NAME (Protected Branch)
# AZURE_ELASTIC_HOST (Protected Branch)
# AZURE_ELASTIC_PASSWORD (Protected Branch/Masked Variable)
# AZURE_INVALID_JWT (Protected Branch)
......@@ -13,52 +14,58 @@
# AZURE_NO_ACCESS_SECRET (Protected Branch/Masked Variable)
# AZURE_PRINCIPAL_ID (Protected Branch)
# AZURE_PRINCIPAL_SECRET (Protected Branch/Masked Variable)
# AZURE_REGISTRY (Protected Branch)
# AZURE_SERVICEBUS_KEY (Protected Branch/Masked Variable)
# AZURE_STORAGE_KEY (Protected Branch/Masked Variable)
# AZURE_SUBSCRIPTION_ID (Protected Branch)
# AZURE_SUBSCRIPTION_NAME (Protected Branch)
# AZURE_TENANT_ID (Protected Branch)
# EXPECTED PIPELINE VARIABLES
# --------------------------------------------------------------------------------
# AZURE_TEST_SUBDIR
.azure_variables:
variables:
# Deploy Section
AZURE_CLIENT_ID: $AZURE_PRINCIPAL_ID
AZURE_CLIENT_SECRET: $AZURE_PRINCIPAL_SECRET
AZURE_RESOURCE_GROUP: ${AZURE_BASENAME}-osdu-r2-app-rg
AZURE_APPSERVICE_PLAN: ${AZURE_BASENAME}-osdu-r2-sp
AZURE_APPSERVICE_NAME: ${AZURE_BASENAME_21}-au-${AZURE_SERVICE}
AZURE_CONTAINER_REGISTRY: ${AZURE_BASE}cr
AZURE_FUNCTIONAPP_NAME: ${AZURE_BASENAME_21}-enque
# Common Section
HOST_URL: https://${AZURE_BASENAME_21}-au-${AZURE_SERVICE}.azurewebsites.net/
ENTITLEMENT_URL: https://${AZURE_BASENAME_21}-au-entitlements.azurewebsites.net/
LEGAL_URL: https://${AZURE_BASENAME_21}-au-legal.azurewebsites.net/
STORAGE_URL: https://${AZURE_BASENAME_21}-au-storage.azurewebsites.net/
SEARCH_URL: https://${AZURE_BASENAME_21}-au-search.azurewebsites.net/api/search/v2/
INDEXER_URL: https://${AZURE_BASENAME_21}-au-indexer.azurewebsites.net/
ENTITLEMENT_URL: https://${AZURE_DNS_NAME}/entitlements/v1/
LEGAL_URL: https://${AZURE_DNS_NAME}/api/legal/v1/
STORAGE_URL: https://${AZURE_DNS_NAME}/api/storage/v2/
SEARCH_URL: https://${AZURE_DNS_NAME}/api/search/v2/
INDEXER_URL: https://${AZURE_DNS_NAME}/api/indexer/v2/
DELIVERY_URL: https://${AZURE_DNS_NAME}/api/delivery/v2/
AZURE_AD_TENANT_ID: $AZURE_TENANT_ID
INTEGRATION_TESTER: $AZURE_PRINCIPAL_ID
AZURE_TESTER_SERVICEPRINCIPAL_SECRET: $AZURE_PRINCIPAL_SECRET
AZURE_AD_APP_RESOURCE_ID: $AZURE_APP_ID
AZURE_STORAGE_ACCOUNT: ${AZURE_BASE}sa
MY_TENANT: opendes
SHARED_TENANT: common
DOMAIN: contoso.com
ELASTIC_HOST: $AZURE_ELASTIC_HOST
ELASTIC_PORT: 9243
ELASTIC_USER_NAME: elastic
ELASTIC_PASSWORD: $AZURE_ELASTIC_PASSWORD
VENDOR: azure
HOST: https://${AZURE_DNS_NAME}
# Entitlement Section
ENTITLEMENT_MEMBER_NAME_VALID: $AZURE_PRINCIPAL_ID
AZURE_AD_OTHER_APP_RESOURCE_ID: $AZURE_APP_ID_OTHER
AZURE_AD_OTHER_APP_RESOURCE_OID: $AZURE_APP_OID_OTHER
EXPIRED_TOKEN: $AZURE_INVALID_JWT
ENTITLEMENT_GROUP_NAME_VALID: integ.test.data.creator
ENTITLEMENT_MEMBER_NAME_INVALID: InvalidTestAdmin
AZURE_AD_USER_EMAIL: integration.test@azureglobal1.onmicrosoft.com
AZURE_AD_USER_OID: 469e9c25-ad0b-42e3-b023-03814437b21e
AZURE_AD_GUEST_EMAIL: integration.test@email.com
AZURE_AD_GUEST_OID: 4cf85597-116b-4aa5-bf03-7665a5b14ed5
# Legal Section
HOST_URL: https://${AZURE_DNS_NAME}/api/legal/v1/
AZURE_LEGAL_STORAGE_ACCOUNT: ${AZURE_BASE}sa
AZURE_LEGAL_STORAGE_KEY: $AZURE_STORAGE_KEY
LEGAL_STORAGE_CONTAINER: legal-service-azure-configuration
AZURE_LEGAL_SERVICEBUS: Endpoint=sb://${AZURE_BASENAME_21}sb.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=${AZURE_SERVICEBUS_KEY}
AZURE_LEGAL_SERVICEBUS: Endpoint=sb://${AZURE_BASENAME_21}-bus.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=${AZURE_SERVICEBUS_KEY}
AZURE_LEGAL_TOPICNAME: legaltags
# Storage Section
TENANT_NAME: opendes
......@@ -68,219 +75,107 @@
PUBSUB_TOKEN: az
DEPLOY_ENV: empty
# Indexer & Search Section
SEARCH_HOST: https://${AZURE_BASENAME_21}-au-search.azurewebsites.net/api/search/v2/
SEARCH_HOST: https://${AZURE_DNS_NAME}/api/search/v2/
aad_client_id: $AZURE_APP_ID
STORAGE_HOST: https://${AZURE_BASENAME_21}-au-storage.azurewebsites.net/
STORAGE_HOST: https://${AZURE_DNS_NAME}/api/storage/v2/
aad_client_id: $AZURE_APP_ID
DEFAULT_DATA_PARTITION_ID_TENANT1: opendes
DEFAULT_DATA_PARTITION_ID_TENANT2: common
DEFAULT_DATA_PARTITION_ID_TENANT2: $AZURE_DEFAULT_DATA_PARTITION_ID_TENANT2 # legal=common search=othertenant2
ENTITLEMENTS_DOMAIN: contoso.com
ENVIRONMENT: CLOUD
LEGAL_TAG: opendes-public-usa-dataset-7643990
OTHER_RELEVANT_DATA_COUNTRIES: US
# Delivery Section
LEGAL_HOST: $LEGAL_URL
DELIVERY_HOST: $DELIVERY_URL
#Schema
PRIVATE_TENANT1: $MY_TENANT
PRIVATE_TENANT2: tenant2
# JOBS
# --------------------------------------------------------------------------------
azure_debug:
stage: build
tags: ["docker-runner"]
image: maven:3.3.9-jdk-8
extends:
- .azure_variables
script: |
echo "# Pipeline Variables"
echo "export AZURE_SERVICE=\"$AZURE_SERVICE\""
echo "export AZURE_BUILD_SUBDIR=\"$AZURE_BUILD_SUBDIR\""
echo "export AZURE_TEST_SUBDIR=\"$AZURE_TEST_SUBDIR\""
echo "# Group Level Variables"
echo "export AZURE_APP_ID=\"$AZURE_APP_ID\""
echo "export AZURE_APP_ID_OTHER=\"$AZURE_APP_ID_OTHER\""
echo "export AZURE_BASE=\"$AZURE_BASE\""
echo "export AZURE_BASENAME=\"$AZURE_BASENAME\""
echo "export AZURE_BASENAME_21=\"$AZURE_BASENAME_21\""
echo "export AZURE_ELASTIC_HOST=\"$AZURE_ELASTIC_HOST\""
echo "export AZURE_ELASTIC_PASSWORD=\"$AZURE_ELASTIC_PASSWORD\""
echo "export AZURE_INVALID_JWT=\"$AZURE_INVALID_JWT\""
echo "export AZURE_NO_ACCESS_SECRET=\"$AZURE_NO_ACCESS_SECRET\""
echo "export AZURE_PRINCIPAL_ID=\"$AZURE_PRINCIPAL_ID\""
echo "export AZURE_PRINCIPAL_SECRET=\"$AZURE_PRINCIPAL_SECRET\""
echo "export AZURE_SERVICEBUS_KEY=\"$AZURE_SERVICEBUS_KEY\""
echo "export AZURE_STORAGE_KEY=\"$AZURE_STORAGE_KEY\""
echo "export AZURE_SUBSCRIPTION_ID=\"$AZURE_SUBSCRIPTION_ID\""
echo "export AZURE_SUBSCRIPTION_NAME=\"$AZURE_SUBSCRIPTION_NAME\""
echo "export AZURE_TENANT_ID=\"$AZURE_TENANT_ID\""
echo "# Deploy Section"
echo "export AZURE_CLIENT_ID=\"\$AZURE_PRINCIPAL_ID\""
echo "export AZURE_CLIENT_SECRET=\"\$AZURE_PRINCIPAL_SECRET\""
echo "export AZURE_RESOURCE_GROUP=\"${AZURE_BASENAME}-osdu-r2-app-rg\""
echo "export AZURE_APPSERVICE_PLAN=\"${AZURE_BASENAME}-osdu-r2-sp\""
echo "export AZURE_APPSERVICE_NAME=\"${AZURE_BASENAME_21}-au-${AZURE_SERVICE}\""
echo "export AZURE_CONTAINER_REGISTRY=\"${AZURE_BASE}cr\""
echo "export AZURE_FUNCTIONAPP_NAME=\"${AZURE_BASENAME_21}-enque\""
echo "# Common Section"
echo "export HOST_URL=\"$HOST_URL\""
echo "export ENTITLEMENT_URL=\"$ENTITLEMENT_URL\""
echo "export LEGAL_URL=\"$LEGAL_URL\""
echo "export STORAGE_URL=\"$STORAGE_URL\""s
echo "export SEARCH_URL=\"$SEARCH_URL\""
echo "export INDEXER_URL=\"$INDEXER_URL\""
echo "export INTEGRATION_TESTER=\"\$AZURE_PRINCIPAL_ID\""
echo "export AZURE_TESTER_SERVICEPRINCIPAL_SECRET=\"\$AZURE_PRINCIPAL_SECRET\""
echo "export AZURE_AD_APP_RESOURCE_ID=\"$AZURE_AD_APP_RESOURCE_ID\""
echo "export MY_TENANT=\"$MY_TENANT\""
echo "export DOMAIN=\"$DOMAIN\""
echo "export ELASTIC_HOST=\"\$AZURE_ELASTIC_HOST\""
echo "export ELASTIC_PORT=\"$ELASTIC_PORT\""
echo "export ELASTIC_USER_NAME=\"$ELASTIC_USER_NAME\""
echo "export ELASTIC_PASSWORD=\"\$AZURE_ELASTIC_PASSWORD\""
echo "# Entitlement Section"
echo "export ENTITLEMENT_MEMBER_NAME_VALID=\"\$AZURE_PRINCIPAL_ID\""
echo "export AZURE_AD_OTHER_APP_RESOURCE_ID=\"\$AZURE_APP_ID_OTHER\""
echo "export EXPIRED_TOKEN=\"\$AZURE_INVALID_JWT\""
echo "export ENTITLEMENT_GROUP_NAME_VALID=\"$ENTITLEMENT_GROUP_NAME_VALID\""
echo "export ENTITLEMENT_MEMBER_NAME_INVALID=\"$ENTITLEMENT_MEMBER_NAME_INVALID\""
echo "# Legal Section"
echo "export AZURE_LEGAL_STORAGE_ACCOUNT=\"$AZURE_LEGAL_STORAGE_ACCOUNT\""
echo "export AZURE_LEGAL_STORAGE_KEY=\"\$AZURE_STORAGE_KEY\""
echo "export LEGAL_STORAGE_CONTAINER=\"$LEGAL_STORAGE_CONTAINER\""
echo "export AZURE_LEGAL_SERVICEBUS=\"Endpoint=sb://${AZURE_BASENAME_21}sb.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=\${AZURE_SERVICEBUS_KEY}\""
echo "export AZURE_LEGAL_TOPICNAME=\"$AZURE_LEGAL_TOPICNAME\""
echo ""
echo "# Storage Section"
echo "export AZURE_AD_TENANT_ID=\"$AZURE_TENANT_ID\""
echo "export TENANT_NAME=\"$TENANT_NAME\""
echo "export TESTER_SERVICEPRINCIPAL_SECRET=\"\$AZURE_PRINCIPAL_SECRET\""
echo "export AZURE_STORAGE_ACCOUNT=\"$AZURE_STORAGE_ACCOUNT\""
echo "export NO_DATA_ACCESS_TESTER=\"$NO_DATA_ACCESS_TESTER\""
echo "export NO_DATA_ACCESS_TESTER_SERVICEPRINCIPAL_SECRET=\"\$AZURE_NO_ACCESS_SECRET\""
echo "export PUBSUB_TOKEN=\"$PUBSUB_TOKEN\""
echo "export DEPLOY_ENV=\"$DEPLOY_ENV\""
echo ""
echo "# Index Section"
echo "export SEARCH_HOST=\"$SEARCH_HOST\""
echo "export STORAGE_HOST=\"$STORAGE_HOST\""
echo "export aad_client_id=\"$aad_client_id\""
echo "export ELASTIC_HOST=\"$ELASTIC_HOST\""
echo "export ELASTIC_USER_NAME=\"$ELASTIC_USER_NAME\""
echo "export DEFAULT_DATA_PARTITION_ID_TENANT1=\"$DEFAULT_DATA_PARTITION_ID_TENANT1\""
echo "export DEFAULT_DATA_PARTITION_ID_TENANT2=\"$DEFAULT_DATA_PARTITION_ID_TENANT2\""
echo "export ENTITLEMENTS_DOMAIN=\"$ENTITLEMENTS_DOMAIN\""
echo "export ENVIRONMENT=\"$ENVIRONMENT\""
echo "export LEGAL_TAG=\"$LEGAL_TAG\""
echo "export OTHER_RELEVANT_DATA_COUNTRIES=\"$OTHER_RELEVANT_DATA_COUNTRIES\""
only:
variables:
- $AZURE_DEBUG == 'true'
azure_containerize:
tags: ["docker-runner"]
image: docker:latest
services:
- docker:dind
tags: ["osdu-medium"]
image: danielscholl/azure-build-image
stage: containerize
needs: ["compile-and-unit-test"]
variables:
IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}
SHA_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:${CI_COMMIT_SHA}
LATEST_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:latest
before_script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- apk add bash make py-pip
- apk add --virtual=build gcc libffi-dev musl-dev openssl-dev python3-dev
- pip3 install azure-cli
- az --version
- az login --service-principal -u $AZURE_PRINCIPAL_ID -p $AZURE_PRINCIPAL_SECRET --tenant $AZURE_TENANT_ID
script:
# Dockerfile
- |
echo 'FROM openjdk:8-jdk-alpine
VOLUME /tmp
ARG JAR_FILE
COPY ${JAR_FILE} app.jar
ENTRYPOINT ["java","-jar","/app.jar"]' > Dockerfile
- |
if [ "$AZURE_SERVICE" == "entitlements" ]; then
TARGET=$(find ./$AZURE_BUILD_SUBDIR/target -name '*.jar' |head -n 1)
else
TARGET=$(find ./$AZURE_BUILD_SUBDIR/target -name '*-spring-boot.jar' |head -n 1)
fi
# Gitlab Container Registry
- TARGET=$(find ./$AZURE_BUILD_SUBDIR/target -name '*.jar' | head -n 1)
- echo "Startup Jar is $TARGET"
- docker build --build-arg JAR_FILE=$TARGET -t ${CI_REGISTRY_IMAGE}/$IMAGE .
- docker tag ${CI_REGISTRY_IMAGE}/$IMAGE ${CI_REGISTRY_IMAGE}/$IMAGE:${CI_BUILD_ID}
- docker push ${CI_REGISTRY_IMAGE}/$IMAGE:${CI_BUILD_ID}
- docker build --build-arg JAR_FILE=$TARGET -t $CI_REGISTRY_IMAGE/$SHA_IMAGE .
- docker push ${CI_REGISTRY_IMAGE}/$SHA_IMAGE
# Azure Container Registry
- az acr login -n ${AZURE_BASE}cr
- docker tag ${CI_REGISTRY_IMAGE}/$IMAGE ${AZURE_BASE}cr.azurecr.io/$IMAGE:${CI_BUILD_ID}
- docker push ${AZURE_BASE}cr.azurecr.io/$IMAGE:${CI_BUILD_ID}
- az acr login -n $AZURE_REGISTRY
- docker tag $CI_REGISTRY_IMAGE/$SHA_IMAGE ${AZURE_REGISTRY}.azurecr.io/$SHA_IMAGE
- docker push ${AZURE_REGISTRY}.azurecr.io/$SHA_IMAGE
- docker tag $CI_REGISTRY_IMAGE/$SHA_IMAGE ${AZURE_REGISTRY}.azurecr.io/$LATEST_IMAGE
- docker push ${AZURE_REGISTRY}.azurecr.io/$LATEST_IMAGE
only:
variables:
- $AZURE == 'true'
except:
variables:
- $AZURE_DEBUG == 'true'
azure_deploy:
extends:
- .maven
- .azure_variables
image: danielscholl/azure-build-image
tags: ["osdu-medium"]
stage: deploy
needs: ["compile-and-unit-test"]
script:
- cd $AZURE_BUILD_SUBDIR && pwd
- |
$MAVEN azure-webapp:deploy \
-DAZURE_TENANT_ID=$AZURE_TENANT_ID \
-Dazure.appservice.subscription=$AZURE_SUBSCRIPTION_ID \
-DAZURE_CLIENT_ID=$AZURE_CLIENT_ID \
-DAZURE_CLIENT_SECRET=$AZURE_CLIENT_SECRET \
-Dazure.appservice.resourcegroup=$AZURE_RESOURCE_GROUP \
-Dazure.appservice.plan=$AZURE_APPSERVICE_PLAN \
-Dazure.appservice.appname=$AZURE_APPSERVICE_NAME
only:
variables:
- $AZURE == 'true'
except:
variables:
- $AZURE_DEBUG == 'true'
azure_config:
image: mcr.microsoft.com/azure-cli
stage: deploy
needs: ["compile-and-unit-test"]
needs: ["azure_containerize"]
variables:
BRANCH: ${CI_COMMIT_REF_SLUG}
TAG: $CI_COMMIT_SHA
extends:
- .azure_variables
before_script:
- az login --service-principal -u $AZURE_PRINCIPAL_ID -p $AZURE_PRINCIPAL_SECRET --tenant $AZURE_TENANT_ID
- az aks get-credentials -g $AZURE_UNIQUE-rg -n $AZURE_UNIQUE-aks
script:
- |
if [ "$AZURE_SERVICE" == "entitlements" ]; then
TARGET=$(find ./$AZURE_BUILD_SUBDIR/target -name '*.jar' |head -n 1)
else
TARGET=$(find ./$AZURE_BUILD_SUBDIR/target -name '*-spring-boot.jar' |head -n 1)
fi
- JAR_FILE=${TARGET##*/}
- echo "Startup Jar is $JAR_FILE"
- JAVA_COMMAND="java -jar /home/site/wwwroot/${JAR_FILE}"
- JSON_TEMPLATE='{"appCommandLine":"%s"}'
- JSON_FILE="config.json"
- echo $(printf "$JSON_TEMPLATE" "$JAVA_COMMAND") > $JSON_FILE
- az webapp config set --resource-group $AZURE_RESOURCE_GROUP --name $AZURE_APPSERVICE_NAME --generic-configurations @$JSON_FILE
- cd devops/azure
- echo "--set image.branch=$BRANCH --set image.tag=$TAG"
# Install Service
- helm upgrade -i osdu-gitlab-$CI_PROJECT_NAME chart --set image.repository=${AZURE_REGISTRY}.azurecr.io --set image.branch=$BRANCH --set image.tag=$TAG
- pod=$(kubectl get pod -n osdu|grep $CI_PROJECT_NAME |tail -1 |awk '{print $1}')
- status=$(kubectl wait -n osdu --for=condition=Ready pod/$pod --timeout=100s)
- if [[ "$status" != *"met"* ]]; then echo "POD didn't start correctly" ; exit 1 ; fi
only:
variables:
- $AZURE == 'true'
except:
variables:
- $AZURE_DEBUG == 'true'
azure_test:
stage: integration
needs: ["azure_config", "azure_deploy"]
needs: ["azure_deploy"]
extends:
- .maven
- .azure_variables
script:
- mvn clean test -f $AZURE_TEST_SUBDIR/pom.xml
- mvn clean verify -f $AZURE_TEST_SUBDIR/pom.xml # This Variable comes from the individual pipeline
only:
variables:
- $AZURE == 'true'
except:
variables:
- $AZURE_DEBUG == 'true' || $AZURE_SKIP_TEST == 'true'
- $AZURE_SKIP_TEST == 'true'
......@@ -17,7 +17,7 @@
GCP_VENDOR: gcp
.gcp:
tags: ['docker-runner']
tags: ['osdu-medium']
image: google/cloud-sdk
environment:
name: GCP
......@@ -72,4 +72,4 @@ gcp-test:
-DSTORAGE_URL=${GCP_STORAGE_URL} \
-DINT_TEST_VENDOR=${GCP_VENDOR} \
-DTENANT_GCP=${GCP_MY_TENANT_PROJECT} \
-DLEGAL_URL=${GCP_LEGAL_HOST_URL}
\ No newline at end of file
-DLEGAL_URL=${GCP_LEGAL_HOST_URL}
.ibm:
tags: ['docker-runner']
tags: ['osdu-medium']
image: openshift/origin-cli
environment:
name: IBM
......@@ -43,6 +43,7 @@ ibm-deploy:
SEARCH_HOST: $IBM_SEARCH_HOST
STORAGE_HOST: $IBM_STORAGE_HOST
DELIVERY_HOST: $IBM_DELIVERY_HOST
SCHEMA_HOST: $IBM_SCHEMA_HOST
# common security
KEYCLOAK_URL: $IBM_KEYCLOAK_URL
......@@ -83,6 +84,21 @@ ibm-deploy:
MY_TENANT_PROJECT: OpenDES_Project
IBM_ENV_PREFIX: $IBM_ENV_PREFIX
IBM_LEGAL_MQ_CONNECTION: $IBM_LEGAL_MQ_CONNECTION
# Schema
VENDOR: ibm
HOST: $IBM_SCHEMA_HOST
# Workflow
FINISHED_WORKFLOW_ID: $IBM_FINISHED_WORKFLOW_ID
WORKFLOW_HOST: $IBM_WORKFLOW_HOST
# File
FILE_SERVICE_HOST: $IBM_FILE_SERVICE_HOST
DATA_PARTITION_ID: $IBM_DATA_PARTITION_ID
# Unit
VIRTUAL_SERVICE_HOST_NAME: $IBM_UNIT_SERVICE_HOST
ibm-test:
stage: integration
......@@ -91,20 +107,45 @@ ibm-test:
- .maven
- .ibm_variables
script:
- $MAVEN install -f ${IBM_INT_TEST_SUBDIR/ibm/core}/pom.xml
- $MAVEN clean test -q -f $IBM_INT_TEST_SUBDIR/pom.xml > test-results.log
- cat $IBM_INT_TEST_SUBDIR/target/surefire-reports/*.txt
- if [[ $IBM_INT_TEST_SUBDIR == *"ibm"* ]]; then $MAVEN install -f ${IBM_INT_TEST_SUBDIR/ibm/core}/pom.xml -DskipTests=true; fi
- export TEST_CMD=${IBM_TEST_CMD:-test}
- $MAVEN $TEST_CMD -q -f $IBM_INT_TEST_SUBDIR/pom.xml > test-results.log
artifacts:
when: always
paths:
- test-results.log
- $IBM_INT_TEST_SUBDIR/target/surefire-reports/TEST-*.xml
- $IBM_INT_TEST_SUBDIR/target/*/TEST-*.xml
reports:
junit:
- $IBM_INT_TEST_SUBDIR/target/surefire-reports/TEST-*.xml
- $IBM_INT_TEST_SUBDIR/target/*/TEST-*.xml
only:
variables:
- $IBM_INT_TEST_SUBDIR && $IBM == 'true'
except:
variables:
- $IBM_SKIP_TEST == 'true'
ibm-test-py: