azure.yml 10.2 KB
Newer Older
Daniel Scholl's avatar
Daniel Scholl committed
1
2
3
4
5
# EXPECTED PIPELINE INHERITED GROUP VARIABLES
# --------------------------------------------------------------------------------
# AZURE                     (Protected Branch)
# AZURE_APP_ID              (Protected Branch)
# AZURE_APP_ID_OTHER        (Protected Branch)
6
# AZURE_APP_OID_OTHER       (Protected Branch)
Daniel Scholl's avatar
Daniel Scholl committed
7
8
# AZURE_BASE                (Protected Branch)
# AZURE_BASENAME_21         (Protected Branch)
Daniel Scholl's avatar
Daniel Scholl committed
9
# AZURE_DNS_NAME            (Protected Branch)
Daniel Scholl's avatar
Daniel Scholl committed
10
11
12
# AZURE_ELASTIC_HOST        (Protected Branch)
# AZURE_ELASTIC_PASSWORD    (Protected Branch/Masked Variable)
# AZURE_INVALID_JWT         (Protected Branch)
Daniel Scholl's avatar
Daniel Scholl committed
13
# AZURE_NO_ACCESS_ID        (Protected Branch)
Daniel Scholl's avatar
Daniel Scholl committed
14
# AZURE_NO_ACCESS_SECRET    (Protected Branch/Masked Variable)
Daniel Scholl's avatar
Daniel Scholl committed
15
# AZURE_PRINCIPAL_ID        (Protected Branch)
Daniel Scholl's avatar
Daniel Scholl committed
16
# AZURE_PRINCIPAL_SECRET    (Protected Branch/Masked Variable)
Daniel Scholl's avatar
Daniel Scholl committed
17
# AZURE_REGISTRY            (Protected Branch)
Daniel Scholl's avatar
Daniel Scholl committed
18
19
20
21
22
23
# AZURE_SERVICEBUS_KEY      (Protected Branch/Masked Variable)
# AZURE_STORAGE_KEY         (Protected Branch/Masked Variable)
# AZURE_SUBSCRIPTION_ID     (Protected Branch)
# AZURE_SUBSCRIPTION_NAME   (Protected Branch)
# AZURE_TENANT_ID           (Protected Branch)

Daniel Scholl's avatar
Daniel Scholl committed
24
25
26
27
# EXPECTED PIPELINE VARIABLES
# --------------------------------------------------------------------------------
# AZURE_TEST_SUBDIR

Daniel Scholl's avatar
Daniel Scholl committed
28
29
30

.azure_variables:
  variables:
Daniel Scholl's avatar
Daniel Scholl committed
31
    LOG_LEVEL: INFO
Daniel Scholl's avatar
Daniel Scholl committed
32
    # Common Section
Daniel Scholl's avatar
Daniel Scholl committed
33
34
35
36
37
    ENTITLEMENT_URL: https://${AZURE_DNS_NAME}/entitlements/v1/
    LEGAL_URL: https://${AZURE_DNS_NAME}/api/legal/v1/
    STORAGE_URL: https://${AZURE_DNS_NAME}/api/storage/v2/
    SEARCH_URL: https://${AZURE_DNS_NAME}/api/search/v2/
    INDEXER_URL: https://${AZURE_DNS_NAME}/api/indexer/v2/
38
    DELIVERY_URL: https://${AZURE_DNS_NAME}/api/delivery/v2/
Nicholas Karsky's avatar
Nicholas Karsky committed
39
    FILE_URL: https://${AZURE_DNS_NAME}/api/file/v2/
Daniel Scholl's avatar
Daniel Scholl committed
40
41
42
43
    AZURE_AD_TENANT_ID: $AZURE_TENANT_ID
    INTEGRATION_TESTER: $AZURE_PRINCIPAL_ID
    AZURE_TESTER_SERVICEPRINCIPAL_SECRET: $AZURE_PRINCIPAL_SECRET
    AZURE_AD_APP_RESOURCE_ID: $AZURE_APP_ID
44
    AZURE_STORAGE_ACCOUNT: ${AZURE_BASE}data
Daniel Scholl's avatar
Daniel Scholl committed
45
    MY_TENANT: opendes
46
    SHARED_TENANT: common
Daniel Scholl's avatar
Daniel Scholl committed
47
48
49
50
51
    DOMAIN: contoso.com
    ELASTIC_HOST: $AZURE_ELASTIC_HOST
    ELASTIC_PORT: 9243
    ELASTIC_USER_NAME: elastic
    ELASTIC_PASSWORD: $AZURE_ELASTIC_PASSWORD
52
53
    VENDOR: azure
    HOST: https://${AZURE_DNS_NAME}
54
55
    ACL_OWNERS: data.test1
    ACL_VIEWERS: data.test1
Daniel Scholl's avatar
Daniel Scholl committed
56
57
58
    # Entitlement Section
    ENTITLEMENT_MEMBER_NAME_VALID: $AZURE_PRINCIPAL_ID
    AZURE_AD_OTHER_APP_RESOURCE_ID: $AZURE_APP_ID_OTHER
59
    AZURE_AD_OTHER_APP_RESOURCE_OID: $AZURE_APP_OID_OTHER
Daniel Scholl's avatar
Daniel Scholl committed
60
61
62
    EXPIRED_TOKEN: $AZURE_INVALID_JWT
    ENTITLEMENT_GROUP_NAME_VALID: integ.test.data.creator
    ENTITLEMENT_MEMBER_NAME_INVALID: InvalidTestAdmin
63
64
65
66
    AZURE_AD_USER_EMAIL: integration.test@azureglobal1.onmicrosoft.com
    AZURE_AD_USER_OID: 469e9c25-ad0b-42e3-b023-03814437b21e
    AZURE_AD_GUEST_EMAIL: integration.test@email.com
    AZURE_AD_GUEST_OID: 4cf85597-116b-4aa5-bf03-7665a5b14ed5
Daniel Scholl's avatar
Daniel Scholl committed
67
    # Legal Section
Daniel Scholl's avatar
Daniel Scholl committed
68
    HOST_URL: https://${AZURE_DNS_NAME}/api/legal/v1/
69
    AZURE_LEGAL_STORAGE_ACCOUNT: ${AZURE_BASE}data
Daniel Scholl's avatar
Daniel Scholl committed
70
71
    AZURE_LEGAL_STORAGE_KEY: $AZURE_STORAGE_KEY
    LEGAL_STORAGE_CONTAINER: legal-service-azure-configuration
Daniel Scholl's avatar
Daniel Scholl committed
72
    AZURE_LEGAL_SERVICEBUS: Endpoint=sb://${AZURE_BASENAME_21}-bus.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=${AZURE_SERVICEBUS_KEY}
Daniel Scholl's avatar
Daniel Scholl committed
73
74
75
76
77
78
79
80
81
    AZURE_LEGAL_TOPICNAME: legaltags
    # Storage Section
    TENANT_NAME: opendes
    TESTER_SERVICEPRINCIPAL_SECRET: $AZURE_PRINCIPAL_SECRET
    NO_DATA_ACCESS_TESTER: $AZURE_NO_ACCESS_ID
    NO_DATA_ACCESS_TESTER_SERVICEPRINCIPAL_SECRET: $AZURE_NO_ACCESS_SECRET
    PUBSUB_TOKEN: az
    DEPLOY_ENV: empty
    # Indexer & Search Section
Daniel Scholl's avatar
Daniel Scholl committed
82
    SEARCH_HOST: https://${AZURE_DNS_NAME}/api/search/v2/
Daniel Scholl's avatar
Daniel Scholl committed
83
    aad_client_id: $AZURE_APP_ID
Daniel Scholl's avatar
Daniel Scholl committed
84
    STORAGE_HOST: https://${AZURE_DNS_NAME}/api/storage/v2/
85
    SCHEMA_HOST: https://${AZURE_DNS_NAME}/api/schema-service/v1
Daniel Scholl's avatar
Daniel Scholl committed
86
    aad_client_id: $AZURE_APP_ID
Daniel Scholl's avatar
Daniel Scholl committed
87
    DEFAULT_DATA_PARTITION_ID_TENANT1: opendes
Daniel Scholl's avatar
Daniel Scholl committed
88
    DEFAULT_DATA_PARTITION_ID_TENANT2: $AZURE_DEFAULT_DATA_PARTITION_ID_TENANT2  # legal=common search=othertenant2
Daniel Scholl's avatar
Daniel Scholl committed
89
90
91
92
    ENTITLEMENTS_DOMAIN: contoso.com
    ENVIRONMENT: CLOUD
    LEGAL_TAG: opendes-public-usa-dataset-7643990
    OTHER_RELEVANT_DATA_COUNTRIES: US
93
94
    # Partition Section
    PARTITION_BASE_URL: https://${AZURE_DNS_NAME}/
95
    # Delivery Section
Nicholas Karsky's avatar
Nicholas Karsky committed
96
97
    LEGAL_HOST: https://${AZURE_DNS_NAME}/api/legal/v1/
    DELIVERY_HOST: https://${AZURE_DNS_NAME}/api/delivery/v2/
98
99
100
    #Schema
    PRIVATE_TENANT1: $MY_TENANT
    PRIVATE_TENANT2: tenant2
Nicholas Karsky's avatar
Nicholas Karsky committed
101
    #File
102
    FILE_SERVICE_HOST: https://${AZURE_DNS_NAME}/api/file/v2
Nicholas Karsky's avatar
Nicholas Karsky committed
103
104
    USER_ID: "osdu-user"
    EXIST_FILE_ID: "8900a83f-18c6-4b1d-8f38-309a208779cc"
Jason's avatar
Jason committed
105
    DATA_PARTITION_ID: "opendes"
106
    TIME_ZONE: "UTC+0"
107
108
    #WKS
    OS_WKS_SCHEMA_KIND: opendes:wks:wellbore:1.0.0
Nicholas Karsky's avatar
Nicholas Karsky committed
109
110
    #Unit
    VIRTUAL_SERVICE_HOST_NAME: $AZURE_DNS_NAME
111
112
113
114
115
    #Register
    REGISTER_BASE_URL: https://${AZURE_DNS_NAME}/
    SUBSCRIPTION_ID: $AZURE_EVENT_SUBSCRIPTION_ID
    SUBSCRIBER_SECRET: $AZURE_EVENT_SUBSCRIBER_SECRET
    TEST_TOPIC_NAME: $AZURE_EVENT_TOPIC_NAME
116
117
118
119
120
121
    # Notification Section
    NOTIFICATION_BASE_URL: https://${AZURE_DNS_NAME}/api/notification/v1/
    NOTIFICATION_REGISTER_BASE_URL: https://${AZURE_DNS_NAME}
    REGISTER_CUSTOM_PUSH_URL_HMAC: https://${AZURE_DNS_NAME}/api/register/v1/test/challenge/1
    TOPIC_ID: $AZURE_EVENT_TOPIC_NAME
    HMAC_SECRET: $AZURE_EVENT_SUBSCRIBER_SECRET
Daniel Scholl's avatar
Daniel Scholl committed
122
123
124
125
126

# JOBS
# --------------------------------------------------------------------------------

azure_containerize:
Daniel Scholl's avatar
Daniel Scholl committed
127
128
129
130
131
  tags: ["osdu-medium"]
  image: danielscholl/azure-build-image
  stage: containerize
  needs: ["compile-and-unit-test"]
  variables:
Daniel Scholl's avatar
Bug Fix    
Daniel Scholl committed
132
    SHA_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:${CI_COMMIT_SHA}
Daniel Scholl's avatar
Daniel Scholl committed
133
    LATEST_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:latest
Daniel Scholl's avatar
Daniel Scholl committed
134
135
136
  before_script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
    - az --version
Daniel Scholl's avatar
Daniel Scholl committed
137
    - az login --service-principal -u $AZURE_PRINCIPAL_ID -p $AZURE_PRINCIPAL_SECRET --tenant $AZURE_TENANT_ID
Daniel Scholl's avatar
Daniel Scholl committed
138
  script:
Daniel Scholl's avatar
Daniel Scholl committed
139
    # Dockerfile
Daniel Scholl's avatar
Daniel Scholl committed
140
141
142
143
144
145
146
    - |
      echo 'FROM openjdk:8-jdk-alpine
            VOLUME /tmp
            ARG JAR_FILE
            COPY ${JAR_FILE} app.jar
            ENTRYPOINT ["java","-jar","/app.jar"]' > Dockerfile
    - |
Daniel Scholl's avatar
Daniel Scholl committed
147
      if [ "$AZURE_SERVICE"  == "entitlements" ]; then
Daniel Scholl's avatar
Daniel Scholl committed
148
149
150
151
        TARGET=$(find ./$AZURE_BUILD_SUBDIR/target -name '*.jar' |head -n 1)
      else
        TARGET=$(find ./$AZURE_BUILD_SUBDIR/target -name '*-spring-boot.jar' |head -n 1)
      fi
Daniel Scholl's avatar
Daniel Scholl committed
152

Daniel Scholl's avatar
Daniel Scholl committed
153
154
    # Gitlab Container Registry
    - echo "Startup Jar is $TARGET"
Daniel Scholl's avatar
Daniel Scholl committed
155
156
    - docker build --build-arg JAR_FILE=$TARGET -t $CI_REGISTRY_IMAGE/$SHA_IMAGE .
    - docker push ${CI_REGISTRY_IMAGE}/$SHA_IMAGE
157
    - docker tag $CI_REGISTRY_IMAGE/$SHA_IMAGE $CI_REGISTRY_IMAGE/$LATEST_IMAGE
158
    - docker push ${CI_REGISTRY_IMAGE}/$LATEST_IMAGE
Daniel Scholl's avatar
Daniel Scholl committed
159
160

    # Azure Container Registry
Daniel Scholl's avatar
Daniel Scholl committed
161
162
163
164
165
    - az acr login -n $AZURE_REGISTRY
    - docker tag $CI_REGISTRY_IMAGE/$SHA_IMAGE ${AZURE_REGISTRY}.azurecr.io/$SHA_IMAGE
    - docker push ${AZURE_REGISTRY}.azurecr.io/$SHA_IMAGE
    - docker tag $CI_REGISTRY_IMAGE/$SHA_IMAGE ${AZURE_REGISTRY}.azurecr.io/$LATEST_IMAGE
    - docker push ${AZURE_REGISTRY}.azurecr.io/$LATEST_IMAGE
Daniel Scholl's avatar
Daniel Scholl committed
166
167
168
169
  only:
    variables:
      - $AZURE == 'true'

Daniel Scholl's avatar
Daniel Scholl committed
170
azure_deploy:
Daniel Scholl's avatar
Daniel Scholl committed
171
172
173
  image: danielscholl/azure-build-image
  tags: ["osdu-medium"]
  stage: deploy
Daniel Scholl's avatar
Daniel Scholl committed
174
  needs: ["azure_containerize"]
Daniel Scholl's avatar
Daniel Scholl committed
175
176
177
178
179
180
  variables:
    BRANCH: ${CI_COMMIT_REF_SLUG}
    TAG: $CI_COMMIT_SHA
  extends:
    - .azure_variables
  before_script:
Daniel Scholl's avatar
Daniel Scholl committed
181
    - az login --service-principal -u $AZURE_PRINCIPAL_ID -p $AZURE_PRINCIPAL_SECRET --tenant $AZURE_TENANT_ID
Daniel Scholl's avatar
Daniel Scholl committed
182
183
    - az aks get-credentials -g $AZURE_UNIQUE-rg -n $AZURE_UNIQUE-aks
  script:
184
185
186
187
188
189
190
191
192
193
194
195
    #Update Crs Conversion Service Copy Dataset
    - |
      search_dir="apachesis_setup/SIS_DATA"
      if [ -d "$search_dir" ]; then
        echo "Starting to upload files for CRS Conversion Service"
        accountKey=$(kubectl get secret airflow -n osdu -o jsonpath='{.data.azurestorageaccountkey}' | base64 -d)
        accountName=$(kubectl get secret airflow -n osdu -o jsonpath='{.data.azurestorageaccountname}' | base64 -d)
        find "$search_dir/" -type f -print0 | while read -d $'\0' file; do
            az storage file upload --account-name $accountName --account-key $accountKey --share-name $SHARE_NAME --source $file
        done
        echo "File upload successfully completed for CRS Conversion Service"
      fi
Daniel Scholl's avatar
Daniel Scholl committed
196
    - cd devops/azure
197
    - echo "--set image.branch=$BRANCH --set image.tag=$TAG"
Daniel Scholl's avatar
Daniel Scholl committed
198

Nicholas Karsky's avatar
Nicholas Karsky committed
199
200
    #Update Catalog
    - |
201
      if [ -f ../../data/*_catalog_*.json ]; then
Nicholas Karsky's avatar
Nicholas Karsky committed
202
203
204
205
206
        accountKey=$(kubectl get secret airflow -n osdu -o jsonpath='{.data.azurestorageaccountkey}' | base64 -d)
        accountName=$(kubectl get secret airflow -n osdu -o jsonpath='{.data.azurestorageaccountname}' | base64 -d)
        az storage file upload --account-name $accountName --account-key $accountKey --share-name $SHARE_NAME --source ../../data/*_catalog_*.json
      fi

Daniel Scholl's avatar
Daniel Scholl committed
207
    # Install Service
Daniel Scholl's avatar
Daniel Scholl committed
208
    - helm upgrade -i osdu-gitlab-$CI_PROJECT_NAME chart --set image.repository=${AZURE_REGISTRY}.azurecr.io --set image.branch=$BRANCH --set image.tag=$TAG
209
210
    # Increasing to 900s as rolling updates are happening and each service is expected to have minimum 2 containers.
    - kubectl rollout status deployment.v1.apps/osdu-gitlab-$CI_PROJECT_NAME -n osdu --timeout=900s
Daniel Scholl's avatar
Daniel Scholl committed
211
    - pod=$(kubectl get pod -n osdu|grep $CI_PROJECT_NAME |tail -1 |awk '{print $1}')
212
    - status=$(kubectl wait -n osdu --for=condition=Ready pod/$pod --timeout=300s)
Daniel Scholl's avatar
Daniel Scholl committed
213
    - if [[ "$status" != *"met"* ]]; then echo "POD didn't start correctly" ; exit 1 ; fi
Daniel Scholl's avatar
Daniel Scholl committed
214
215
216
217
218
  only:
    variables:
      - $AZURE == 'true'

azure_test:
Nicholas Karsky's avatar
Nicholas Karsky committed
219
  image: $CI_REGISTRY/danielscholl/azure-maven/azure-maven:v1.0
Daniel Scholl's avatar
Daniel Scholl committed
220
  stage: integration
Daniel Scholl's avatar
Daniel Scholl committed
221
  needs: ["azure_deploy"]
Daniel Scholl's avatar
Daniel Scholl committed
222
223
224
225
  extends:
    - .maven
    - .azure_variables
  script:
Nicholas Karsky's avatar
Nicholas Karsky committed
226
    - |
Nicholas Karsky's avatar
Nicholas Karsky committed
227
228
229
230
      if [ "$AZURE_TEST_TYPE" == "python" ]; then
        cd $AZURE_TEST_SUBDIR
        chmod +x ./run-integration-tests.sh
        ./run-integration-tests.sh
Nicholas Karsky's avatar
Nicholas Karsky committed
231
      else
Nicholas Karsky's avatar
Nicholas Karsky committed
232
233
234
235
236
        if [ "$AZURE_SERVICE"  == "file" ] || [ "$AZURE_SERVICE"  == "delivery" ] || [ "$AZURE_SERVICE" == "ingestion-workflow" ]; then
          $MAVEN clean verify -f $AZURE_TEST_SUBDIR/pom.xml    # This Variable comes from the individual pipeline
        else
          mvn clean verify -f $AZURE_TEST_SUBDIR/pom.xml
        fi
Nicholas Karsky's avatar
Nicholas Karsky committed
237
      fi
Daniel Scholl's avatar
Daniel Scholl committed
238
239
240
241
242
  only:
    variables:
      - $AZURE == 'true'
  except:
    variables:
Daniel Scholl's avatar
Daniel Scholl committed
243
      - $AZURE_SKIP_TEST == 'true'
244