azure.yml 7.68 KB
Newer Older
Daniel Scholl's avatar
Daniel Scholl committed
1
2
3
4
5
# EXPECTED PIPELINE INHERITED GROUP VARIABLES
# --------------------------------------------------------------------------------
# AZURE                     (Protected Branch)
# AZURE_APP_ID              (Protected Branch)
# AZURE_APP_ID_OTHER        (Protected Branch)
6
# AZURE_APP_OID_OTHER       (Protected Branch)
Daniel Scholl's avatar
Daniel Scholl committed
7
8
# AZURE_BASE                (Protected Branch)
# AZURE_BASENAME_21         (Protected Branch)
Daniel Scholl's avatar
Daniel Scholl committed
9
# AZURE_DNS_NAME            (Protected Branch)
Daniel Scholl's avatar
Daniel Scholl committed
10
11
12
# AZURE_ELASTIC_HOST        (Protected Branch)
# AZURE_ELASTIC_PASSWORD    (Protected Branch/Masked Variable)
# AZURE_INVALID_JWT         (Protected Branch)
Daniel Scholl's avatar
Daniel Scholl committed
13
# AZURE_NO_ACCESS_ID        (Protected Branch)
Daniel Scholl's avatar
Daniel Scholl committed
14
# AZURE_NO_ACCESS_SECRET    (Protected Branch/Masked Variable)
Daniel Scholl's avatar
Daniel Scholl committed
15
# AZURE_PRINCIPAL_ID        (Protected Branch)
Daniel Scholl's avatar
Daniel Scholl committed
16
# AZURE_PRINCIPAL_SECRET    (Protected Branch/Masked Variable)
Daniel Scholl's avatar
Daniel Scholl committed
17
# AZURE_REGISTRY            (Protected Branch)
Daniel Scholl's avatar
Daniel Scholl committed
18
19
20
21
22
23
# AZURE_SERVICEBUS_KEY      (Protected Branch/Masked Variable)
# AZURE_STORAGE_KEY         (Protected Branch/Masked Variable)
# AZURE_SUBSCRIPTION_ID     (Protected Branch)
# AZURE_SUBSCRIPTION_NAME   (Protected Branch)
# AZURE_TENANT_ID           (Protected Branch)

Daniel Scholl's avatar
Daniel Scholl committed
24
25
26
27
# EXPECTED PIPELINE VARIABLES
# --------------------------------------------------------------------------------
# AZURE_TEST_SUBDIR

Daniel Scholl's avatar
Daniel Scholl committed
28
29
30
31

.azure_variables:
  variables:
    # Common Section
Daniel Scholl's avatar
Daniel Scholl committed
32
33
34
35
36
    ENTITLEMENT_URL: https://${AZURE_DNS_NAME}/entitlements/v1/
    LEGAL_URL: https://${AZURE_DNS_NAME}/api/legal/v1/
    STORAGE_URL: https://${AZURE_DNS_NAME}/api/storage/v2/
    SEARCH_URL: https://${AZURE_DNS_NAME}/api/search/v2/
    INDEXER_URL: https://${AZURE_DNS_NAME}/api/indexer/v2/
37
    DELIVERY_URL: https://${AZURE_DNS_NAME}/api/delivery/v2/
Nicholas Karsky's avatar
Nicholas Karsky committed
38
    FILE_URL: https://${AZURE_DNS_NAME}/api/file/v2/
Daniel Scholl's avatar
Daniel Scholl committed
39
40
41
42
43
44
    AZURE_AD_TENANT_ID: $AZURE_TENANT_ID
    INTEGRATION_TESTER: $AZURE_PRINCIPAL_ID
    AZURE_TESTER_SERVICEPRINCIPAL_SECRET: $AZURE_PRINCIPAL_SECRET
    AZURE_AD_APP_RESOURCE_ID: $AZURE_APP_ID
    AZURE_STORAGE_ACCOUNT: ${AZURE_BASE}sa
    MY_TENANT: opendes
45
    SHARED_TENANT: common
Daniel Scholl's avatar
Daniel Scholl committed
46
47
48
49
50
    DOMAIN: contoso.com
    ELASTIC_HOST: $AZURE_ELASTIC_HOST
    ELASTIC_PORT: 9243
    ELASTIC_USER_NAME: elastic
    ELASTIC_PASSWORD: $AZURE_ELASTIC_PASSWORD
51
52
    VENDOR: azure
    HOST: https://${AZURE_DNS_NAME}
Daniel Scholl's avatar
Daniel Scholl committed
53
54
55
    # Entitlement Section
    ENTITLEMENT_MEMBER_NAME_VALID: $AZURE_PRINCIPAL_ID
    AZURE_AD_OTHER_APP_RESOURCE_ID: $AZURE_APP_ID_OTHER
56
    AZURE_AD_OTHER_APP_RESOURCE_OID: $AZURE_APP_OID_OTHER
Daniel Scholl's avatar
Daniel Scholl committed
57
58
59
    EXPIRED_TOKEN: $AZURE_INVALID_JWT
    ENTITLEMENT_GROUP_NAME_VALID: integ.test.data.creator
    ENTITLEMENT_MEMBER_NAME_INVALID: InvalidTestAdmin
60
61
62
63
    AZURE_AD_USER_EMAIL: integration.test@azureglobal1.onmicrosoft.com
    AZURE_AD_USER_OID: 469e9c25-ad0b-42e3-b023-03814437b21e
    AZURE_AD_GUEST_EMAIL: integration.test@email.com
    AZURE_AD_GUEST_OID: 4cf85597-116b-4aa5-bf03-7665a5b14ed5
Daniel Scholl's avatar
Daniel Scholl committed
64
    # Legal Section
Daniel Scholl's avatar
Daniel Scholl committed
65
    HOST_URL: https://${AZURE_DNS_NAME}/api/legal/v1/
Daniel Scholl's avatar
Daniel Scholl committed
66
    AZURE_LEGAL_STORAGE_ACCOUNT: ${AZURE_BASE}sa
Daniel Scholl's avatar
Daniel Scholl committed
67
68
    AZURE_LEGAL_STORAGE_KEY: $AZURE_STORAGE_KEY
    LEGAL_STORAGE_CONTAINER: legal-service-azure-configuration
Daniel Scholl's avatar
Daniel Scholl committed
69
    AZURE_LEGAL_SERVICEBUS: Endpoint=sb://${AZURE_BASENAME_21}-bus.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=${AZURE_SERVICEBUS_KEY}
Daniel Scholl's avatar
Daniel Scholl committed
70
71
72
73
74
75
76
77
78
    AZURE_LEGAL_TOPICNAME: legaltags
    # Storage Section
    TENANT_NAME: opendes
    TESTER_SERVICEPRINCIPAL_SECRET: $AZURE_PRINCIPAL_SECRET
    NO_DATA_ACCESS_TESTER: $AZURE_NO_ACCESS_ID
    NO_DATA_ACCESS_TESTER_SERVICEPRINCIPAL_SECRET: $AZURE_NO_ACCESS_SECRET
    PUBSUB_TOKEN: az
    DEPLOY_ENV: empty
    # Indexer & Search Section
Daniel Scholl's avatar
Daniel Scholl committed
79
    SEARCH_HOST: https://${AZURE_DNS_NAME}/api/search/v2/
Daniel Scholl's avatar
Daniel Scholl committed
80
    aad_client_id: $AZURE_APP_ID
Daniel Scholl's avatar
Daniel Scholl committed
81
    STORAGE_HOST: https://${AZURE_DNS_NAME}/api/storage/v2/
Daniel Scholl's avatar
Daniel Scholl committed
82
    aad_client_id: $AZURE_APP_ID
Daniel Scholl's avatar
Daniel Scholl committed
83
    DEFAULT_DATA_PARTITION_ID_TENANT1: opendes
Daniel Scholl's avatar
Daniel Scholl committed
84
    DEFAULT_DATA_PARTITION_ID_TENANT2: $AZURE_DEFAULT_DATA_PARTITION_ID_TENANT2  # legal=common search=othertenant2
Daniel Scholl's avatar
Daniel Scholl committed
85
86
87
88
    ENTITLEMENTS_DOMAIN: contoso.com
    ENVIRONMENT: CLOUD
    LEGAL_TAG: opendes-public-usa-dataset-7643990
    OTHER_RELEVANT_DATA_COUNTRIES: US
89
    # Delivery Section
Nicholas Karsky's avatar
Nicholas Karsky committed
90
91
    LEGAL_HOST: https://${AZURE_DNS_NAME}/api/legal/v1/
    DELIVERY_HOST: https://${AZURE_DNS_NAME}/api/delivery/v2/
92
93
94
    #Schema
    PRIVATE_TENANT1: $MY_TENANT
    PRIVATE_TENANT2: tenant2
Nicholas Karsky's avatar
Nicholas Karsky committed
95
    #File
Nicholas Karsky's avatar
Nicholas Karsky committed
96
    FILE_SERVICE_HOST: $FILE_URL
Nicholas Karsky's avatar
Nicholas Karsky committed
97
98
99
    USER_ID: "osdu-user"
    EXIST_FILE_ID: "8900a83f-18c6-4b1d-8f38-309a208779cc"

Daniel Scholl's avatar
Daniel Scholl committed
100
101
102
103
104
105
106



# JOBS
# --------------------------------------------------------------------------------

azure_containerize:
Daniel Scholl's avatar
Daniel Scholl committed
107
108
109
110
111
  tags: ["osdu-medium"]
  image: danielscholl/azure-build-image
  stage: containerize
  needs: ["compile-and-unit-test"]
  variables:
Daniel Scholl's avatar
Bug Fix    
Daniel Scholl committed
112
    SHA_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:${CI_COMMIT_SHA}
Daniel Scholl's avatar
Daniel Scholl committed
113
    LATEST_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:latest
Daniel Scholl's avatar
Daniel Scholl committed
114
115
116
  before_script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
    - az --version
Daniel Scholl's avatar
Daniel Scholl committed
117
    - az login --service-principal -u $AZURE_PRINCIPAL_ID -p $AZURE_PRINCIPAL_SECRET --tenant $AZURE_TENANT_ID
Daniel Scholl's avatar
Daniel Scholl committed
118
  script:
Daniel Scholl's avatar
Daniel Scholl committed
119
    # Dockerfile
Daniel Scholl's avatar
Daniel Scholl committed
120
121
122
123
124
125
126
    - |
      echo 'FROM openjdk:8-jdk-alpine
            VOLUME /tmp
            ARG JAR_FILE
            COPY ${JAR_FILE} app.jar
            ENTRYPOINT ["java","-jar","/app.jar"]' > Dockerfile
    - |
127
      if [ "$AZURE_SERVICE"  == "entitlements" ] || [ "$AZURE_SERVICE"  == "partition" ]; then
Daniel Scholl's avatar
Daniel Scholl committed
128
129
130
131
        TARGET=$(find ./$AZURE_BUILD_SUBDIR/target -name '*.jar' |head -n 1)
      else
        TARGET=$(find ./$AZURE_BUILD_SUBDIR/target -name '*-spring-boot.jar' |head -n 1)
      fi
Daniel Scholl's avatar
Daniel Scholl committed
132

Daniel Scholl's avatar
Daniel Scholl committed
133
134
    # Gitlab Container Registry
    - echo "Startup Jar is $TARGET"
Daniel Scholl's avatar
Daniel Scholl committed
135
136
    - docker build --build-arg JAR_FILE=$TARGET -t $CI_REGISTRY_IMAGE/$SHA_IMAGE .
    - docker push ${CI_REGISTRY_IMAGE}/$SHA_IMAGE
137
    - docker tag $CI_REGISTRY_IMAGE/$SHA_IMAGE $CI_REGISTRY_IMAGE/$LATEST_IMAGE
138
    - docker push ${CI_REGISTRY_IMAGE}/$LATEST_IMAGE
Daniel Scholl's avatar
Daniel Scholl committed
139
140

    # Azure Container Registry
Daniel Scholl's avatar
Daniel Scholl committed
141
142
143
144
145
    - az acr login -n $AZURE_REGISTRY
    - docker tag $CI_REGISTRY_IMAGE/$SHA_IMAGE ${AZURE_REGISTRY}.azurecr.io/$SHA_IMAGE
    - docker push ${AZURE_REGISTRY}.azurecr.io/$SHA_IMAGE
    - docker tag $CI_REGISTRY_IMAGE/$SHA_IMAGE ${AZURE_REGISTRY}.azurecr.io/$LATEST_IMAGE
    - docker push ${AZURE_REGISTRY}.azurecr.io/$LATEST_IMAGE
Daniel Scholl's avatar
Daniel Scholl committed
146
147
148
149
  only:
    variables:
      - $AZURE == 'true'

Daniel Scholl's avatar
Daniel Scholl committed
150
azure_deploy:
Daniel Scholl's avatar
Daniel Scholl committed
151
152
153
  image: danielscholl/azure-build-image
  tags: ["osdu-medium"]
  stage: deploy
Daniel Scholl's avatar
Daniel Scholl committed
154
  needs: ["azure_containerize"]
Daniel Scholl's avatar
Daniel Scholl committed
155
156
157
158
159
160
  variables:
    BRANCH: ${CI_COMMIT_REF_SLUG}
    TAG: $CI_COMMIT_SHA
  extends:
    - .azure_variables
  before_script:
Daniel Scholl's avatar
Daniel Scholl committed
161
    - az login --service-principal -u $AZURE_PRINCIPAL_ID -p $AZURE_PRINCIPAL_SECRET --tenant $AZURE_TENANT_ID
Daniel Scholl's avatar
Daniel Scholl committed
162
163
164
    - az aks get-credentials -g $AZURE_UNIQUE-rg -n $AZURE_UNIQUE-aks
  script:
    - cd devops/azure
165
    - echo "--set image.branch=$BRANCH --set image.tag=$TAG"
Daniel Scholl's avatar
Daniel Scholl committed
166
167

    # Install Service
Daniel Scholl's avatar
Daniel Scholl committed
168
    - helm upgrade -i osdu-gitlab-$CI_PROJECT_NAME chart --set image.repository=${AZURE_REGISTRY}.azurecr.io --set image.branch=$BRANCH --set image.tag=$TAG
Daniel Scholl's avatar
Daniel Scholl committed
169
    - pod=$(kubectl get pod -n osdu|grep $CI_PROJECT_NAME |tail -1 |awk '{print $1}')
170
    - status=$(kubectl wait -n osdu --for=condition=Ready pod/$pod --timeout=200s)
Daniel Scholl's avatar
Daniel Scholl committed
171
    - if [[ "$status" != *"met"* ]]; then echo "POD didn't start correctly" ; exit 1 ; fi
Daniel Scholl's avatar
Daniel Scholl committed
172
173
174
175
176
177
  only:
    variables:
      - $AZURE == 'true'

azure_test:
  stage: integration
Daniel Scholl's avatar
Daniel Scholl committed
178
  needs: ["azure_deploy"]
Daniel Scholl's avatar
Daniel Scholl committed
179
180
181
182
  extends:
    - .maven
    - .azure_variables
  script:
Nicholas Karsky's avatar
Nicholas Karsky committed
183
184
185
186
187
188
    - |
      if [ "$AZURE_SERVICE"  == "file" ] || [ "$AZURE_SERVICE"  == "delivery" ] || [ "$AZURE_SERVICE" == "ingestion-workflow" ]; then
        $MAVEN clean verify -f $AZURE_TEST_SUBDIR/pom.xml    # This Variable comes from the individual pipeline
      else
        mvn clean verify -f $AZURE_TEST_SUBDIR/pom.xml
      fi
Daniel Scholl's avatar
Daniel Scholl committed
189
190
191
192
193
  only:
    variables:
      - $AZURE == 'true'
  except:
    variables:
Daniel Scholl's avatar
Daniel Scholl committed
194
      - $AZURE_SKIP_TEST == 'true'