azure.yml 7.79 KB
Newer Older
Daniel Scholl's avatar
Daniel Scholl committed
1
2
3
4
5
# EXPECTED PIPELINE INHERITED GROUP VARIABLES
# --------------------------------------------------------------------------------
# AZURE                     (Protected Branch)
# AZURE_APP_ID              (Protected Branch)
# AZURE_APP_ID_OTHER        (Protected Branch)
6
# AZURE_APP_OID_OTHER       (Protected Branch)
Daniel Scholl's avatar
Daniel Scholl committed
7
8
# AZURE_BASE                (Protected Branch)
# AZURE_BASENAME_21         (Protected Branch)
Daniel Scholl's avatar
Daniel Scholl committed
9
# AZURE_DNS_NAME            (Protected Branch)
Daniel Scholl's avatar
Daniel Scholl committed
10
11
12
# AZURE_ELASTIC_HOST        (Protected Branch)
# AZURE_ELASTIC_PASSWORD    (Protected Branch/Masked Variable)
# AZURE_INVALID_JWT         (Protected Branch)
Daniel Scholl's avatar
Daniel Scholl committed
13
# AZURE_NO_ACCESS_ID        (Protected Branch)
Daniel Scholl's avatar
Daniel Scholl committed
14
# AZURE_NO_ACCESS_SECRET    (Protected Branch/Masked Variable)
Daniel Scholl's avatar
Daniel Scholl committed
15
# AZURE_PRINCIPAL_ID        (Protected Branch)
Daniel Scholl's avatar
Daniel Scholl committed
16
# AZURE_PRINCIPAL_SECRET    (Protected Branch/Masked Variable)
Daniel Scholl's avatar
Daniel Scholl committed
17
# AZURE_REGISTRY            (Protected Branch)
Daniel Scholl's avatar
Daniel Scholl committed
18
19
20
21
22
23
# AZURE_SERVICEBUS_KEY      (Protected Branch/Masked Variable)
# AZURE_STORAGE_KEY         (Protected Branch/Masked Variable)
# AZURE_SUBSCRIPTION_ID     (Protected Branch)
# AZURE_SUBSCRIPTION_NAME   (Protected Branch)
# AZURE_TENANT_ID           (Protected Branch)

Daniel Scholl's avatar
Daniel Scholl committed
24
25
26
27
# EXPECTED PIPELINE VARIABLES
# --------------------------------------------------------------------------------
# AZURE_TEST_SUBDIR

Daniel Scholl's avatar
Daniel Scholl committed
28
29
30

.azure_variables:
  variables:
Daniel Scholl's avatar
Daniel Scholl committed
31
    LOG_LEVEL: INFO
Daniel Scholl's avatar
Daniel Scholl committed
32
    # Common Section
Daniel Scholl's avatar
Daniel Scholl committed
33
34
35
36
37
    ENTITLEMENT_URL: https://${AZURE_DNS_NAME}/entitlements/v1/
    LEGAL_URL: https://${AZURE_DNS_NAME}/api/legal/v1/
    STORAGE_URL: https://${AZURE_DNS_NAME}/api/storage/v2/
    SEARCH_URL: https://${AZURE_DNS_NAME}/api/search/v2/
    INDEXER_URL: https://${AZURE_DNS_NAME}/api/indexer/v2/
38
    DELIVERY_URL: https://${AZURE_DNS_NAME}/api/delivery/v2/
Nicholas Karsky's avatar
Nicholas Karsky committed
39
    FILE_URL: https://${AZURE_DNS_NAME}/api/file/v2/
Daniel Scholl's avatar
Daniel Scholl committed
40
41
42
43
    AZURE_AD_TENANT_ID: $AZURE_TENANT_ID
    INTEGRATION_TESTER: $AZURE_PRINCIPAL_ID
    AZURE_TESTER_SERVICEPRINCIPAL_SECRET: $AZURE_PRINCIPAL_SECRET
    AZURE_AD_APP_RESOURCE_ID: $AZURE_APP_ID
44
    AZURE_STORAGE_ACCOUNT: ${AZURE_BASE}data
Daniel Scholl's avatar
Daniel Scholl committed
45
    MY_TENANT: opendes
46
    SHARED_TENANT: common
Daniel Scholl's avatar
Daniel Scholl committed
47
48
49
50
51
    DOMAIN: contoso.com
    ELASTIC_HOST: $AZURE_ELASTIC_HOST
    ELASTIC_PORT: 9243
    ELASTIC_USER_NAME: elastic
    ELASTIC_PASSWORD: $AZURE_ELASTIC_PASSWORD
52
53
    VENDOR: azure
    HOST: https://${AZURE_DNS_NAME}
Daniel Scholl's avatar
Daniel Scholl committed
54
55
56
    # Entitlement Section
    ENTITLEMENT_MEMBER_NAME_VALID: $AZURE_PRINCIPAL_ID
    AZURE_AD_OTHER_APP_RESOURCE_ID: $AZURE_APP_ID_OTHER
57
    AZURE_AD_OTHER_APP_RESOURCE_OID: $AZURE_APP_OID_OTHER
Daniel Scholl's avatar
Daniel Scholl committed
58
59
60
    EXPIRED_TOKEN: $AZURE_INVALID_JWT
    ENTITLEMENT_GROUP_NAME_VALID: integ.test.data.creator
    ENTITLEMENT_MEMBER_NAME_INVALID: InvalidTestAdmin
61
62
63
64
    AZURE_AD_USER_EMAIL: integration.test@azureglobal1.onmicrosoft.com
    AZURE_AD_USER_OID: 469e9c25-ad0b-42e3-b023-03814437b21e
    AZURE_AD_GUEST_EMAIL: integration.test@email.com
    AZURE_AD_GUEST_OID: 4cf85597-116b-4aa5-bf03-7665a5b14ed5
Daniel Scholl's avatar
Daniel Scholl committed
65
    # Legal Section
Daniel Scholl's avatar
Daniel Scholl committed
66
    HOST_URL: https://${AZURE_DNS_NAME}/api/legal/v1/
67
    AZURE_LEGAL_STORAGE_ACCOUNT: ${AZURE_BASE}data
Daniel Scholl's avatar
Daniel Scholl committed
68
69
    AZURE_LEGAL_STORAGE_KEY: $AZURE_STORAGE_KEY
    LEGAL_STORAGE_CONTAINER: legal-service-azure-configuration
Daniel Scholl's avatar
Daniel Scholl committed
70
    AZURE_LEGAL_SERVICEBUS: Endpoint=sb://${AZURE_BASENAME_21}-bus.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=${AZURE_SERVICEBUS_KEY}
Daniel Scholl's avatar
Daniel Scholl committed
71
72
73
74
75
76
77
78
79
    AZURE_LEGAL_TOPICNAME: legaltags
    # Storage Section
    TENANT_NAME: opendes
    TESTER_SERVICEPRINCIPAL_SECRET: $AZURE_PRINCIPAL_SECRET
    NO_DATA_ACCESS_TESTER: $AZURE_NO_ACCESS_ID
    NO_DATA_ACCESS_TESTER_SERVICEPRINCIPAL_SECRET: $AZURE_NO_ACCESS_SECRET
    PUBSUB_TOKEN: az
    DEPLOY_ENV: empty
    # Indexer & Search Section
Daniel Scholl's avatar
Daniel Scholl committed
80
    SEARCH_HOST: https://${AZURE_DNS_NAME}/api/search/v2/
Daniel Scholl's avatar
Daniel Scholl committed
81
    aad_client_id: $AZURE_APP_ID
Daniel Scholl's avatar
Daniel Scholl committed
82
    STORAGE_HOST: https://${AZURE_DNS_NAME}/api/storage/v2/
Daniel Scholl's avatar
Daniel Scholl committed
83
    aad_client_id: $AZURE_APP_ID
Daniel Scholl's avatar
Daniel Scholl committed
84
    DEFAULT_DATA_PARTITION_ID_TENANT1: opendes
Daniel Scholl's avatar
Daniel Scholl committed
85
    DEFAULT_DATA_PARTITION_ID_TENANT2: $AZURE_DEFAULT_DATA_PARTITION_ID_TENANT2  # legal=common search=othertenant2
Daniel Scholl's avatar
Daniel Scholl committed
86
87
88
89
    ENTITLEMENTS_DOMAIN: contoso.com
    ENVIRONMENT: CLOUD
    LEGAL_TAG: opendes-public-usa-dataset-7643990
    OTHER_RELEVANT_DATA_COUNTRIES: US
90
91
    # Partition Section
    PARTITION_BASE_URL: https://${AZURE_DNS_NAME}/
92
    # Delivery Section
Nicholas Karsky's avatar
Nicholas Karsky committed
93
94
    LEGAL_HOST: https://${AZURE_DNS_NAME}/api/legal/v1/
    DELIVERY_HOST: https://${AZURE_DNS_NAME}/api/delivery/v2/
95
96
97
    #Schema
    PRIVATE_TENANT1: $MY_TENANT
    PRIVATE_TENANT2: tenant2
Nicholas Karsky's avatar
Nicholas Karsky committed
98
    #File
99
    FILE_SERVICE_HOST: https://${AZURE_DNS_NAME}/api/file/v2
Nicholas Karsky's avatar
Nicholas Karsky committed
100
101
    USER_ID: "osdu-user"
    EXIST_FILE_ID: "8900a83f-18c6-4b1d-8f38-309a208779cc"
Jason's avatar
Jason committed
102
    DATA_PARTITION_ID: "opendes"
Nicholas Karsky's avatar
Nicholas Karsky committed
103

Daniel Scholl's avatar
Daniel Scholl committed
104
105
106
107
108
109
110



# JOBS
# --------------------------------------------------------------------------------

azure_containerize:
Daniel Scholl's avatar
Daniel Scholl committed
111
112
113
114
115
  tags: ["osdu-medium"]
  image: danielscholl/azure-build-image
  stage: containerize
  needs: ["compile-and-unit-test"]
  variables:
Daniel Scholl's avatar
Bug Fix    
Daniel Scholl committed
116
    SHA_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:${CI_COMMIT_SHA}
Daniel Scholl's avatar
Daniel Scholl committed
117
    LATEST_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:latest
Daniel Scholl's avatar
Daniel Scholl committed
118
119
120
  before_script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
    - az --version
Daniel Scholl's avatar
Daniel Scholl committed
121
    - az login --service-principal -u $AZURE_PRINCIPAL_ID -p $AZURE_PRINCIPAL_SECRET --tenant $AZURE_TENANT_ID
Daniel Scholl's avatar
Daniel Scholl committed
122
  script:
Daniel Scholl's avatar
Daniel Scholl committed
123
    # Dockerfile
Daniel Scholl's avatar
Daniel Scholl committed
124
125
126
127
128
129
130
    - |
      echo 'FROM openjdk:8-jdk-alpine
            VOLUME /tmp
            ARG JAR_FILE
            COPY ${JAR_FILE} app.jar
            ENTRYPOINT ["java","-jar","/app.jar"]' > Dockerfile
    - |
Daniel Scholl's avatar
Daniel Scholl committed
131
      if [ "$AZURE_SERVICE"  == "entitlements" ]; then
Daniel Scholl's avatar
Daniel Scholl committed
132
133
134
135
        TARGET=$(find ./$AZURE_BUILD_SUBDIR/target -name '*.jar' |head -n 1)
      else
        TARGET=$(find ./$AZURE_BUILD_SUBDIR/target -name '*-spring-boot.jar' |head -n 1)
      fi
Daniel Scholl's avatar
Daniel Scholl committed
136

Daniel Scholl's avatar
Daniel Scholl committed
137
138
    # Gitlab Container Registry
    - echo "Startup Jar is $TARGET"
Daniel Scholl's avatar
Daniel Scholl committed
139
140
    - docker build --build-arg JAR_FILE=$TARGET -t $CI_REGISTRY_IMAGE/$SHA_IMAGE .
    - docker push ${CI_REGISTRY_IMAGE}/$SHA_IMAGE
141
    - docker tag $CI_REGISTRY_IMAGE/$SHA_IMAGE $CI_REGISTRY_IMAGE/$LATEST_IMAGE
142
    - docker push ${CI_REGISTRY_IMAGE}/$LATEST_IMAGE
Daniel Scholl's avatar
Daniel Scholl committed
143
144

    # Azure Container Registry
Daniel Scholl's avatar
Daniel Scholl committed
145
146
147
148
149
    - az acr login -n $AZURE_REGISTRY
    - docker tag $CI_REGISTRY_IMAGE/$SHA_IMAGE ${AZURE_REGISTRY}.azurecr.io/$SHA_IMAGE
    - docker push ${AZURE_REGISTRY}.azurecr.io/$SHA_IMAGE
    - docker tag $CI_REGISTRY_IMAGE/$SHA_IMAGE ${AZURE_REGISTRY}.azurecr.io/$LATEST_IMAGE
    - docker push ${AZURE_REGISTRY}.azurecr.io/$LATEST_IMAGE
Daniel Scholl's avatar
Daniel Scholl committed
150
151
152
153
  only:
    variables:
      - $AZURE == 'true'

Daniel Scholl's avatar
Daniel Scholl committed
154
azure_deploy:
Daniel Scholl's avatar
Daniel Scholl committed
155
156
157
  image: danielscholl/azure-build-image
  tags: ["osdu-medium"]
  stage: deploy
Daniel Scholl's avatar
Daniel Scholl committed
158
  needs: ["azure_containerize"]
Daniel Scholl's avatar
Daniel Scholl committed
159
160
161
162
163
164
  variables:
    BRANCH: ${CI_COMMIT_REF_SLUG}
    TAG: $CI_COMMIT_SHA
  extends:
    - .azure_variables
  before_script:
Daniel Scholl's avatar
Daniel Scholl committed
165
    - az login --service-principal -u $AZURE_PRINCIPAL_ID -p $AZURE_PRINCIPAL_SECRET --tenant $AZURE_TENANT_ID
Daniel Scholl's avatar
Daniel Scholl committed
166
167
168
    - az aks get-credentials -g $AZURE_UNIQUE-rg -n $AZURE_UNIQUE-aks
  script:
    - cd devops/azure
169
    - echo "--set image.branch=$BRANCH --set image.tag=$TAG"
Daniel Scholl's avatar
Daniel Scholl committed
170
171

    # Install Service
Daniel Scholl's avatar
Daniel Scholl committed
172
    - helm upgrade -i osdu-gitlab-$CI_PROJECT_NAME chart --set image.repository=${AZURE_REGISTRY}.azurecr.io --set image.branch=$BRANCH --set image.tag=$TAG
Daniel Scholl's avatar
Daniel Scholl committed
173
    - pod=$(kubectl get pod -n osdu|grep $CI_PROJECT_NAME |tail -1 |awk '{print $1}')
174
    - status=$(kubectl wait -n osdu --for=condition=Ready pod/$pod --timeout=300s)
Daniel Scholl's avatar
Daniel Scholl committed
175
    - if [[ "$status" != *"met"* ]]; then echo "POD didn't start correctly" ; exit 1 ; fi
Daniel Scholl's avatar
Daniel Scholl committed
176
177
178
179
180
181
  only:
    variables:
      - $AZURE == 'true'

azure_test:
  stage: integration
Daniel Scholl's avatar
Daniel Scholl committed
182
  needs: ["azure_deploy"]
Daniel Scholl's avatar
Daniel Scholl committed
183
184
185
186
  extends:
    - .maven
    - .azure_variables
  script:
Nicholas Karsky's avatar
Nicholas Karsky committed
187
188
189
190
191
192
    - |
      if [ "$AZURE_SERVICE"  == "file" ] || [ "$AZURE_SERVICE"  == "delivery" ] || [ "$AZURE_SERVICE" == "ingestion-workflow" ]; then
        $MAVEN clean verify -f $AZURE_TEST_SUBDIR/pom.xml    # This Variable comes from the individual pipeline
      else
        mvn clean verify -f $AZURE_TEST_SUBDIR/pom.xml
      fi
Daniel Scholl's avatar
Daniel Scholl committed
193
194
195
196
197
  only:
    variables:
      - $AZURE == 'true'
  except:
    variables:
Daniel Scholl's avatar
Daniel Scholl committed
198
      - $AZURE_SKIP_TEST == 'true'