azure.yml 7.93 KB
Newer Older
Daniel Scholl's avatar
Daniel Scholl committed
1
2
3
4
5
# EXPECTED PIPELINE INHERITED GROUP VARIABLES
# --------------------------------------------------------------------------------
# AZURE                     (Protected Branch)
# AZURE_APP_ID              (Protected Branch)
# AZURE_APP_ID_OTHER        (Protected Branch)
6
# AZURE_APP_OID_OTHER       (Protected Branch)
Daniel Scholl's avatar
Daniel Scholl committed
7
8
# AZURE_BASE                (Protected Branch)
# AZURE_BASENAME_21         (Protected Branch)
Daniel Scholl's avatar
Daniel Scholl committed
9
# AZURE_DNS_NAME            (Protected Branch)
Daniel Scholl's avatar
Daniel Scholl committed
10
11
12
# AZURE_ELASTIC_HOST        (Protected Branch)
# AZURE_ELASTIC_PASSWORD    (Protected Branch/Masked Variable)
# AZURE_INVALID_JWT         (Protected Branch)
Daniel Scholl's avatar
Daniel Scholl committed
13
# AZURE_NO_ACCESS_ID        (Protected Branch)
Daniel Scholl's avatar
Daniel Scholl committed
14
# AZURE_NO_ACCESS_SECRET    (Protected Branch/Masked Variable)
Daniel Scholl's avatar
Daniel Scholl committed
15
# AZURE_PRINCIPAL_ID        (Protected Branch)
Daniel Scholl's avatar
Daniel Scholl committed
16
# AZURE_PRINCIPAL_SECRET    (Protected Branch/Masked Variable)
Daniel Scholl's avatar
Daniel Scholl committed
17
# AZURE_REGISTRY            (Protected Branch)
Daniel Scholl's avatar
Daniel Scholl committed
18
19
20
21
22
23
# AZURE_SERVICEBUS_KEY      (Protected Branch/Masked Variable)
# AZURE_STORAGE_KEY         (Protected Branch/Masked Variable)
# AZURE_SUBSCRIPTION_ID     (Protected Branch)
# AZURE_SUBSCRIPTION_NAME   (Protected Branch)
# AZURE_TENANT_ID           (Protected Branch)

Daniel Scholl's avatar
Daniel Scholl committed
24
25
26
27
# EXPECTED PIPELINE VARIABLES
# --------------------------------------------------------------------------------
# AZURE_TEST_SUBDIR

Daniel Scholl's avatar
Daniel Scholl committed
28
29
30

.azure_variables:
  variables:
Daniel Scholl's avatar
Daniel Scholl committed
31
    LOG_LEVEL: INFO
Daniel Scholl's avatar
Daniel Scholl committed
32
    # Common Section
Daniel Scholl's avatar
Daniel Scholl committed
33
34
35
36
37
    ENTITLEMENT_URL: https://${AZURE_DNS_NAME}/entitlements/v1/
    LEGAL_URL: https://${AZURE_DNS_NAME}/api/legal/v1/
    STORAGE_URL: https://${AZURE_DNS_NAME}/api/storage/v2/
    SEARCH_URL: https://${AZURE_DNS_NAME}/api/search/v2/
    INDEXER_URL: https://${AZURE_DNS_NAME}/api/indexer/v2/
38
    DELIVERY_URL: https://${AZURE_DNS_NAME}/api/delivery/v2/
Nicholas Karsky's avatar
Nicholas Karsky committed
39
    FILE_URL: https://${AZURE_DNS_NAME}/api/file/v2/
Daniel Scholl's avatar
Daniel Scholl committed
40
41
42
43
    AZURE_AD_TENANT_ID: $AZURE_TENANT_ID
    INTEGRATION_TESTER: $AZURE_PRINCIPAL_ID
    AZURE_TESTER_SERVICEPRINCIPAL_SECRET: $AZURE_PRINCIPAL_SECRET
    AZURE_AD_APP_RESOURCE_ID: $AZURE_APP_ID
44
    AZURE_STORAGE_ACCOUNT: ${AZURE_BASE}data
Daniel Scholl's avatar
Daniel Scholl committed
45
    MY_TENANT: opendes
46
    SHARED_TENANT: common
Daniel Scholl's avatar
Daniel Scholl committed
47
48
49
50
51
    DOMAIN: contoso.com
    ELASTIC_HOST: $AZURE_ELASTIC_HOST
    ELASTIC_PORT: 9243
    ELASTIC_USER_NAME: elastic
    ELASTIC_PASSWORD: $AZURE_ELASTIC_PASSWORD
52
53
    VENDOR: azure
    HOST: https://${AZURE_DNS_NAME}
54
55
    ACL_OWNERS: data.test1
    ACL_VIEWERS: data.test1
Daniel Scholl's avatar
Daniel Scholl committed
56
57
58
    # Entitlement Section
    ENTITLEMENT_MEMBER_NAME_VALID: $AZURE_PRINCIPAL_ID
    AZURE_AD_OTHER_APP_RESOURCE_ID: $AZURE_APP_ID_OTHER
59
    AZURE_AD_OTHER_APP_RESOURCE_OID: $AZURE_APP_OID_OTHER
Daniel Scholl's avatar
Daniel Scholl committed
60
61
62
    EXPIRED_TOKEN: $AZURE_INVALID_JWT
    ENTITLEMENT_GROUP_NAME_VALID: integ.test.data.creator
    ENTITLEMENT_MEMBER_NAME_INVALID: InvalidTestAdmin
63
64
65
66
    AZURE_AD_USER_EMAIL: integration.test@azureglobal1.onmicrosoft.com
    AZURE_AD_USER_OID: 469e9c25-ad0b-42e3-b023-03814437b21e
    AZURE_AD_GUEST_EMAIL: integration.test@email.com
    AZURE_AD_GUEST_OID: 4cf85597-116b-4aa5-bf03-7665a5b14ed5
Daniel Scholl's avatar
Daniel Scholl committed
67
    # Legal Section
Daniel Scholl's avatar
Daniel Scholl committed
68
    HOST_URL: https://${AZURE_DNS_NAME}/api/legal/v1/
69
    AZURE_LEGAL_STORAGE_ACCOUNT: ${AZURE_BASE}data
Daniel Scholl's avatar
Daniel Scholl committed
70
71
    AZURE_LEGAL_STORAGE_KEY: $AZURE_STORAGE_KEY
    LEGAL_STORAGE_CONTAINER: legal-service-azure-configuration
Daniel Scholl's avatar
Daniel Scholl committed
72
    AZURE_LEGAL_SERVICEBUS: Endpoint=sb://${AZURE_BASENAME_21}-bus.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=${AZURE_SERVICEBUS_KEY}
Daniel Scholl's avatar
Daniel Scholl committed
73
74
75
76
77
78
79
80
81
    AZURE_LEGAL_TOPICNAME: legaltags
    # Storage Section
    TENANT_NAME: opendes
    TESTER_SERVICEPRINCIPAL_SECRET: $AZURE_PRINCIPAL_SECRET
    NO_DATA_ACCESS_TESTER: $AZURE_NO_ACCESS_ID
    NO_DATA_ACCESS_TESTER_SERVICEPRINCIPAL_SECRET: $AZURE_NO_ACCESS_SECRET
    PUBSUB_TOKEN: az
    DEPLOY_ENV: empty
    # Indexer & Search Section
Daniel Scholl's avatar
Daniel Scholl committed
82
    SEARCH_HOST: https://${AZURE_DNS_NAME}/api/search/v2/
Daniel Scholl's avatar
Daniel Scholl committed
83
    aad_client_id: $AZURE_APP_ID
Daniel Scholl's avatar
Daniel Scholl committed
84
    STORAGE_HOST: https://${AZURE_DNS_NAME}/api/storage/v2/
Daniel Scholl's avatar
Daniel Scholl committed
85
    aad_client_id: $AZURE_APP_ID
Daniel Scholl's avatar
Daniel Scholl committed
86
    DEFAULT_DATA_PARTITION_ID_TENANT1: opendes
Daniel Scholl's avatar
Daniel Scholl committed
87
    DEFAULT_DATA_PARTITION_ID_TENANT2: $AZURE_DEFAULT_DATA_PARTITION_ID_TENANT2  # legal=common search=othertenant2
Daniel Scholl's avatar
Daniel Scholl committed
88
89
90
91
    ENTITLEMENTS_DOMAIN: contoso.com
    ENVIRONMENT: CLOUD
    LEGAL_TAG: opendes-public-usa-dataset-7643990
    OTHER_RELEVANT_DATA_COUNTRIES: US
92
93
    # Partition Section
    PARTITION_BASE_URL: https://${AZURE_DNS_NAME}/
94
    # Delivery Section
Nicholas Karsky's avatar
Nicholas Karsky committed
95
96
    LEGAL_HOST: https://${AZURE_DNS_NAME}/api/legal/v1/
    DELIVERY_HOST: https://${AZURE_DNS_NAME}/api/delivery/v2/
97
98
99
    #Schema
    PRIVATE_TENANT1: $MY_TENANT
    PRIVATE_TENANT2: tenant2
Nicholas Karsky's avatar
Nicholas Karsky committed
100
    #File
101
    FILE_SERVICE_HOST: https://${AZURE_DNS_NAME}/api/file/v2
Nicholas Karsky's avatar
Nicholas Karsky committed
102
103
    USER_ID: "osdu-user"
    EXIST_FILE_ID: "8900a83f-18c6-4b1d-8f38-309a208779cc"
Jason's avatar
Jason committed
104
    DATA_PARTITION_ID: "opendes"
105
    TIME_ZONE: "UTC+0"
106
107
    #WKS
    OS_WKS_SCHEMA_KIND: opendes:wks:wellbore:1.0.0
Daniel Scholl's avatar
Daniel Scholl committed
108
109
110
111
112
113


# JOBS
# --------------------------------------------------------------------------------

azure_containerize:
Daniel Scholl's avatar
Daniel Scholl committed
114
115
116
117
118
  tags: ["osdu-medium"]
  image: danielscholl/azure-build-image
  stage: containerize
  needs: ["compile-and-unit-test"]
  variables:
Daniel Scholl's avatar
Bug Fix    
Daniel Scholl committed
119
    SHA_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:${CI_COMMIT_SHA}
Daniel Scholl's avatar
Daniel Scholl committed
120
    LATEST_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:latest
Daniel Scholl's avatar
Daniel Scholl committed
121
122
123
  before_script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
    - az --version
Daniel Scholl's avatar
Daniel Scholl committed
124
    - az login --service-principal -u $AZURE_PRINCIPAL_ID -p $AZURE_PRINCIPAL_SECRET --tenant $AZURE_TENANT_ID
Daniel Scholl's avatar
Daniel Scholl committed
125
  script:
Daniel Scholl's avatar
Daniel Scholl committed
126
    # Dockerfile
Daniel Scholl's avatar
Daniel Scholl committed
127
128
129
130
131
132
133
    - |
      echo 'FROM openjdk:8-jdk-alpine
            VOLUME /tmp
            ARG JAR_FILE
            COPY ${JAR_FILE} app.jar
            ENTRYPOINT ["java","-jar","/app.jar"]' > Dockerfile
    - |
Daniel Scholl's avatar
Daniel Scholl committed
134
      if [ "$AZURE_SERVICE"  == "entitlements" ]; then
Daniel Scholl's avatar
Daniel Scholl committed
135
136
137
138
        TARGET=$(find ./$AZURE_BUILD_SUBDIR/target -name '*.jar' |head -n 1)
      else
        TARGET=$(find ./$AZURE_BUILD_SUBDIR/target -name '*-spring-boot.jar' |head -n 1)
      fi
Daniel Scholl's avatar
Daniel Scholl committed
139

Daniel Scholl's avatar
Daniel Scholl committed
140
141
    # Gitlab Container Registry
    - echo "Startup Jar is $TARGET"
Daniel Scholl's avatar
Daniel Scholl committed
142
143
    - docker build --build-arg JAR_FILE=$TARGET -t $CI_REGISTRY_IMAGE/$SHA_IMAGE .
    - docker push ${CI_REGISTRY_IMAGE}/$SHA_IMAGE
144
    - docker tag $CI_REGISTRY_IMAGE/$SHA_IMAGE $CI_REGISTRY_IMAGE/$LATEST_IMAGE
145
    - docker push ${CI_REGISTRY_IMAGE}/$LATEST_IMAGE
Daniel Scholl's avatar
Daniel Scholl committed
146
147

    # Azure Container Registry
Daniel Scholl's avatar
Daniel Scholl committed
148
149
150
151
152
    - az acr login -n $AZURE_REGISTRY
    - docker tag $CI_REGISTRY_IMAGE/$SHA_IMAGE ${AZURE_REGISTRY}.azurecr.io/$SHA_IMAGE
    - docker push ${AZURE_REGISTRY}.azurecr.io/$SHA_IMAGE
    - docker tag $CI_REGISTRY_IMAGE/$SHA_IMAGE ${AZURE_REGISTRY}.azurecr.io/$LATEST_IMAGE
    - docker push ${AZURE_REGISTRY}.azurecr.io/$LATEST_IMAGE
Daniel Scholl's avatar
Daniel Scholl committed
153
154
155
156
  only:
    variables:
      - $AZURE == 'true'

Daniel Scholl's avatar
Daniel Scholl committed
157
azure_deploy:
Daniel Scholl's avatar
Daniel Scholl committed
158
159
160
  image: danielscholl/azure-build-image
  tags: ["osdu-medium"]
  stage: deploy
Daniel Scholl's avatar
Daniel Scholl committed
161
  needs: ["azure_containerize"]
Daniel Scholl's avatar
Daniel Scholl committed
162
163
164
165
166
167
  variables:
    BRANCH: ${CI_COMMIT_REF_SLUG}
    TAG: $CI_COMMIT_SHA
  extends:
    - .azure_variables
  before_script:
Daniel Scholl's avatar
Daniel Scholl committed
168
    - az login --service-principal -u $AZURE_PRINCIPAL_ID -p $AZURE_PRINCIPAL_SECRET --tenant $AZURE_TENANT_ID
Daniel Scholl's avatar
Daniel Scholl committed
169
170
171
    - az aks get-credentials -g $AZURE_UNIQUE-rg -n $AZURE_UNIQUE-aks
  script:
    - cd devops/azure
172
    - echo "--set image.branch=$BRANCH --set image.tag=$TAG"
Daniel Scholl's avatar
Daniel Scholl committed
173
174

    # Install Service
Daniel Scholl's avatar
Daniel Scholl committed
175
    - helm upgrade -i osdu-gitlab-$CI_PROJECT_NAME chart --set image.repository=${AZURE_REGISTRY}.azurecr.io --set image.branch=$BRANCH --set image.tag=$TAG
Daniel Scholl's avatar
Daniel Scholl committed
176
    - pod=$(kubectl get pod -n osdu|grep $CI_PROJECT_NAME |tail -1 |awk '{print $1}')
177
    - status=$(kubectl wait -n osdu --for=condition=Ready pod/$pod --timeout=300s)
Daniel Scholl's avatar
Daniel Scholl committed
178
    - if [[ "$status" != *"met"* ]]; then echo "POD didn't start correctly" ; exit 1 ; fi
Daniel Scholl's avatar
Daniel Scholl committed
179
180
181
182
183
184
  only:
    variables:
      - $AZURE == 'true'

azure_test:
  stage: integration
Daniel Scholl's avatar
Daniel Scholl committed
185
  needs: ["azure_deploy"]
Daniel Scholl's avatar
Daniel Scholl committed
186
187
188
189
  extends:
    - .maven
    - .azure_variables
  script:
Nicholas Karsky's avatar
Nicholas Karsky committed
190
191
192
193
194
195
    - |
      if [ "$AZURE_SERVICE"  == "file" ] || [ "$AZURE_SERVICE"  == "delivery" ] || [ "$AZURE_SERVICE" == "ingestion-workflow" ]; then
        $MAVEN clean verify -f $AZURE_TEST_SUBDIR/pom.xml    # This Variable comes from the individual pipeline
      else
        mvn clean verify -f $AZURE_TEST_SUBDIR/pom.xml
      fi
Daniel Scholl's avatar
Daniel Scholl committed
196
197
198
199
200
  only:
    variables:
      - $AZURE == 'true'
  except:
    variables:
Daniel Scholl's avatar
Daniel Scholl committed
201
      - $AZURE_SKIP_TEST == 'true'