Commit a0a76eef authored by harshit aggarwal's avatar harshit aggarwal Committed by Hema Vishnu Pola [Microsoft]
Browse files

Adding integration job for CI/CD

parent b495dfef
......@@ -20,3 +20,8 @@ analyze:
type: mvn
target: provider/wks-gcp/pom.xml
path: .
- name: os-wks-azure
type: mvn
target: provider/wks-azure/pom.xml
path: .
......@@ -2,7 +2,6 @@ variables:
AZURE_SERVICE: wks
AZURE_BUILD_SUBDIR: provider/wks-azure
AZURE_TEST_SUBDIR: testing/wks-test-core
AZURE_SKIP_TEST: 'true'
include:
- project: 'osdu/platform/ci-cd-pipelines'
......@@ -22,5 +21,5 @@ include:
file: 'scanners/gitlab-ultimate.yml'
- project: "osdu/platform/ci-cd-pipelines"
ref: master
file: "cloud-providers/azure.yml"
This diff is collapsed.
......@@ -115,7 +115,7 @@ stages:
INTEGRATION_TEST_AUDIENCE: $(INTEGRATION_TEST_AUDIENCE)
INTEGRATION_TESTER : $(INTEGRATION_TESTER)
WKS_BUCKET_NAME : $(WKS_BUCKET_NAME)
STORAGE_SERVICE_URL : $(STORAGE_SERVICE_URL)
STORAGE_URL : $(STORAGE_URL)
- task: PublishBuildArtifacts@1
......
......@@ -14,8 +14,8 @@ def upload_files():
local_file = join(local_folder, file)
blob_client = blob_service_client.get_blob_client(container=container_name, blob=file)
with open(local_file, "rb") as data:
blob_client.upload_blob(data, blob_type="BlockBlob")
blob_client.upload_blob(data, blob_type="BlockBlob", overwrite=True)
if __name__=="__main__":
upload_files()
\ No newline at end of file
upload_files()
os
azure.storage.blob
\ No newline at end of file
......@@ -15,22 +15,33 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ .Release.Name }}-{{ .Chart.Name }}
name: {{ .Release.Name }}
namespace: osdu
spec:
selector:
matchLabels:
app: {{ .Release.Name }}-{{ .Chart.Name }}
app: {{ .Chart.Name }}
template:
metadata:
labels:
app: {{ .Release.Name }}-{{ .Chart.Name }}
app: {{ .Chart.Name }}
aadpodidbinding: osdu-identity
spec:
volumes:
- name: azure-keyvault
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: azure-keyvault
containers:
- name: {{ .Release.Name }}-{{ .Chart.Name }}
- name: {{ .Chart.Name }}
image: {{ .Values.image.repository }}/{{ .Chart.Name }}-{{ .Values.image.branch }}:{{ .Values.image.tag | default .Chart.AppVersion }}
imagePullPolicy: Always
volumeMounts:
- name: azure-keyvault
mountPath: "/mnt/azure-keyvault"
readOnly: true
env:
- name: spring_application_name
value: wks
......@@ -82,14 +93,16 @@ spec:
name: servicebus
key: servicebus
- name: servicebus_topic_name
value: recordstopic
value: {{ .Values.azure.servicebusTopic }}
- name: storage_account
valueFrom:
configMapKeyRef:
name: osdu-svc-properties
key: ENV_STORAGE_ACCOUNT
- name: servicebus_topic_subscription
value: "recordstopicsubscription"
value: {{ .Values.azure.servicebusSubscription }}
- name: storage_container
value: {{ .Values.azure.storageContainer }}
- name: server_port
value: "80"
- name: storage_service_endpoint
......
......@@ -15,19 +15,19 @@
apiVersion: keda.k8s.io/v1alpha1
kind: ScaledObject
metadata:
name: {{ .Release.Name }}-{{ .Chart.Name }}
name: {{ .Chart.Name }}
namespace: osdu
labels:
deploymentName: {{ .Release.Name }}-{{ .Chart.Name }}
deploymentName: {{ .Chart.Name }}
spec:
scaleTargetRef:
deploymentName: {{ .Release.Name }}-{{ .Chart.Name }}
deploymentName: {{ .Chart.Name }}
triggers:
- type: azure-servicebus
metadata:
type: serviceBusTrigger
direction: in
name: message
subscriptionName: {{ .Values.config.azure.servicebusSubscription }}
topicName: {{ .Values.config.azure.servicebusTopic }}
subscriptionName: {{ .Values.azure.servicebusSubscription }}
topicName: {{ .Values.azure.servicebusTopic }}
connection: SERVICE_BUS
......@@ -12,10 +12,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
config:
name: wks
azure:
servicebusSubscription: recordstopicsubscription
servicebusTopic: recordstopic
azure:
servicebusSubscription: wkssubscription
servicebusTopic: recordstopic
storageContainer: osdu-wks-mappings
image:
branch: master
tag: latest
repository: community.opengroup.org:5555/osdu/platform/data-flow/enrichment/wks
\ No newline at end of file
---
# Source: wks/templates/deployment.yaml
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: apps/v1
kind: Deployment
metadata:
name: osdu-gitlab-wks
namespace: osdu
spec:
selector:
matchLabels:
app: wks
template:
metadata:
labels:
app: wks
aadpodidbinding: osdu-identity
spec:
volumes:
- name: azure-keyvault
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: azure-keyvault
containers:
- name: wks
image: community.opengroup.org:5555/osdu/platform/data-flow/enrichment/wks/wks-master:latest
imagePullPolicy: Always
volumeMounts:
- name: azure-keyvault
mountPath: "/mnt/azure-keyvault"
readOnly: true
env:
- name: spring_application_name
value: wks
- name: KEYVAULT_URI
valueFrom:
configMapKeyRef:
name: osdu-svc-properties
key: ENV_KEYVAULT
- name: AZURE_TENANT_ID
valueFrom:
configMapKeyRef:
name: osdu-svc-properties
key: ENV_TENANT_ID
- name: AZURE_CLIENT_ID
valueFrom:
secretKeyRef:
name: clientid
key: clientid
- name: AZURE_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: clientpassword
key: clientpassword
- name: aad_client_id
valueFrom:
secretKeyRef:
name: appid
key: appid
- name: AZURE_APP_RESOURCE_ID
valueFrom:
secretKeyRef:
name: appid
key: appid
- name: appinsights_key
valueFrom:
secretKeyRef:
name: appinsights
key: appinsights
- name: cosmosdb_database
value: osdu-db
- name: servicebus_namespace_name
valueFrom:
configMapKeyRef:
name: osdu-svc-properties
key: ENV_SERVICEBUS_NAMESPACE
- name: servicebus_connection_string
valueFrom:
secretKeyRef:
name: servicebus
key: servicebus
- name: servicebus_topic_name
value: recordstopic
- name: storage_account
valueFrom:
configMapKeyRef:
name: osdu-svc-properties
key: ENV_STORAGE_ACCOUNT
- name: servicebus_topic_subscription
value: wkssubscription
- name: storage_container
value: osdu-wks-mappings
- name: server_port
value: "80"
- name: storage_service_endpoint
value: "http://osdu-flux-storage/api/storage/v2"
- name: max_concurrent_calls
value: "1"
- name: executor_n_threads
value: "2"
- name: max_lock_renew_duration_seconds
value: "120"
---
# Source: wks/templates/scaled_object.yaml
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: keda.k8s.io/v1alpha1
kind: ScaledObject
metadata:
name: wks
namespace: osdu
labels:
deploymentName: wks
spec:
scaleTargetRef:
deploymentName: wks
triggers:
- type: azure-servicebus
metadata:
type: serviceBusTrigger
direction: in
name: message
subscriptionName: wkssubscription
topicName: recordstopic
connection: SERVICE_BUS
FROM openjdk:8-jdk-alpine
ARG JAR_FILE=target/*spring-boot.jar
COPY ${JAR_FILE} app.jar
ENTRYPOINT ["java","-jar","/app.jar"]
\ No newline at end of file
......@@ -19,34 +19,23 @@ In order to run the service locally, you will need to have the following environ
**System Environment required to run service**
Refer to [application.properties](./src/main/resources/application.properties)
Definitions for some variables used
Definitions for Azure Environment variables used
| name | value | description | sensitive? | source |
| --- | --- | --- | --- | --- |
| `LOG_PREFIX` | `storage` | Logging prefix | no | - |
| `server.servlet.contextPath` | `/api/storage/v2/` | Servlet context path | no | - |
| `STORAGE_API` | ex `https://foo-storage.azurewebsites.net` | Storage API endpoint | no | output of infrastructure deployment |
| `AUTHORIZE_API` | ex `https://foo-entitlements.azurewebsites.net` | Entitlements API endpoint | no | output of infrastructure deployment |
| `AUTHORIZE_API_KEY` | `********` | The API key clients will need to use when calling the entitlements | yes | -- |
| `azure.application-insights.instrumentation-key` | `********` | API Key for App Insights | yes | output of infrastructure deployment |
| `azure.activedirectory.client-id` | `********` | AAD client application ID | yes | output of infrastructure deployment |
| `azure.activedirectory.AppIdUri` | `api://${azure.activedirectory.client-id}` | URI for AAD Application | no | -- |
| `azure.activedirectory.session-stateless` | `true` | Flag run in stateless mode (needed by AAD dependency) | no | -- |
| `cosmosdb_account` | ex `devintosdur2cosmosacct` | Cosmos account name | no | output of infrastructure deployment |
| `cosmosdb_key` | `********` | Key for CosmosDB | yes | output of infrastructure deployments |
| `storage_service_endpoint` | ex `https://foo-storage.azurewebsites.net` | Storage API endpoint | no | output of infrastructure deployment |
| `appinsights_key` | `********` | API Key for App Insights | yes | output of infrastructure deployment |
| `cosmosdb_database` | ex `dev-osdu-r2-db` | Cosmos database for storage documents | no | output of infrastructure deployment |
| `azure.storage.account-name` | ex `foo-storage-account` | Storage account for storing documents | no | output of infrastructure deployment |
| `azure.storage.enable-https` | `true` | Used by spring boot starter library | no | - |
| `storage_account` | ex `foo-storage-account` | Storage account for storing documents | no | output of infrastructure deployment |
| `storage_container` | `opendes` | Storage Container name | no | - |
| `servicebus_topic_name` | `recordstopic` | Topic for async messaging | no | output of infrastructure deployment |
| `servicebus_namespace_name` | ex `foo-sb-namespace` | Namespace for async messaging | no | output of infrastructure deployment |
| `servicebus_connection_string` | `` | Connection string for service bus | no | |
| `KEYVAULT_URI` | ex `https://foo-keyvault.vault.azure.net/` | URI of KeyVault that holds application secrets | no | output of infrastructure deployment |
| `AZURE_CLIENT_ID` | `********` | Identity to run the service locally. This enables access to Azure resources. You only need this if running locally | yes | keyvault secret: `$KEYVAULT_URI/secrets/app-dev-sp-username` |
| `AZURE_TENANT_ID` | `********` | AD tenant to authenticate users from | yes | keyvault secret: `$KEYVAULT_URI/secrets/app-dev-sp-tenant-id` |
| `AZURE_CLIENT_SECRET` | `********` | Secret for `$AZURE_CLIENT_ID` | yes | keyvault secret: `$KEYVAULT_URI/secrets/app-dev-sp-password` |
### Build and run the application
After configuring your environment as specified above, you can follow these steps to build and run the application. These steps should be invoked from the *repository root*.
```bash
......
......@@ -30,7 +30,7 @@ azure.keyvault.url=${KEYVAULT_URI}
# Azure Storage configuration
azure.storage.account-name=${storage_account}
azure.storage.container-name=opendes
azure.storage.container-name=${storage_container}
# Azure Service Bus configuration
azure.servicebus.topic-name=${servicebus_topic_name}
......
......@@ -22,11 +22,11 @@ wks-core-test contains the step definitions and features for the Cucumber Integr
```
INTEGRATION_TESTER (System identity to assume for API calls. Note: this user must have entitlements configured already)
TESTER_SERVICEPRINCIPAL_SECRET (service account key which has admins api access)
AZURE_TESTER_SERVICEPRINCIPAL_SECRET (service account key which has admins api access)
AZURE_AD_TENANT_ID (AD tenant to authenticate users from)
AZURE_AD_APP_RESOURCE_ID (AAD client application ID)
VENDOR (cloud provider name eg. azure, gcp)
STORAGE_SERVICE_URL (storage service endpoint)
STORAGE_URL (storage service endpoint)
```
......
......@@ -350,7 +350,8 @@ public class IntegrationTestStepDefs implements En {
JsonElement jsonBody = new Gson().fromJson(payload, JsonElement.class);
payload = new Gson().toJson(jsonBody);
this.context.setInputPayload(payload);
String storageURL = System.getProperty("STORAGE_SERVICE_URL", System.getenv("STORAGE_SERVICE_URL"));
// Changing environment variable name to STORAGE_URL to maintain consistancy in naming
String storageURL = System.getProperty("STORAGE_URL", System.getenv("STORAGE_URL"));
HttpRequest httpRequest = HttpRequest.builder().url(storageURL + AutomationConstants.POST_ENDPOINT)
.body(jsonBody.toString()).httpMethod(HttpRequest.PUT).requestHeaders(this.context.getAuthHeaders())
.build();
......@@ -360,7 +361,7 @@ public class IntegrationTestStepDefs implements En {
}
private void deleteFromStorage(String recordId) {
String storageURL = System.getProperty("STORAGE_SERVICE_URL", System.getenv("STORAGE_SERVICE_URL"))
String storageURL = System.getProperty("STORAGE_URL", System.getenv("STORAGE_URL"))
+ AutomationConstants.DELETE_ENDPOINT + recordId;
HttpRequest httpRequest = HttpRequest.builder().url(storageURL)
.httpMethod(HttpRequest.DELETE).requestHeaders(this.context.getAuthHeaders())
......@@ -376,7 +377,7 @@ public class IntegrationTestStepDefs implements En {
}
private String storageGetResponse(String recordId) {
String reqURI = System.getProperty("STORAGE_SERVICE_URL", System.getenv("STORAGE_SERVICE_URL"))
String reqURI = System.getProperty("STORAGE_URL", System.getenv("STORAGE_URL"))
+ AutomationConstants.GET_ENDPOINT + recordId;
HttpRequest httpRequest = HttpRequest.builder().url(reqURI).httpMethod(HttpRequest.GET)
.requestHeaders(this.context.getAuthHeaders()).build();
......
......@@ -17,7 +17,7 @@ public class AuthUtil {
case "azure":
String clientId = System.getProperty("INTEGRATION_TESTER", System.getenv("INTEGRATION_TESTER"));
String clientSecret = System.getProperty("TESTER_SERVICEPRINCIPAL_SECRET", System.getenv("TESTER_SERVICEPRINCIPAL_SECRET"));
String clientSecret = System.getProperty("AZURE_TESTER_SERVICEPRINCIPAL_SECRET", System.getenv("AZURE_TESTER_SERVICEPRINCIPAL_SECRET"));
String tenantId = System.getProperty("AZURE_AD_TENANT_ID", System.getenv("AZURE_AD_TENANT_ID"));
String appResourceId = System.getProperty("AZURE_AD_APP_RESOURCE_ID", System.getenv("AZURE_AD_APP_RESOURCE_ID"));
token = new AzureServicePrincipal().getIdToken(clientId, clientSecret, tenantId, appResourceId);
......
<configuration>
<root level="info">
</root>
</configuration>
\ No newline at end of file
<configuration>
<root level="info">
</root>
</configuration>
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment