Commit 900f054b authored by neelesh thakur's avatar neelesh thakur
Browse files

Merge branch 'table-storage' into 'master'

update API to delegate secret retrieval via service and change azure backend to table storage

See merge request !14
parents cf959a19 017345ad
Pipeline #12055 passed with stages
in 15 minutes and 18 seconds
......@@ -38,6 +38,7 @@ load-tests/*.pyc
# Environment configuration
*.env
.sts4*
.envrc
# Intellij module setting file
*.iml
......@@ -47,4 +48,4 @@ load-tests/*.pyc
.DS_STORE
dist/
\ No newline at end of file
dist/
......@@ -4,6 +4,11 @@ variables:
AWS_SERVICE: partition
AWS_ENVIRONMENT: dev
AZURE_SERVICE: partition
AZURE_BUILD_SUBDIR: provider/partition-azure
AZURE_TEST_SUBDIR: testing/partition-test-azure
include:
- project: "osdu/platform/ci-cd-pipelines"
file: "standard-setup.yml"
......@@ -18,4 +23,7 @@ include:
file: "scanners/fossa.yml"
- project: 'osdu/platform/ci-cd-pipelines'
file: 'cloud-providers/aws.yml'
\ No newline at end of file
file: 'cloud-providers/aws.yml'
- project: "osdu/platform/ci-cd-pipelines"
file: "cloud-providers/azure.yml"
......@@ -36,11 +36,11 @@ The following software have components provided under the terms of this license:
- Apache Commons Collections (from http://commons.apache.org/proper/commons-collections/)
- Apache Commons Lang (from http://commons.apache.org/proper/commons-lang/)
- Apache Commons Logging (from http://commons.apache.org/proper/commons-logging/)
- Apache Commons Logging (from http://commons.apache.org/proper/commons-logging/)
- Apache Commons Text (from http://commons.apache.org/proper/commons-text/)
- Apache Commons Validator (from http://commons.apache.org/proper/commons-validator/)
- Apache Commons Validator (from http://commons.apache.org/proper/commons-validator/)
- Apache HttpClient (from http://hc.apache.org/httpcomponents-client)
- Apache HttpClient Cache (from http://hc.apache.org/httpcomponents-client)
- Apache HttpCore (from http://hc.apache.org/httpcomponents-core-ga)
- Apache Log4j API (from )
- Apache Log4j Core (from )
......@@ -158,6 +158,8 @@ The following software have components provided under the terms of this license:
- Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure SDK for Key Vault (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure Storage Client SDK (from https://github.com/Azure/azure-storage-java)
- Mockito (from http://mockito.org)
- Mockito (from http://mockito.org)
- Mojo's Maven plugin for Cobertura (from http://mojo.codehaus.org/cobertura-maven-plugin/)
......@@ -536,6 +538,7 @@ The following software have components provided under the terms of this license:
- Microsoft Azure Java Core Library (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure SDK annotations (from https://github.com/Microsoft/java-api-annotations)
- Microsoft Azure SDK for Key Vault (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure SDK for Service Bus (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure client library for Blob Storage (from https://github.com/Azure/azure-sdk-for-java)
......
# Pipeline Support Commands
```bash
AZURE_SERVICE="partition"
REPO_BRANCH="master"
TAG="latest"
PARTIAL=${REPO_BRANCH/\//-}
BRANCH=${PARTIAL/./-}
echo "--set image.branch=$BRANCH --set image.tag=$TAG"
# Install the Service
helm upgrade -i osdu-gitlab-$AZURE_SERVICE chart --set image.branch=$BRANCH --set image.tag=$TAG
pod=$(kubectl get pod |grep $AZURE_SERVICE | tail -1 | awk '{print $1}')
status=$(kubectl wait --for=condition=Ready pod/$pod --timeout=60s)
if [[ "$status" != *"met"* ]]; then echo "POD didn't start correctly" ; exit 1 ; fi
```
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: v2
name: partition
appVersion: "latest"
description: Helm Chart for installing storage service.
version: 0.1.0
type: application
# This file contains the essential configs for the osdu on azure helm chart
global:
# Service(s) Replica Count
replicaCount: 2
################################################################################
# Specify the Gitlab branch being used for image creation
# ie: community.opengroup.org:5555/osdu/platform/system/storage/{{ .Values.global.branch }}/storage:latest
#
image:
repository: #{container-registry}#.azurecr.io
branch: #{ENVIRONMENT_NAME}#
tag: #{Build.SourceVersion}#
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ .Release.Name }}
namespace: osdu
spec:
replicas: {{ .Values.global.replicaCount }}
selector:
matchLabels:
app: {{ .Chart.Name }}
template:
metadata:
labels:
app: {{ .Chart.Name }}
aadpodidbinding: osdu-identity
spec:
volumes:
- name: azure-keyvault
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: azure-keyvault
containers:
- name: {{ .Chart.Name }}
image: {{ .Values.image.repository }}/{{ .Chart.Name }}-{{ .Values.image.branch }}:{{ .Values.image.tag | default .Chart.AppVersion }}
imagePullPolicy: Always
ports:
- containerPort: 80
readinessProbe:
httpGet:
path: /api/partition/v1/swagger-ui.html
port: 80
volumeMounts:
- name: azure-keyvault
mountPath: "/mnt/azure-keyvault"
readOnly: true
env:
- name: spring_application_name
value: partition
- name: server.servlet.contextPath
value: /api/partition/v1/
- name: server_port
value: "80"
- name: ACCEPT_HTTP # TEMPORARY UNTIL HTTPS
value: "true"
- name: KEYVAULT_URI
valueFrom:
configMapKeyRef:
name: osdu-svc-properties
key: ENV_KEYVAULT
- name: AZURE_TENANT_ID
valueFrom:
secretKeyRef:
name: active-directory
key: tenantid
- name: AZURE_CLIENT_ID
valueFrom:
secretKeyRef:
name: active-directory
key: principal-clientid
- name: AZURE_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: active-directory
key: principal-clientpassword
- name: appinsights_key
valueFrom:
secretKeyRef:
name: central-logging
key: appinsights
- name: aad_client_id
valueFrom:
secretKeyRef:
name: active-directory
key: application-appid
- name: azure_activedirectory_AppIdUri
value: "api://$(aad_client_id)"
- name: azure_activedirectory_session_stateless
value: "true"
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: v1
kind: Service
metadata:
name: {{ .Chart.Name }}
namespace: osdu
spec:
type: ClusterIP
ports:
- protocol: TCP
port: 80
targetPort: 80
selector:
app: {{ .Chart.Name }}
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
global:
replicaCount: 1
image:
repository: community.opengroup.org:5555/osdu/platform/system/partition
branch: master
tag: latest
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
trigger:
batch: true
branches:
include:
- master
paths:
exclude:
- /**/*.md
- .gitignore
- /docs
- /provider/partition-aws
resources:
repositories:
- repository: FluxRepo
type: git
name: k8-gitops-manifests
- repository: TemplateRepo
type: git
name: infra-azure-provisioning
variables:
- group: 'Azure - OSDU'
- group: 'Azure - OSDU Secrets'
- name: serviceName
value: "partition"
- name: chartPath
value: "devops/azure/chart"
- name: valuesFile
value: "devops/azure/chart/helm-config.yaml"
- name: 'MANIFEST_REPO'
value: $[ resources.repositories['FluxRepo'].name ]
stages:
- template: /devops/build-stage.yml@TemplateRepo
parameters:
mavenGoal: 'package'
mavenPublishJUnitResults: true
serviceCoreMavenOptions: '-P partition-core'
mavenOptions: '-P partition-azure'
copyFileContents: |
pom.xml
provider/partition-azure/maven/settings.xml
provider/partition-azure/pom.xml
provider/partition-azure/target/*-spring-boot.jar
copyFileContentsToFlatten: ''
mavenSettingsFile: './maven/settings.xml'
serviceBase: ${{ variables.serviceName }}
testingRootFolder: 'testing'
chartPath: ${{ variables.chartPath }}
- template: /devops/deploy-stages.yml@TemplateRepo
parameters:
serviceName: ${{ variables.serviceName }}
chartPath: ${{ variables.chartPath }}
valuesFile: ${{ variables.valuesFile }}
testCoreMavenPomFile: 'testing/partition-test-core/pom.xml'
testCoreMavenOptions: '--settings $(System.DefaultWorkingDirectory)/drop/deploy/testing/maven/settings.xml'
skipDeploy: ${{ variables.SKIP_DEPLOY }}
skipTest: ${{ variables.SKIP_TESTS }}
providers:
- name: Azure
environments: ['dev']
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
trigger:
batch: true
branches:
include:
- master
paths:
exclude:
- /**/*.md
- .gitignore
- /docs
- /provider/partition-aws
resources:
repositories:
- repository: FluxRepo
type: git
name: k8-gitops-manifests
- repository: TemplateRepo
type: git
name: infra-azure-provisioning
variables:
- group: 'Azure - OSDU'
- group: 'Azure - OSDU Secrets'
- name: serviceName
value: "partition"
- name: chartPath
value: "devops/azure/chart"
- name: valuesFile
value: "devops/azure/chart/helm-config.yaml"
- name: 'MANIFEST_REPO'
value: $[ resources.repositories['FluxRepo'].name ]
stages:
- template: /devops/build-stage.yml@TemplateRepo
parameters:
mavenGoal: 'package'
mavenPublishJUnitResults: true
serviceCoreMavenOptions: '-P partition-core'
mavenOptions: '-P partition-azure'
copyFileContents: |
pom.xml
provider/partition-azure/maven/settings.xml
provider/partition-azure/pom.xml
provider/partition-azure/target/*-spring-boot.jar
copyFileContentsToFlatten: ''
mavenSettingsFile: './maven/settings.xml'
serviceBase: ${{ variables.serviceName }}
testingRootFolder: 'testing'
chartPath: ${{ variables.chartPath }}
- template: /devops/deploy-stages.yml@TemplateRepo
parameters:
serviceName: ${{ variables.serviceName }}
chartPath: ${{ variables.chartPath }}
valuesFile: ${{ variables.valuesFile }}
testCoreMavenPomFile: 'testing/partition-test-core/pom.xml'
testCoreMavenOptions: '--settings $(System.DefaultWorkingDirectory)/drop/deploy/testing/maven/settings.xml'
skipDeploy: ${{ variables.SKIP_DEPLOY }}
skipTest: ${{ variables.SKIP_TESTS }}
providers:
- name: Azure
environments: ['demo']
---
# Source: partition/templates/service.yaml
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: v1
kind: Service
metadata:
name: partition
namespace: osdu
spec:
type: ClusterIP
ports:
- protocol: TCP
port: 80
targetPort: 80
selector:
app: partition
---
# Source: partition/templates/deployment.yaml
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: apps/v1
kind: Deployment
metadata:
name: osdu-gitlab-partition
namespace: osdu
spec:
replicas: 1
selector:
matchLabels:
app: partition
template:
metadata:
labels:
app: partition
aadpodidbinding: osdu-identity
spec:
volumes:
- name: azure-keyvault
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: azure-keyvault
containers:
- name: partition
image: community.opengroup.org:5555/osdu/platform/system/partition/partition-trusted-azure-pipeline:latest
imagePullPolicy: Always
ports:
- containerPort: 80
readinessProbe:
httpGet:
path: /api/partition/v1/swagger-ui.html
port: 80
volumeMounts:
- name: azure-keyvault
mountPath: "/mnt/azure-keyvault"
readOnly: true
env:
- name: spring_application_name
value: partition
- name: server.servlet.contextPath
value: /api/partition/v1/
- name: server_port
value: "80"
- name: ACCEPT_HTTP # TEMPORARY UNTIL HTTPS
value: "true"
- name: KEYVAULT_URI
valueFrom:
configMapKeyRef:
name: osdu-svc-properties
key: ENV_KEYVAULT
- name: AZURE_TENANT_ID
valueFrom:
secretKeyRef:
name: active-directory
key: tenantid
- name: AZURE_CLIENT_ID
valueFrom:
secretKeyRef:
name: active-directory
key: principal-clientid
- name: AZURE_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: active-directory
key: principal-clientpassword
- name: appinsights_key
valueFrom:
secretKeyRef:
name: central-logging
key: appinsights
- name: aad_client_id
valueFrom:
secretKeyRef:
name: active-directory
key: application-appid
- name: azure_activedirectory_AppIdUri
value: "api://$(aad_client_id)"
- name: azure_activedirectory_session_stateless
value: "true"
......@@ -84,7 +84,9 @@ paths:
'200':
description: OK
schema:
$ref: '#/definitions/PartitionInfo'
type: object
additionalProperties:
"$ref": "#/definitions/Property"
'401':
description: Unauthorized
'403':
......@@ -161,13 +163,26 @@ definitions:
PartitionInfo:
type: object
properties:
labels:
properties:
type: object
description: 'Free form key value pair object for any data partition specific values'
additionalProperties:
"$ref": "#/definitions/Property"
example:
id: 'common'
compliance-ruleset: 'shared'
elastic-username: 'elastic'
cosmos-endpoint: 'https://ado-dev-n-abc123-cosmosdb.documents.azure.com:443/'
elastic-endpoint: 'https://partition-dev.evd.ece-osdu.cloud.osdu-ds.com:9243'
storage-account-name: 'myStorageAccount'
\ No newline at end of file
properties:
compliance-ruleset:
sensitive: false
value: 'shared'
elastic-endpoint:
sensitive: true
value: 'elastic-endpoint'
cosmos-connection:
sensitive: true
value: 'cosmos-connection'
Property:
type: object
properties:
sensitive:
type: boolean
value:
type: object
\ No newline at end of file
......@@ -49,13 +49,34 @@ A sample output is shown below.
```
{
"elastic-username": "elastic",
"elastic-endpoint": "test-elastic-endpoint",
"compliance-ruleset": "shared",
"storage-account-name": "sampleAcc",
"elastic-password": "test-password",
"storage-account-key": "sampleKey",
"id": "common"