diff --git a/.fossa.yml b/.fossa.yml
index 2a2c0a8a01f4dea0d6b188ffb214f391f6417fde..7700b8de0384fdb0e24e3b63f298bc513e18f1a9 100644
--- a/.fossa.yml
+++ b/.fossa.yml
@@ -8,27 +8,27 @@ cli:
project: Partition
analyze:
modules:
- - name: partition
- type: mvn
- target: pom.xml
- path: .
- - name: partition-core
- type: mvn
- target: partition-core/pom.xml
- path: .
- - name: partition-azure
- type: mvn
- target: provider/partition-azure/pom.xml
- path: .
- - name: partition-aws
- type: mvn
- target: provider/partition-aws/pom.xml
- path: .
- - name: partition-ibm
- type: mvn
- target: provider/partition-ibm/pom.xml
- path: .
- - name: partition-gcp
- type: mvn
- target: provider/partition-gcp/pom.xml
- path: .
+ - name: partition
+ type: mvn
+ target: pom.xml
+ path: .
+ - name: partition-core
+ type: mvn
+ target: partition-core/pom.xml
+ path: .
+ - name: partition-azure
+ type: mvn
+ target: provider/partition-azure/pom.xml
+ path: .
+ - name: partition-aws
+ type: mvn
+ target: provider/partition-aws/pom.xml
+ path: .
+ - name: partition-ibm
+ type: mvn
+ target: provider/partition-ibm/pom.xml
+ path: .
+ - name: partition-gcp
+ type: mvn
+ target: provider/partition-gc/pom.xml
+ path: .
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3d1e72981c6f88d3fe7d015695273e8a7c058346..19662940b73721eee0655db25806777c00709eca 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -17,9 +17,6 @@ variables:
IBM_HELM_CONFIG_PATH: devops/ibm/ibm-partition-config
IBM_HELM_DEPLOY_PATH: devops/ibm/ibm-partition-deploy
- # FIXME remove when all services are migrated to a single helm
- OSDU_GCP_ENABLE_HELM_CONFIG: "false"
-
include:
- project: "osdu/platform/ci-cd-pipelines"
file: "standard-setup.yml"
@@ -46,9 +43,9 @@ include:
file: "cloud-providers/ibm.yml"
- project: "osdu/platform/ci-cd-pipelines"
- file: "cloud-providers/osdu-gcp-global.yml"
+ file: "cloud-providers/gc-global.yml"
- - local: "devops/gcp/pipeline/override-stages.yml"
+ - local: "devops/gc/pipeline/override-stages.yml"
- project: "osdu/platform/ci-cd-pipelines"
file: "publishing/pages.yml"
diff --git a/NOTICE b/NOTICE
index 81e986f45f4db50e197089c3df71407c22a61188..d7b0026901f9c96a25201c892b542a8dc43f2cd0 100644
--- a/NOTICE
+++ b/NOTICE
@@ -352,7 +352,7 @@ The following software have components provided under the terms of this license:
- Apache Maven Wagon :: Providers :: SSH External Provider (from https://repo1.maven.org/maven2/org/apache/maven/wagon/wagon-ssh-external)
- Apache Maven Wagon :: Providers :: SSH Provider (from https://repo1.maven.org/maven2/org/apache/maven/wagon/wagon-ssh)
- Apache Velocity (from http://velocity.apache.org/engine/devel/, http://velocity.apache.org/engine/releases/velocity-1.6.1/, http://velocity.apache.org/engine/releases/velocity-1.6.2/)
-- AssertJ fluent assertions (from ${project.parent.url}#${project.artifactId}, https://repo1.maven.org/maven2/org/assertj/assertj-core)
+- AssertJ Core (from ${project.organization.url}#${project.artifactId})
- Asynchronous Http Client (from https://repo1.maven.org/maven2/org/asynchttpclient/async-http-client)
- Asynchronous Http Client Netty Utils (from https://repo1.maven.org/maven2/org/asynchttpclient/async-http-client-netty-utils)
- AutoValue Annotations (from https://github.com/google/auto/tree/master/value, https://repo1.maven.org/maven2/com/google/auto/value/auto-value-annotations)
@@ -481,7 +481,6 @@ The following software have components provided under the terms of this license:
- Microsoft Azure Java Core Library (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java)
-- Microsoft Azure Storage Client SDK (from https://github.com/Azure/azure-storage-java)
- Mockito (from http://mockito.org, https://github.com/mockito/mockito)
- Mojo's Maven plugin for Cobertura (from http://mojo.codehaus.org/cobertura-maven-plugin/)
- MongoDB Driver (from https://www.mongodb.com/)
@@ -622,8 +621,8 @@ The following software have components provided under the terms of this license:
- proto-google-cloud-iamcredentials-v1 (from https://github.com/googleapis/google-cloud-java, https://github.com/googleapis/java-iamcredentials/proto-google-cloud-iamcredentials-v1, https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-cloud-iamcredentials-v1)
- proto-google-cloud-logging-v2 (from https://github.com/googleapis/java-logging/proto-google-cloud-logging-v2, https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-cloud-logging-v2)
- proto-google-cloud-pubsub-v1 (from https://github.com/googleapis/googleapis, https://github.com/googleapis/java-pubsub/proto-google-cloud-pubsub-v1)
-- proto-google-common-protos (from https://github.com/googleapis/api-client-staging, https://github.com/googleapis/googleapis, https://github.com/googleapis/java-iam/proto-google-common-protos)
-- proto-google-iam-v1 (from https://github.com/googleapis/googleapis, https://github.com/googleapis/java-iam/proto-google-iam-v1)
+- proto-google-common-protos (from https://github.com/googleapis/api-client-staging, https://github.com/googleapis/gapic-generator-java, https://github.com/googleapis/googleapis, https://github.com/googleapis/java-iam/proto-google-common-protos)
+- proto-google-iam-v1 (from https://github.com/googleapis/gapic-generator-java, https://github.com/googleapis/googleapis, https://github.com/googleapis/java-iam/proto-google-iam-v1)
- resilience4j (from https://github.com/resilience4j/resilience4j, https://resilience4j.readme.io, ttps://resilience4j.readme.io)
- spring-security-crypto (from http://spring.io/spring-security, https://spring.io/projects/spring-security, https://spring.io/spring-security)
- spring-security-oauth2-client (from http://spring.io/spring-security, https://spring.io/projects/spring-security, https://spring.io/spring-security)
@@ -647,7 +646,7 @@ BSD-2-Clause
========================================================================
The following software have components provided under the terms of this license:
-- API Common (from https://github.com/googleapis, https://github.com/googleapis/api-common-java)
+- API Common (from https://github.com/googleapis, https://github.com/googleapis/api-common-java, https://repo1.maven.org/maven2/com/google/api/api-common)
- GAX (Google Api eXtensions) for Java (Core) (from https://repo1.maven.org/maven2/com/google/api/gax)
- GAX (Google Api eXtensions) for Java (HTTP JSON) (from https://repo1.maven.org/maven2/com/google/api/gax-httpjson)
- GAX (Google Api eXtensions) for Java (gRPC) (from https://repo1.maven.org/maven2/com/google/api/gax-grpc)
@@ -667,7 +666,7 @@ BSD-3-Clause
========================================================================
The following software have components provided under the terms of this license:
-- API Common (from https://github.com/googleapis, https://github.com/googleapis/api-common-java)
+- API Common (from https://github.com/googleapis, https://github.com/googleapis/api-common-java, https://repo1.maven.org/maven2/com/google/api/api-common)
- ASM Analysis (from http://asm.ow2.io/)
- ASM Commons (from http://asm.ow2.io/, https://repo1.maven.org/maven2/org/ow2/asm/asm-commons)
- ASM Core (from http://asm.ow2.io/, http://asm.ow2.org/)
@@ -685,7 +684,7 @@ The following software have components provided under the terms of this license:
- Hamcrest Core (from http://hamcrest.org/, http://hamcrest.org/JavaHamcrest/, https://repo1.maven.org/maven2/org/hamcrest/hamcrest-core)
- JSch (from http://www.jcraft.com/jsch/)
- Jackson module: Afterburner (from http://wiki.fasterxml.com/JacksonHome, https://github.com/FasterXML/jackson-modules-base)
-- Jakarta Activation API (from https://github.com/eclipse-ee4j/jaf, https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api)
+- Jakarta Activation API (from https://github.com/eclipse-ee4j/jaf, https://github.com/jakartaee/jaf-api, https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api)
- Jakarta Annotations API (from https://projects.eclipse.org/projects/ee4j.ca)
- Jakarta XML Binding API (from https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api, https://repo1.maven.org/maven2/org/jboss/spec/javax/xml/bind/jboss-jaxb-api_2.3_spec)
- Microsoft Application Insights Java Agent (from https://github.com/Microsoft/ApplicationInsights-Java)
@@ -760,7 +759,7 @@ The following software have components provided under the terms of this license:
- Apache Log4j Core (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core)
- Apache Maven Reporting Implementation (from https://repo1.maven.org/maven2/org/apache/maven/reporting/maven-reporting-impl)
-- Jakarta Activation API (from https://github.com/eclipse-ee4j/jaf, https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api)
+- Jakarta Activation API (from https://github.com/eclipse-ee4j/jaf, https://github.com/jakartaee/jaf-api, https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api)
- Java Architecture for XML Binding (from http://jaxb.java.net/, https://repo1.maven.org/maven2/javax/xml/bind/jaxb-api)
- JavaBeans Activation Framework (from https://repo1.maven.org/maven2/com/sun/activation/javax.activation)
- Servlet Specification 2.5 API (from http://jetty.mortbay.org, https://repo1.maven.org/maven2/org/mortbay/jetty/servlet-api-2.5)
@@ -796,7 +795,7 @@ The following software have components provided under the terms of this license:
- JUnit Jupiter Params (from http://junit.org/junit5/, https://junit.org/junit5/)
- JUnit Platform Commons (from http://junit.org/junit5/, https://junit.org/junit5/)
- JUnit Platform Engine API (from http://junit.org/junit5/, https://junit.org/junit5/)
-- Jakarta Activation API (from https://github.com/eclipse-ee4j/jaf, https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api)
+- Jakarta Activation API (from https://github.com/eclipse-ee4j/jaf, https://github.com/jakartaee/jaf-api, https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api)
- Jakarta Annotations API (from https://projects.eclipse.org/projects/ee4j.ca)
- Jakarta Bean Validation API (from https://beanvalidation.org)
- Jakarta XML Binding API (from https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api, https://repo1.maven.org/maven2/org/jboss/spec/javax/xml/bind/jboss-jaxb-api_2.3_spec)
@@ -822,7 +821,7 @@ The following software have components provided under the terms of this license:
- JUnit Jupiter Params (from http://junit.org/junit5/, https://junit.org/junit5/)
- JUnit Platform Commons (from http://junit.org/junit5/, https://junit.org/junit5/)
- JUnit Platform Engine API (from http://junit.org/junit5/, https://junit.org/junit5/)
-- Jakarta Activation API (from https://github.com/eclipse-ee4j/jaf, https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api)
+- Jakarta Activation API (from https://github.com/eclipse-ee4j/jaf, https://github.com/jakartaee/jaf-api, https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api)
- Jakarta Annotations API (from https://projects.eclipse.org/projects/ee4j.ca)
- Jakarta Bean Validation API (from https://beanvalidation.org)
- Jakarta XML Binding API (from https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api, https://repo1.maven.org/maven2/org/jboss/spec/javax/xml/bind/jboss-jaxb-api_2.3_spec)
@@ -860,7 +859,7 @@ GPL-2.0-with-classpath-exception
The following software have components provided under the terms of this license:
- Checker Qual (from https://checkerframework.org)
-- Jakarta Activation API (from https://github.com/eclipse-ee4j/jaf, https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api)
+- Jakarta Activation API (from https://github.com/eclipse-ee4j/jaf, https://github.com/jakartaee/jaf-api, https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api)
- Jakarta Annotations API (from https://projects.eclipse.org/projects/ee4j.ca)
- Jakarta Bean Validation API (from https://beanvalidation.org)
- Jakarta XML Binding API (from https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api, https://repo1.maven.org/maven2/org/jboss/spec/javax/xml/bind/jboss-jaxb-api_2.3_spec)
@@ -981,10 +980,10 @@ The following software have components provided under the terms of this license:
- Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure SDK annotations (from https://github.com/Microsoft/java-api-annotations)
- Microsoft Azure SDK for EventGrid Management (from https://github.com/Azure/azure-sdk-for-java)
-- Microsoft Azure SDK for Key Vault Core (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure SDK for Service Bus (from https://github.com/Azure/azure-sdk-for-java, https://github.com/Azure/azure-service-bus-java)
- Microsoft Azure SDK for eventgrid (from https://github.com/Azure/azure-sdk-for-java)
+- Microsoft Azure client library for Azure Tables (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure client library for Blob Storage (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure client library for File Storage Data Lake (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure client library for Identity (from https://github.com/Azure/azure-sdk-for-java)
@@ -1107,7 +1106,7 @@ unknown
========================================================================
The following software have components provided under the terms of this license:
-- Jakarta Activation API (from https://github.com/eclipse-ee4j/jaf, https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api)
+- Jakarta Activation API (from https://github.com/eclipse-ee4j/jaf, https://github.com/jakartaee/jaf-api, https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api)
- xml-apis (from https://repo1.maven.org/maven2/xml-apis/xml-apis)
========================================================================
diff --git a/README.md b/README.md
index 211aa1c5b1eb7385fb252fde7e93355f56bd55ef..52ac8e9c66e5801b412eaa4ee36af6048b8a8c52 100644
--- a/README.md
+++ b/README.md
@@ -1,24 +1,32 @@
-# Introduction
+# Introduction
The Partition service is responsible for creating and retrieving partition specific properties on behalf of other services whether they are secret values or not. It is a Maven multi-module project with each cloud implementation placed in its submodule.
-
+
## Running Locally - AWS
+
Instructions for running the AWS implementation locally can be found [here](./provider/partition-aws/README.md)
+
## Running Locally - Azure
+
Instructions for running the Azure implementation locally can be found [here](./provider/partition-azure/README.md)
+
## Running Locally - Google Cloud
-Instructions for running the Google Cloud implementation locally can be found [here](./provider/partition-gcp/README.md)
+
+Instructions for running the Google Cloud implementation locally can be found [here](./provider/partition-gc/README.md)
+
## Running Locally - IBM
## Running Integration Tests
+
Instructions for running the integration tests can be found [here](./testing/README.md)
## License
+
Copyright 2017-2020, Schlumberger
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
+You may obtain a copy of the License at
[http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0)
diff --git a/devops/gcp/deploy/Chart.yaml b/devops/gc/deploy/Chart.yaml
similarity index 95%
rename from devops/gcp/deploy/Chart.yaml
rename to devops/gc/deploy/Chart.yaml
index 8db1c028981b97515fa192f90f0dcb87ea893e8b..a86d5fff32665a4e4eb88c985fc34fc305c89bb4 100644
--- a/devops/gcp/deploy/Chart.yaml
+++ b/devops/gc/deploy/Chart.yaml
@@ -1,5 +1,5 @@
apiVersion: v2
-name: gcp-partition-deploy
+name: gc-partition-deploy
description: A Helm chart for Kubernetes
# A chart can be either an 'application' or a 'library' chart.
#
@@ -17,4 +17,4 @@ version: 0.1.0
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
-appVersion: 1.16.0
+appVersion: 1.19.0
diff --git a/devops/gcp/deploy/README.md b/devops/gc/deploy/README.md
similarity index 56%
rename from devops/gcp/deploy/README.md
rename to devops/gc/deploy/README.md
index 01ef2b66d613e8a86d56e55806a58e5c3ef80ccf..c94fb9d6ecb41702e3ead8eccca1ead37b616a83 100644
--- a/devops/gcp/deploy/README.md
+++ b/devops/gc/deploy/README.md
@@ -96,50 +96,59 @@ First you need to set variables in **values.yaml** file using any code editor. S
| Name | Description | Type | Default |Required |
|------|-------------|------|---------|---------|
-**logLevel** | logging level | string | INFO | yes
-**springProfilesActive** | active spring profile | string | gcp | yes
-**projectId** | your Google Cloud project id | string | - | only in case of Google Cloud installation
-**dataProjectId** | in case of multiproject cloud installation (services and data stored in different project) the name of data project | string | - | only in case of multiproject installation
-**partitionAdminAccounts** | admin accounts validated by partition service | string | - | yes
-**serviceAccountTail** | ending of Google Cloud service account | string | .iam.gserviceaccount.com | yes
-**partitionName** | partition host | string | partition | yes
-**partitionNamespace** | datastore namespace where partition will store the data | string | partition | yes
-**dataPartitionId** | data partition id | string | - | yes
-**datafierSa** | datafier service account | string | datafier | yes
-**bucketPrefix** | minio bucket name prefix | string | refi | only in case of Reference installation when _springProfilesActive_ is set to "_anthos_"
-**minioExternalEndpoint** | api url for external minio, if external minio is configured - this value will be set for MINIO_ENDPOINT and FILE_MINIO_ENDPOINT in bootstrap configmap | string | - | no
+**data.logLevel** | logging level | string | INFO | yes
+**data.springProfilesActive** | active spring profile | string | gcp | yes
+**data.projectId** | your Google Cloud project id | string | - | only in case of Google Cloud installation
+**data.dataProjectId** | in case of multiproject cloud installation (services and data stored in different project) the name of data project | string | - | only in case of multiproject installation
+**data.partitionName** | partition host | string | partition | yes
+**data.partitionNamespace** | datastore namespace where partition will store the data | string | partition | yes
+**data.dataPartitionId** | data partition id | string | - | yes
+**data.datafierSa** | datafier service account | string | datafier | yes
+**data.bucketPrefix** | minio bucket name prefix | string | refi | only in case of Reference installation when _springProfilesActive_ is set to "_anthos_"
+**data.minioExternalEndpoint** | api url for external minio, if external minio is configured - this value will be set for MINIO_ENDPOINT and FILE_MINIO_ENDPOINT in bootstrap configmap | string | - | no
### Deployment variables
| Name | Description | Type | Default |Required |
|------|-------------|------|---------|---------|
-**requestsCpu** | amount of requests CPU | string | 0.1 | yes
-**requestsMemory** | amount of requests memory | string | 260M | yes
-**limitsCpu** | CPU limit | string | 1 | yes
-**limitsMemory** | memory limit | string | 1G | yes
-**serviceAccountName** | name of your service account | string | partition | yes
-**image** | path to the image in a registry | string | - | yes
-**imagePullPolicy** | when to pull the image | string | IfNotPresent | yes
-**bootstrapImage** | name of the bootstrap image | string | - | yes
+**data.requestsCpu** | amount of requests CPU | string | 40m | yes
+**data.requestsMemory** | amount of requests memory | string | 256Mi | yes
+**data.limitsCpu** | CPU limit | string | 500m | yes
+**data.imitsMemory** | memory limit | string | 1G | yes
+**data.serviceAccountName** | name of your service account | string | partition | yes
+**data.image** | path to the image in a registry | string | - | yes
+**data.imagePullPolicy** | when to pull the image | string | IfNotPresent | yes
+**data.bootstrapImage** | name of the bootstrap image | string | - | yes
### Configuration variables
| Name | Description | Type | Default |Required |
|------|-------------|------|---------|---------|
-**appName** | name of the app | string | partition | yes
-**configmap** | configmap to be used | string | partition-config | yes
-**domain** | your domain | string | - | yes
-**onPremEnabled** | whether on-prem is enabled | boolean | false | yes
-**secret** | secret for postgres | string | partition-postgres-secret | yes
-**mtlsMode** | MTLS mode | string | STRICT | yes
-**realm** | realm in keycloak | string | osdu | yes
+**conf.appName** | name of the app | string | partition | yes
+**conf.configmap** | configmap to be used | string | partition-config | yes
+**conf.domain** | your domain | string | - | yes
+**conf.onPremEnabled** | whether on-prem is enabled | boolean | false | yes
+**conf.secret** | secret for postgres | string | partition-postgres-secret | yes
+**auth.realm** | realm in keycloak | string | osdu | yes
+
+### ISTIO variables
+
+| Name | Description | Type | Default |Required |
+|------|-------------|------|---------|---------|
+**istio.proxyCPU** | CPU request for Envoy sidecars | string | 10m | yes
+**istio.proxyCPULimit** | CPU limit for Envoy sidecars | string | 500m | yes
+**istio.proxyMemory** | memory request for Envoy sidecars | string | 32Mi | yes
+**istio.proxyMemoryLimit** | memory limit for Envoy sidecars | string | 512Mi | yes
+**istio.bootstrapProxyCPU** | CPU request for Envoy sidecars | string | 10m | yes
+**istio.bootstrapProxyCPULimit** | CPU limit for Envoy sidecars | string | 100m | yes
+**istio.sidecarInject** | whether to inject sidecar | boolean | true | yes
### Install the helm chart
Run this command from within this directory:
```console
-helm install gcp-partition-deploy .
+helm install gc-partition-deploy .
```
## Uninstalling the Chart
@@ -147,7 +156,7 @@ helm install gcp-partition-deploy .
To uninstall the helm deployment:
```console
-helm uninstall gcp-partition-deploy
+helm uninstall gc-partition-deploy
```
To delete secrets and PVCs:
diff --git a/devops/gcp/deploy/templates/authorization-policy.yaml b/devops/gc/deploy/templates/authorization-policy.yaml
similarity index 100%
rename from devops/gcp/deploy/templates/authorization-policy.yaml
rename to devops/gc/deploy/templates/authorization-policy.yaml
diff --git a/devops/gcp/deploy/templates/configmap-bootstrap.yaml b/devops/gc/deploy/templates/configmap-bootstrap.yaml
similarity index 100%
rename from devops/gcp/deploy/templates/configmap-bootstrap.yaml
rename to devops/gc/deploy/templates/configmap-bootstrap.yaml
diff --git a/devops/gcp/deploy/templates/configmap.yaml b/devops/gc/deploy/templates/configmap.yaml
similarity index 86%
rename from devops/gcp/deploy/templates/configmap.yaml
rename to devops/gc/deploy/templates/configmap.yaml
index a37b4d672e7ed1a559eb05892bab2131533138d1..46b1a22831846f8b729bc1c0f849e269e9ea7466 100644
--- a/devops/gcp/deploy/templates/configmap.yaml
+++ b/devops/gc/deploy/templates/configmap.yaml
@@ -10,6 +10,5 @@ data:
SPRING_PROFILES_ACTIVE: {{ .Values.data.springProfilesActive | quote }}
{{- if not .Values.conf.onPremEnabled }}
GOOGLE_CLOUD_PROJECT: {{ .Values.data.projectId | quote }}
- PARTITION_ADMIN_ACCOUNTS: {{ .Values.data.partitionAdminAccounts | quote }}
PARTITION_NAMESPACE: {{ .Values.data.partitionNamespace | quote }}
{{- end }}
diff --git a/devops/gcp/deploy/templates/deploy-bootstrap.yaml b/devops/gc/deploy/templates/deploy-bootstrap.yaml
similarity index 94%
rename from devops/gcp/deploy/templates/deploy-bootstrap.yaml
rename to devops/gc/deploy/templates/deploy-bootstrap.yaml
index 2fce369da2e6b9c2648db6ff14168d9992eaf3f2..d73892e207baf7ae06c03664ff08ec0eee4ede0c 100644
--- a/devops/gcp/deploy/templates/deploy-bootstrap.yaml
+++ b/devops/gc/deploy/templates/deploy-bootstrap.yaml
@@ -12,6 +12,7 @@ spec:
metadata:
labels:
app: {{ printf "%s-bootstrap" .Values.conf.appName | quote }}
+ sidecar.istio.io/inject: {{ .Values.istio.sidecarInject | quote }}
annotations:
rollme: {{ randAlphaNum 5 | quote }}
sidecar.istio.io/proxyCPU: {{ .Values.istio.bootstrapProxyCPU | quote }}
diff --git a/devops/gcp/deploy/templates/deploy.yaml b/devops/gc/deploy/templates/deploy.yaml
similarity index 95%
rename from devops/gcp/deploy/templates/deploy.yaml
rename to devops/gc/deploy/templates/deploy.yaml
index 7199c830fa151e6daf3e55525e578ab81f7ec179..8ef1f1360817aa9e9189b1cb43e9c1090aab356a 100644
--- a/devops/gcp/deploy/templates/deploy.yaml
+++ b/devops/gc/deploy/templates/deploy.yaml
@@ -12,6 +12,7 @@ spec:
metadata:
labels:
app: {{ .Values.conf.appName | quote }}
+ sidecar.istio.io/inject: {{ .Values.istio.sidecarInject | quote }}
annotations:
rollme: {{ randAlphaNum 5 | quote }}
sidecar.istio.io/proxyCPU: {{ .Values.istio.proxyCPU | quote }}
diff --git a/devops/gcp/deploy/templates/service-account.yaml b/devops/gc/deploy/templates/service-account.yaml
similarity index 100%
rename from devops/gcp/deploy/templates/service-account.yaml
rename to devops/gc/deploy/templates/service-account.yaml
diff --git a/devops/gcp/deploy/templates/service.yaml b/devops/gc/deploy/templates/service.yaml
similarity index 100%
rename from devops/gcp/deploy/templates/service.yaml
rename to devops/gc/deploy/templates/service.yaml
diff --git a/devops/gcp/deploy/templates/virtual-service.yaml b/devops/gc/deploy/templates/virtual-service.yaml
similarity index 100%
rename from devops/gcp/deploy/templates/virtual-service.yaml
rename to devops/gc/deploy/templates/virtual-service.yaml
diff --git a/devops/gcp/deploy/values.yaml b/devops/gc/deploy/values.yaml
similarity index 83%
rename from devops/gcp/deploy/values.yaml
rename to devops/gc/deploy/values.yaml
index fbba5075b0291bf2f5845a5e650cb3474cf2ccc9..47ea2ed6132ce09d55af1f0065cbfd8e2017290f 100644
--- a/devops/gcp/deploy/values.yaml
+++ b/devops/gc/deploy/values.yaml
@@ -6,16 +6,15 @@ data:
dataProjectId: ""
dataPartitionId: ""
dataPartitionIdList: []
- partitionAdminAccounts: ""
partitionName: "partition"
partitionNamespace: "partition"
datafierSa: "datafier"
bucketPrefix: "refi"
minioExternalEndpoint: "" # use only if external minio is configured
# deployments
- requestsCpu: "0.1"
- requestsMemory: "260M"
- limitsCpu: "1"
+ requestsCpu: "40m"
+ requestsMemory: "256Mi"
+ limitsCpu: "500m"
limitsMemory: "1G"
serviceAccountName: "partition"
imagePullPolicy: "IfNotPresent"
@@ -24,20 +23,17 @@ data:
# bootstrap onprem
conf:
appName: "partition"
- cicdEnabled: false
configmap: "partition-config"
domain: ""
onPremEnabled: false
- publicAvailable: false
secret: "partition-postgres-secret"
-namespacePolicy:
- mtlsMode: STRICT
auth:
realm: "osdu"
istio:
- proxyCPU: "50m"
+ proxyCPU: "10m"
proxyCPULimit: "500m"
proxyMemory: "64Mi"
proxyMemoryLimit: "512Mi"
bootstrapProxyCPU: "10m"
bootstrapProxyCPULimit: "100m"
+ sidecarInject: "true"
diff --git a/devops/gc/pipeline/override-stages.yml b/devops/gc/pipeline/override-stages.yml
new file mode 100644
index 0000000000000000000000000000000000000000..b2b0a6fab76f9885ec88281eac13db46fcc6e806
--- /dev/null
+++ b/devops/gc/pipeline/override-stages.yml
@@ -0,0 +1,303 @@
+variables:
+ GC_ENABLE_BOOTSTRAP: "true"
+ GC_SERVICE: partition
+ GC_VENDOR: gc
+ GC_HELM_ENV_DIR: "devops/gc/tests/env"
+ ENV_BUILD_PATH: "devops/gc/tests/build/Dockerfile"
+ # FIXME remove when all services are migrated to a single helm
+ GC_ENABLE_HELM_CONFIG: "false"
+
+.common_cleanup:
+ script:
+ - git clone https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-gcp-provisioning.git
+ - cp infra-gcp-provisioning/tools/datastore-cleanup/* devops/gc/tests/clean_up/
+
+gc-containerize-bootstrap-env-gitlab:
+ stage: build
+ image: docker:19.03.15
+ tags: ["osdu-small"]
+ services:
+ - docker:20.10.7-dind
+ variables:
+ BUILD_BOOTSTRAP_PATH: "devops/gc/tests/build/Dockerfile"
+ script:
+ - export EXTRA_DOCKER_TAG=""
+ - >
+ if [[ "$CI_COMMIT_TAG" != "" ]];
+ then EXTRA_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/$IMAGE_BOOTSTRAP_NAME-env:$CI_COMMIT_TAG";
+ elif [[ "$CI_COMMIT_REF_NAME" = "$CI_DEFAULT_BRANCH" ]];
+ then EXTRA_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/$IMAGE_BOOTSTRAP_NAME-env:latest"; fi
+ - docker build -t $CI_REGISTRY_IMAGE/$IMAGE_BOOTSTRAP_NAME-env:$CI_COMMIT_SHORT_SHA $EXTRA_DOCKER_TAG -f $BUILD_BOOTSTRAP_PATH .
+ - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+ - docker push $CI_REGISTRY_IMAGE/$IMAGE_BOOTSTRAP_NAME-env
+ rules:
+ - if: "$GC == '1' && $GC_ENABLE_BOOTSTRAP == 'true'"
+ when: on_success
+
+gc-anthos-deploy-test-env: #infra deploy for on-prem install
+ environment:
+ name: gc
+ extends: .gc-anthos-variables
+ tags: ["osdu-small"]
+ image: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine
+ stage: deploy
+ needs: ["gc-containerize-bootstrap-env-gitlab", "compile-and-unit-test"]
+ retry: 1
+ variables:
+ GC_HELM_NAMESPACE: partition-int-test
+ script:
+ - gcloud auth activate-service-account --key-file $GC_ANTHOS_DEPLOY_FILE
+ - !reference [.common_config, script]
+ - export PASSWORD=$(</dev/urandom tr -dc _A-Z-a-z-0-9 | head -c10)
+ - cd $GC_HELM_ENV_DIR
+ - helm dependency update
+ - >
+ helm upgrade $GC_SERVICE-env .
+ --install
+ --create-namespace
+ --namespace=$GC_HELM_NAMESPACE
+ --wait
+ --history-max=3
+ --set postgresql.global.postgresql.auth.postgresPassword=$PASSWORD
+ --set bootstrap.postgres.image=$CI_REGISTRY_IMAGE/gc-bootstrap-$GC_SERVICE-env:$CI_COMMIT_SHORT_SHA
+ $GC_HELM_TIMEOUT
+ rules:
+ - if: "$CI_COMMIT_BRANCH =~ /^release/"
+ when: never
+ - if: "$CI_COMMIT_TAG"
+ when: never
+ - if: '$GC == "1"'
+ when: on_success
+
+gc-anthos-deploy-deployment: # reuse common deploy job for test deployment
+ needs: ["gc-anthos-deploy-test-env", "gc-containerize-gitlab"]
+ variables:
+ GC_HELM_NAMESPACE: partition-int-test
+ GC_TENANT: test
+ GC_HELM_SETS: >-
+ --set istio.sidecarInject=false
+
+gc-anthos-test:
+ variables:
+ GC_VENDOR: anthos
+ PARTITION_BASE_URL: https://test.ref.gcp.gnrg-osdu.projects.epam.com/
+
+gc-anthos-cleanup-env: # clean-up env after tests
+ environment:
+ name: gc
+ stage: cleanup
+ extends: .gc-anthos-variables
+ image: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine
+ needs: ["gc-anthos-test"]
+ tags: ["osdu-small"]
+ variables:
+ PARTITION_NAMESPACE: partition-test
+ GC_HELM_NAMESPACE: partition-int-test
+ script:
+ - gcloud auth activate-service-account --key-file $GC_ANTHOS_DEPLOY_FILE
+ - !reference [.common_config, script]
+ - export GOOGLE_APPLICATION_CREDENTIALS="$GC_ANTHOS_DEPLOY_FILE"
+ - !reference [.common_cleanup, script]
+ - chmod +x devops/gc/tests/clean_up/clean_up.sh && devops/gc/tests/clean_up/clean_up.sh --cluster
+ rules:
+ - if: "$CI_COMMIT_BRANCH =~ /^release/"
+ when: never
+ - if: "$CI_COMMIT_TAG"
+ when: never
+ - if: '$GC == "1"'
+ when: always
+
+gc-anthos-deploy: # verified deploy after tests
+ environment:
+ name: gc
+ extends: .gc-anthos-variables
+ tags: ["osdu-small"]
+ image: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine
+ stage: verified-deploy
+ needs: ["gc-anthos-test"]
+ retry: 1
+ script:
+ - gcloud auth activate-service-account --key-file $GC_ANTHOS_DEPLOY_FILE
+ - !reference [.common_config, script]
+ - >
+ helm upgrade $GC_SERVICE-deploy $GC_HELM_DEPLOYMENT_DIR
+ --install
+ --create-namespace
+ --namespace=$GC_HELM_NAMESPACE
+ --wait
+ --history-max=3
+ --set data.bootstrapImage=$CI_REGISTRY_IMAGE/gc-bootstrap-$GC_SERVICE:$CI_COMMIT_SHORT_SHA
+ --set data.image=$CI_REGISTRY_IMAGE/gc-$GC_SERVICE:$CI_COMMIT_SHORT_SHA
+ --set data.serviceAccountName=$GC_SERVICE
+ --set data.bucketPrefix=osdu-anthos
+ --set data.dataPartitionId=$GC_TENANT
+ --set data.logLevel=INFO
+ --set data.springProfilesActive=anthos
+ --set conf.onPremEnabled=true
+ --set conf.domain=$GC_DOMAIN
+ $GC_HELM_TIMEOUT
+ - !reference [.verify_deploy, script]
+ - !reference [.verify_bootstrap, script]
+ rules:
+ - if: "$CI_COMMIT_BRANCH =~ /^release/"
+ when: never
+ - if: "$CI_COMMIT_TAG"
+ when: never
+ - if: '$GC == "1"'
+ when: on_success
+
+gc-deploy-deployment: # reuse common deploy job for test deployment
+ variables:
+ GC_HELM_NAMESPACE: partition-int-test
+ GC_HELM_SETS: >-
+ --set data.dataPartitionId=test
+ --set data.partitionNamespace=partition-test
+ --set istio.sidecarInject=false
+
+gc-test:
+ variables:
+ CLIENT_TENANT: test
+ PARTITION_BASE_URL: https://test.community.gcp.gnrg-osdu.projects.epam.com/
+
+gc-cleanup-env: # clean-up env after tests
+ environment:
+ name: gc
+ stage: cleanup
+ extends: .gc-variables
+ image: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine
+ needs: ["gc-test"]
+ tags: ["osdu-small"]
+ variables:
+ PARTITION_NAMESPACE: partition-test
+ GC_HELM_NAMESPACE: partition-int-test
+ script:
+ - gcloud auth activate-service-account --key-file $GC_DEPLOY_FILE
+ - !reference [.common_config, script]
+ - export GOOGLE_APPLICATION_CREDENTIALS="$GC_DEPLOY_FILE"
+ - !reference [.common_cleanup, script]
+ - chmod +x devops/gc/tests/clean_up/clean_up.sh && devops/gc/tests/clean_up/clean_up.sh --all
+ rules:
+ - if: "$CI_COMMIT_BRANCH =~ /^release/"
+ when: never
+ - if: "$CI_COMMIT_TAG"
+ when: never
+ - if: '$GC == "1"'
+ when: always
+
+gc-deploy: # verified deploy after tests
+ environment:
+ name: gc
+ extends: .gc-variables
+ tags: ["osdu-small"]
+ image: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine
+ stage: verified-deploy
+ needs: ["gc-test"]
+ retry: 1
+ script:
+ - gcloud auth activate-service-account --key-file $GC_DEPLOY_FILE
+ - !reference [.common_config, script]
+ - >
+ helm upgrade $GC_SERVICE-deploy $GC_HELM_DEPLOYMENT_DIR
+ --install
+ --create-namespace
+ --namespace=$GC_HELM_NAMESPACE
+ --wait
+ --history-max=3
+ --set data.bootstrapImage=$CI_REGISTRY_IMAGE/gc-bootstrap-$GC_SERVICE:$CI_COMMIT_SHORT_SHA
+ --set data.image=$CI_REGISTRY_IMAGE/gc-$GC_SERVICE:$CI_COMMIT_SHORT_SHA
+ --set data.serviceAccountName=$GC_SERVICE-k8s
+ --set data.projectId=$GC_PROJECT
+ --set data.dataPartitionId=$GC_TENANT
+ --set data.logLevel=INFO
+ --set conf.domain=$GC_DOMAIN
+ $GC_HELM_TIMEOUT
+ - !reference [.verify_deploy, script]
+ - !reference [.verify_bootstrap, script]
+ rules:
+ - if: "$CI_COMMIT_BRANCH =~ /^release/"
+ when: never
+ - if: "$CI_COMMIT_TAG"
+ when: never
+ - if: '$GC == "1"'
+ when: on_success
+
+gc-dev2-deploy-deployment: # reuse common deploy job for test deployment
+ variables:
+ GC_HELM_NAMESPACE: partition-int-test
+ GC_HELM_SETS: >-
+ --set data.dataPartitionId=test
+ --set data.partitionNamespace=partition-test
+ --set istio.sidecarInject=false
+
+gc-dev2-test:
+ variables:
+ CLIENT_TENANT: test
+ PARTITION_BASE_URL: https://test.dev2.gcp.gnrg-osdu.projects.epam.com/
+
+gc-dev2-cleanup-env: # clean-up env after tests
+ environment:
+ name: gc
+ stage: cleanup
+ extends: .gc-dev2-variables
+ image: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine
+ needs: [gc-dev2-test]
+ tags: ["osdu-small"]
+ variables:
+ PARTITION_NAMESPACE: partition-test
+ GC_HELM_NAMESPACE: partition-int-test
+ script:
+ - gcloud auth activate-service-account --key-file $GC_DEPLOY_FILE
+ - !reference [.common_config, script]
+ - export GOOGLE_APPLICATION_CREDENTIALS="$GC_DEPLOY_FILE"
+ - !reference [.common_cleanup, script]
+ - chmod +x devops/gc/tests/clean_up/clean_up.sh && devops/gc/tests/clean_up/clean_up.sh --all
+ rules:
+ - if: '$GC == "1" && $CI_COMMIT_BRANCH =~ /^release/'
+ when: always
+ - if: '$GC == "1" && $CI_COMMIT_TAG'
+ when: always
+
+gc-dev2-deploy: # verified deploy after tests
+ tags: ["osdu-small"]
+ extends: .gc-dev2-variables
+ image: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine
+ stage: verified-deploy
+ needs: ["gc-containerize-gitlab", "gc-dev2-test"]
+ retry: 1
+ script:
+ - gcloud auth activate-service-account --key-file $GC_DEV2_DEPLOY_FILE
+ - !reference [.common_config, script]
+ - !reference [.define_version, script]
+ - !reference [.set_image_name, script]
+ - helm repo add $GC_SERVICE ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/helm/stable
+ - >
+ helm upgrade $GC_SERVICE-deploy $GC_SERVICE/gc-$GC_SERVICE-deploy
+ --version $VERSION
+ --install
+ --create-namespace
+ --namespace=$GC_HELM_NAMESPACE
+ --wait
+ --history-max=3
+ --set data.imagePullPolicy=Always
+ --set data.bootstrapImage=$CI_REGISTRY_IMAGE/gc-bootstrap-$GC_SERVICE:${CI_COMMIT_TAG:-$CI_COMMIT_SHORT_SHA}
+ --set data.image=$CI_REGISTRY_IMAGE/$IMAGE_NAME:${CI_COMMIT_TAG:-$CI_COMMIT_SHORT_SHA}
+ --set data.serviceAccountName=$GC_SERVICE-k8s
+ --set data.projectId=osdu-dev2
+ --set data.dataPartitionId=$GC_TENANT
+ --set data.logLevel=INFO
+ --set conf.domain=$GC_DOMAIN
+ $GC_HELM_TIMEOUT
+ - !reference [.verify_deploy, script]
+ - !reference [.verify_bootstrap, script]
+ rules:
+ - if: '$GC == "1" && $CI_COMMIT_BRANCH =~ /^release/'
+ when: on_success
+ - if: '$GC == "1" && $CI_COMMIT_TAG'
+ when: on_success
+
+gc-preship-deploy-deployment:
+ extends: .gc-preship-variables
+ tags: ["osdu-small"]
+ image: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine
+ stage: deploy_preship
+ needs: ["gc-dev2-deploy", "gc-dev2-test"]
diff --git a/devops/gcp/tests/build/Dockerfile b/devops/gc/tests/build/Dockerfile
similarity index 82%
rename from devops/gcp/tests/build/Dockerfile
rename to devops/gc/tests/build/Dockerfile
index 46883347de00e7e9a5a59bcd325781a31aef0617..7846085d35f6834fb6bc6cf5788b38c99c650a08 100644
--- a/devops/gcp/tests/build/Dockerfile
+++ b/devops/gc/tests/build/Dockerfile
@@ -3,7 +3,7 @@ FROM alpine:3.16
RUN apk update && \
apk -q add jq postgresql-client bash
-COPY ./devops/gcp/tests/build/ ./opt
+COPY ./devops/gc/tests/build/ ./opt
RUN chmod +x /opt/bootstrap.sh
WORKDIR /opt
diff --git a/devops/gcp/tests/build/bootstrap.sh b/devops/gc/tests/build/bootstrap.sh
similarity index 100%
rename from devops/gcp/tests/build/bootstrap.sh
rename to devops/gc/tests/build/bootstrap.sh
diff --git a/devops/gcp/tests/build/bootstrap.sql b/devops/gc/tests/build/bootstrap.sql
similarity index 100%
rename from devops/gcp/tests/build/bootstrap.sql
rename to devops/gc/tests/build/bootstrap.sql
diff --git a/devops/gcp/tests/build/validate-env.sh b/devops/gc/tests/build/validate-env.sh
similarity index 100%
rename from devops/gcp/tests/build/validate-env.sh
rename to devops/gc/tests/build/validate-env.sh
diff --git a/devops/gcp/tests/clean_up/README.md b/devops/gc/tests/clean_up/README.md
similarity index 83%
rename from devops/gcp/tests/clean_up/README.md
rename to devops/gc/tests/clean_up/README.md
index a59337c29e3ffa85ab6563f6ea4c01ae714f120a..2165a8a3cef9fbd19263cfb99dbb75405018a0b3 100644
--- a/devops/gcp/tests/clean_up/README.md
+++ b/devops/gc/tests/clean_up/README.md
@@ -65,20 +65,20 @@ $ pip install -r requirements.txt
**Cluster cleanup:**
-* Define variable _OSDU_GCP_HELM_NAMESPACE_ and start the script with _--cluster_ key:
+* Define variable _GC_HELM_NAMESPACE_ and start the script with _--cluster_ key:
```sh
-export OSDU_GCP_HELM_NAMESPACE=<test-namespace> # k8s namespace where test resources deployed
+export GC_HELM_NAMESPACE=<test-namespace> # k8s namespace where test resources deployed
./clean_up.sh --cluster
```
**Datastore cleanup:**
-* Define variables _OSDU_GCP_PROJECT_ and _PARTITION_NAMESPACE_, and start the script with _--datastore_ key:
+* Define variables _GC_PROJECT_ and _PARTITION_NAMESPACE_, and start the script with _--datastore_ key:
```sh
-export OSDU_GCP_PROJECT=<project-id> # Google project id where Datastore used
+export GC_PROJECT=<project-id> # Google project id where Datastore used
export PARTITION_NAMESPACE=<test namespace in Datastore> # test namespace in Datastore
./clean_up.sh --datastore
@@ -89,8 +89,8 @@ export PARTITION_NAMESPACE=<test namespace in Datastore> # test namespace in Dat
* Define variables for both cases and start script with _--all_ key:
```sh
-export OSDU_GCP_HELM_NAMESPACE=<test-namespace>
-export OSDU_GCP_PROJECT=<project-id>
+export GC_HELM_NAMESPACE=<test-namespace>
+export GC_PROJECT=<project-id>
export PARTITION_NAMESPACE=<test namespace in Datastore>
./clean_up.sh --all
diff --git a/devops/gcp/tests/clean_up/clean_up.sh b/devops/gc/tests/clean_up/clean_up.sh
similarity index 70%
rename from devops/gcp/tests/clean_up/clean_up.sh
rename to devops/gc/tests/clean_up/clean_up.sh
index 0c048301fe08447fc787767f29e748ddecaab7a6..dc5bc8db1f221ba35a1fa16d5ff7d03496fad1a6 100644
--- a/devops/gcp/tests/clean_up/clean_up.sh
+++ b/devops/gc/tests/clean_up/clean_up.sh
@@ -2,13 +2,13 @@
cluster_cleanup() {
# check variable for namespace
- if [[ "$OSDU_GCP_HELM_NAMESPACE" == *test ]]
+ if [[ "$GC_HELM_NAMESPACE" == *test ]]
then
# uninstall partition helm releases
- LIST=$(helm -n $OSDU_GCP_HELM_NAMESPACE list -o json | jq -r .[].name)
+ LIST=$(helm -n $GC_HELM_NAMESPACE list -o json | jq -r .[].name)
for RELEASE in $LIST
do
- helm -n $OSDU_GCP_HELM_NAMESPACE uninstall $RELEASE
+ helm -n $GC_HELM_NAMESPACE uninstall $RELEASE
done
echo "Cluster cleanup Finished"
else
@@ -23,8 +23,8 @@ datastore_cleanup () {
apk add py3-pip python3-dev libffi-dev
pip install --upgrade pip
pip install --upgrade setuptools
- pip install -q -r devops/gcp/tests/clean_up/requirements.txt
- python3 devops/gcp/tests/clean_up/datastore_clean_up.py -p "$OSDU_GCP_PROJECT" -n $PARTITION_NAMESPACE --delete-all
+ pip install -q -r devops/gc/tests/clean_up/requirements.txt
+ python3 devops/gc/tests/clean_up/datastore_clean_up.py -p "$GC_PROJECT" -n $PARTITION_NAMESPACE --delete-all
echo "Datastore cleanup finished"
}
diff --git a/devops/gcp/tests/env/Chart.yaml b/devops/gc/tests/env/Chart.yaml
similarity index 100%
rename from devops/gcp/tests/env/Chart.yaml
rename to devops/gc/tests/env/Chart.yaml
diff --git a/devops/gcp/tests/env/templates/db-bootstrap-deployment.yaml b/devops/gc/tests/env/templates/db-bootstrap-deployment.yaml
similarity index 100%
rename from devops/gcp/tests/env/templates/db-bootstrap-deployment.yaml
rename to devops/gc/tests/env/templates/db-bootstrap-deployment.yaml
diff --git a/devops/gcp/tests/env/templates/secrets.yaml b/devops/gc/tests/env/templates/secrets.yaml
similarity index 97%
rename from devops/gcp/tests/env/templates/secrets.yaml
rename to devops/gc/tests/env/templates/secrets.yaml
index 04d4ba32f8d9d1e7a1fb060030f913e3ed3cad9c..7793ac24c699ad3fc97f10b309192a08aa352b24 100644
--- a/devops/gcp/tests/env/templates/secrets.yaml
+++ b/devops/gc/tests/env/templates/secrets.yaml
@@ -25,4 +25,4 @@ data:
POSTGRESQL_PORT: {{ .Values.bootstrap.postgres.secret.postgresqlPort | b64enc | quote }}
POSTGRESQL_USERNAME: {{ .Values.bootstrap.postgres.secret.postgresqlUser | b64enc | quote }}
POSTGRESQL_PASSWORD: {{ .Values.postgresql.global.postgresql.auth.postgresPassword | b64enc | quote }}
- POSTGRESQL_DATABASE: {{ .Values.postgresql.global.postgresql.auth.database | b64enc | quote }}
\ No newline at end of file
+ POSTGRESQL_DATABASE: {{ .Values.postgresql.global.postgresql.auth.database | b64enc | quote }}
diff --git a/devops/gcp/tests/env/values.yaml b/devops/gc/tests/env/values.yaml
similarity index 100%
rename from devops/gcp/tests/env/values.yaml
rename to devops/gc/tests/env/values.yaml
diff --git a/devops/gcp/configmap/README.md b/devops/gcp/configmap/README.md
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/devops/gcp/deploy/templates/peer-authentication.yaml b/devops/gcp/deploy/templates/peer-authentication.yaml
deleted file mode 100644
index 2df6ac7c9313a015affed42ce3843861654dbe33..0000000000000000000000000000000000000000
--- a/devops/gcp/deploy/templates/peer-authentication.yaml
+++ /dev/null
@@ -1,10 +0,0 @@
-{{- if .Values.conf.onPremEnabled }}
-apiVersion: security.istio.io/v1beta1
-kind: PeerAuthentication
-metadata:
- name: {{ (print .Release.Namespace "-peer-policy") | lower | quote }}
- namespace: {{ .Release.Namespace | quote }}
-spec:
- mtls:
- mode: {{ .Values.namespacePolicy.mtlsMode | quote }}
-{{- end }}
diff --git a/devops/gcp/pipeline/override-stages.yml b/devops/gcp/pipeline/override-stages.yml
deleted file mode 100644
index 419751eae19b4e83d4a5515b81a67d4dcba65d24..0000000000000000000000000000000000000000
--- a/devops/gcp/pipeline/override-stages.yml
+++ /dev/null
@@ -1,300 +0,0 @@
-variables:
- OSDU_GCP_ENABLE_BOOTSTRAP: "true"
- OSDU_GCP_SERVICE: partition
- OSDU_GCP_VENDOR: gcp
- OSDU_GCP_HELM_ENV_DIR: "devops/gcp/tests/env"
- ENV_BUILD_PATH: "devops/gcp/tests/build/Dockerfile"
-
-.common_cleanup:
- script:
- - git clone https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-gcp-provisioning.git
- - cp infra-gcp-provisioning/tools/datastore-cleanup/* devops/gcp/tests/clean_up/
-
-osdu-gcp-containerize-bootstrap-env-gitlab:
- stage: build
- image: docker:19.03.15
- tags: ["osdu-small"]
- services:
- - docker:20.10.7-dind
- variables:
- BUILD_BOOTSTRAP_PATH: "devops/gcp/tests/build/Dockerfile"
- script:
- - export EXTRA_DOCKER_TAG=""
- - >
- if [[ "$CI_COMMIT_TAG" != "" ]];
- then EXTRA_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/$IMAGE_BOOTSTRAP_NAME-env:$CI_COMMIT_TAG";
- elif [[ "$CI_COMMIT_REF_NAME" = "$CI_DEFAULT_BRANCH" ]];
- then EXTRA_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/$IMAGE_BOOTSTRAP_NAME-env:latest"; fi
- - docker build -t $CI_REGISTRY_IMAGE/$IMAGE_BOOTSTRAP_NAME-env:$CI_COMMIT_SHORT_SHA $EXTRA_DOCKER_TAG -f $BUILD_BOOTSTRAP_PATH .
- - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- - docker push $CI_REGISTRY_IMAGE/$IMAGE_BOOTSTRAP_NAME-env
- rules:
- - if: "$OSDU_GCP == '1' && $OSDU_GCP_ENABLE_BOOTSTRAP == 'true'"
- when: on_success
-
-osdu-gcp-anthos-deploy-test-env: #infra deploy for on-prem install
- environment:
- name: GCP
- extends: .osdu-gcp-anthos-variables
- tags: ["osdu-small"]
- image: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine
- stage: deploy
- needs: ["osdu-gcp-containerize-bootstrap-env-gitlab", "compile-and-unit-test"]
- retry: 1
- variables:
- OSDU_GCP_HELM_NAMESPACE: partition-int-test
- script:
- - gcloud auth activate-service-account --key-file $OSDU_GCP_ANTHOS_DEPLOY_FILE
- - !reference [.common_config, script]
- - export PASSWORD=$(</dev/urandom tr -dc _A-Z-a-z-0-9 | head -c10)
- - cd $OSDU_GCP_HELM_ENV_DIR
- - helm dependency update
- - >
- helm upgrade $OSDU_GCP_SERVICE-env .
- --install
- --create-namespace
- --namespace=$OSDU_GCP_HELM_NAMESPACE
- --wait
- --history-max=3
- --set postgresql.global.postgresql.auth.postgresPassword=$PASSWORD
- --set bootstrap.postgres.image=$CI_REGISTRY_IMAGE/osdu-gcp-bootstrap-$OSDU_GCP_SERVICE-env:$CI_COMMIT_SHORT_SHA
- $OSDU_GCP_HELM_TIMEOUT
- rules:
- - if: "$CI_COMMIT_BRANCH =~ /^release/"
- when: never
- - if: "$CI_COMMIT_TAG"
- when: never
- - if: '$OSDU_GCP == "1"'
- when: on_success
-
-osdu-gcp-anthos-deploy-deployment: # reuse common deploy job for test deployment
- needs: ["osdu-gcp-anthos-deploy-test-env", "osdu-gcp-containerize-gitlab"]
- variables:
- OSDU_GCP_HELM_NAMESPACE: partition-int-test
- ISTIO_ENABLED: "false"
- OSDU_GCP_TENANT: test
-
-osdu-gcp-anthos-test:
- variables:
- OSDU_GCP_VENDOR: anthos
- PARTITION_BASE_URL: https://test.ref.gcp.gnrg-osdu.projects.epam.com/
-
-osdu-gcp-anthos-cleanup-env: # clean-up env after tests
- environment:
- name: GCP
- stage: cleanup
- extends: .osdu-gcp-anthos-variables
- image: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine
- needs: [osdu-gcp-anthos-test]
- tags: ["osdu-small"]
- variables:
- PARTITION_NAMESPACE: partition-test
- OSDU_GCP_HELM_NAMESPACE: partition-int-test
- script:
- - gcloud auth activate-service-account --key-file $OSDU_GCP_ANTHOS_DEPLOY_FILE
- - !reference [.common_config, script]
- - export GOOGLE_APPLICATION_CREDENTIALS="$OSDU_GCP_ANTHOS_DEPLOY_FILE"
- - !reference [.common_cleanup, script]
- - chmod +x devops/gcp/tests/clean_up/clean_up.sh && devops/gcp/tests/clean_up/clean_up.sh --cluster
- rules:
- - if: "$CI_COMMIT_BRANCH =~ /^release/"
- when: never
- - if: "$CI_COMMIT_TAG"
- when: never
- - if: '$OSDU_GCP == "1"'
- when: always
-
-osdu-gcp-anthos-deploy: # verified deploy after tests
- environment:
- name: GCP
- extends: .osdu-gcp-anthos-variables
- tags: ["osdu-small"]
- image: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine
- stage: verified-deploy
- needs: ["osdu-gcp-anthos-test"]
- retry: 1
- script:
- - gcloud auth activate-service-account --key-file $OSDU_GCP_ANTHOS_DEPLOY_FILE
- - !reference [.common_config, script]
- - >
- helm upgrade $OSDU_GCP_SERVICE-deploy $OSDU_GCP_HELM_DEPLOYMENT_DIR
- --install
- --create-namespace
- --namespace=$OSDU_GCP_HELM_NAMESPACE
- --wait
- --history-max=3
- --set data.bootstrapImage=$CI_REGISTRY_IMAGE/osdu-gcp-bootstrap-$OSDU_GCP_SERVICE:$CI_COMMIT_SHORT_SHA
- --set data.image=$CI_REGISTRY_IMAGE/osdu-gcp-$OSDU_GCP_SERVICE:$CI_COMMIT_SHORT_SHA
- --set data.serviceAccountName=$OSDU_GCP_SERVICE
- --set data.bucketPrefix=osdu-anthos
- --set data.dataPartitionId=$OSDU_GCP_TENANT
- --set data.logLevel=INFO
- --set data.springProfilesActive=anthos
- --set conf.cicdEnabled=true
- --set conf.publicAvailable=true
- --set conf.onPremEnabled=true
- --set conf.domain=$OSDU_GCP_DOMAIN
- $OSDU_GCP_HELM_TIMEOUT
- - !reference [.verify_deploy, script]
- - !reference [.verify_bootstrap, script]
- rules:
- - if: "$CI_COMMIT_BRANCH =~ /^release/"
- when: never
- - if: "$CI_COMMIT_TAG"
- when: never
- - if: '$OSDU_GCP == "1"'
- when: on_success
-
-osdu-gcp-deploy-deployment: # reuse common deploy job for test deployment
- variables:
- OSDU_GCP_HELM_NAMESPACE: partition-int-test
- DATA_PARTITION_ID: test
- OSDU_GCP_PARTITION_NAMESPACE: partition-test
-
-osdu-gcp-test:
- variables:
- CLIENT_TENANT: test
- PARTITION_BASE_URL: https://test.community.gcp.gnrg-osdu.projects.epam.com/
-
-osdu-gcp-cleanup-env: # clean-up env after tests
- environment:
- name: GCP
- stage: cleanup
- extends: .osdu-gcp-variables
- image: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine
- needs: [osdu-gcp-test]
- tags: ["osdu-small"]
- variables:
- PARTITION_NAMESPACE: partition-test
- OSDU_GCP_HELM_NAMESPACE: partition-int-test
- script:
- - gcloud auth activate-service-account --key-file $OSDU_GCP_DEPLOY_FILE
- - !reference [.common_config, script]
- - export GOOGLE_APPLICATION_CREDENTIALS="$OSDU_GCP_DEPLOY_FILE"
- - !reference [.common_cleanup, script]
- - chmod +x devops/gcp/tests/clean_up/clean_up.sh && devops/gcp/tests/clean_up/clean_up.sh --all
- rules:
- - if: "$CI_COMMIT_BRANCH =~ /^release/"
- when: never
- - if: "$CI_COMMIT_TAG"
- when: never
- - if: '$OSDU_GCP == "1"'
- when: always
-
-osdu-gcp-deploy: # verified deploy after tests
- environment:
- name: GCP
- extends: .osdu-gcp-variables
- tags: ["osdu-small"]
- image: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine
- stage: verified-deploy
- needs: ["osdu-gcp-test"]
- retry: 1
- script:
- - gcloud auth activate-service-account --key-file $OSDU_GCP_DEPLOY_FILE
- - !reference [.common_config, script]
- - >
- helm upgrade $OSDU_GCP_SERVICE-deploy $OSDU_GCP_HELM_DEPLOYMENT_DIR
- --install
- --create-namespace
- --namespace=$OSDU_GCP_HELM_NAMESPACE
- --wait
- --history-max=3
- --set data.bootstrapImage=$CI_REGISTRY_IMAGE/osdu-gcp-bootstrap-$OSDU_GCP_SERVICE:$CI_COMMIT_SHORT_SHA
- --set data.image=$CI_REGISTRY_IMAGE/osdu-gcp-$OSDU_GCP_SERVICE:$CI_COMMIT_SHORT_SHA
- --set data.serviceAccountName=$OSDU_GCP_SERVICE-k8s
- --set data.projectId=$OSDU_GCP_PROJECT
- --set data.partitionAdminAccounts=$OSDU_GCP_PARTITION_ADMIN_ACCOUNTS
- --set data.dataPartitionId=$OSDU_GCP_TENANT
- --set data.logLevel=INFO
- --set conf.domain=$OSDU_GCP_DOMAIN
- $OSDU_GCP_HELM_TIMEOUT
- - !reference [.verify_deploy, script]
- - !reference [.verify_bootstrap, script]
- rules:
- - if: "$CI_COMMIT_BRANCH =~ /^release/"
- when: never
- - if: "$CI_COMMIT_TAG"
- when: never
- - if: '$OSDU_GCP == "1"'
- when: on_success
-
-osdu-gcp-dev2-deploy-deployment: # reuse common deploy job for test deployment
- variables:
- OSDU_GCP_HELM_NAMESPACE: partition-int-test
- DATA_PARTITION_ID: test
- OSDU_GCP_PARTITION_NAMESPACE: partition-test
-
-osdu-gcp-dev2-test:
- variables:
- CLIENT_TENANT: test
- PARTITION_BASE_URL: https://test.dev2.gcp.gnrg-osdu.projects.epam.com/
-
-osdu-gcp-dev2-cleanup-env: # clean-up env after tests
- environment:
- name: GCP
- stage: cleanup
- extends: .osdu-gcp-dev2-variables
- image: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine
- needs: [osdu-gcp-dev2-test]
- tags: ["osdu-small"]
- variables:
- PARTITION_NAMESPACE: partition-test
- OSDU_GCP_HELM_NAMESPACE: partition-int-test
- script:
- - gcloud auth activate-service-account --key-file $OSDU_GCP_DEPLOY_FILE
- - !reference [.common_config, script]
- - export GOOGLE_APPLICATION_CREDENTIALS="$OSDU_GCP_DEPLOY_FILE"
- - !reference [.common_cleanup, script]
- - chmod +x devops/gcp/tests/clean_up/clean_up.sh && devops/gcp/tests/clean_up/clean_up.sh --all
- rules:
- - if: '$OSDU_GCP == "1" && $CI_COMMIT_BRANCH =~ /^release/'
- when: always
- - if: '$OSDU_GCP == "1" && $CI_COMMIT_TAG'
- when: always
-
-osdu-gcp-dev2-deploy: # verified deploy after tests
- tags: ["osdu-small"]
- extends: .osdu-gcp-dev2-variables
- image: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine
- stage: verified-deploy
- needs: ["osdu-gcp-containerize-gitlab", "osdu-gcp-dev2-test"]
- retry: 1
- script:
- - gcloud auth activate-service-account --key-file $OSDU_GCP_DEV2_DEPLOY_FILE
- - !reference [.common_config, script]
- - !reference [.define_version, script]
- - !reference [.set_image_name,script]
- - helm repo add $OSDU_GCP_SERVICE ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/helm/stable
- - >
- helm upgrade $OSDU_GCP_SERVICE-deploy $OSDU_GCP_SERVICE/gcp-$OSDU_GCP_SERVICE-deploy
- --version $VERSION
- --install
- --create-namespace
- --namespace=$OSDU_GCP_HELM_NAMESPACE
- --wait
- --history-max=3
- --set data.imagePullPolicy=Always
- --set data.bootstrapImage=$CI_REGISTRY_IMAGE/osdu-gcp-bootstrap-$OSDU_GCP_SERVICE:${CI_COMMIT_TAG:-$CI_COMMIT_SHORT_SHA}
- --set data.image=$CI_REGISTRY_IMAGE/$IMAGE_NAME:${CI_COMMIT_TAG:-$CI_COMMIT_SHORT_SHA}
- --set data.serviceAccountName=$OSDU_GCP_SERVICE-k8s
- --set data.projectId=osdu-dev2
- --set data.partitionAdminAccounts=$OSDU_GCP_DEV2_PARTITION_ADMIN_ACCOUNTS
- --set data.dataPartitionId=$OSDU_GCP_TENANT
- --set data.logLevel=INFO
- --set conf.domain=$OSDU_GCP_DOMAIN
- $OSDU_GCP_HELM_TIMEOUT
- - !reference [.verify_deploy, script]
- - !reference [.verify_bootstrap, script]
- rules:
- - if: '$OSDU_GCP == "1" && $CI_COMMIT_BRANCH =~ /^release/'
- when: on_success
- - if: '$OSDU_GCP == "1" && $CI_COMMIT_TAG'
- when: on_success
-
-osdu-gcp-preship-deploy-deployment:
- extends: .osdu-gcp-preship-variables
- tags: ["osdu-small"]
- image: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine
- stage: deploy_preship
- needs: ["osdu-gcp-dev2-deploy", "osdu-gcp-dev2-test"]
diff --git a/pom.xml b/pom.xml
index 531a1b78d3acc3aefe4537e41144ddc3a47a5a25..bc8ac9cd708bcbc79b2d7442697c68f306aa5296 100644
--- a/pom.xml
+++ b/pom.xml
@@ -128,7 +128,7 @@
<module>provider/partition-azure</module>
<module>provider/partition-aws</module>
<module>provider/partition-ibm</module>
- <module>provider/partition-gcp</module>
+ <module>provider/partition-gc</module>
</modules>
<profiles>
diff --git a/provider/partition-aws/pom.xml b/provider/partition-aws/pom.xml
index e098fe6efbeb8a3b7f9c2ac2af4f9cb85f70cb8c..362b9161032fff58b58eee1d10326c013bd4a233 100644
--- a/provider/partition-aws/pom.xml
+++ b/provider/partition-aws/pom.xml
@@ -214,6 +214,9 @@
<version>6.2.2</version>
<executions>
<execution>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
<goals>
<goal>check</goal>
</goals>
diff --git a/provider/partition-azure/pom.xml b/provider/partition-azure/pom.xml
index b9b02b7fb1a90b36c66f8db4c8374ca63bf8be18..25d0f6e46f220b378bde1aa409d9c2a9608e44f7 100644
--- a/provider/partition-azure/pom.xml
+++ b/provider/partition-azure/pom.xml
@@ -110,11 +110,19 @@
</dependency>
<!-- Azure dependencies -->
+ <!-- https://mvnrepository.com/artifact/com.azure/azure-storage-blob -->
<dependency>
- <groupId>com.microsoft.azure</groupId>
- <artifactId>azure-storage</artifactId>
- <version>8.6.6</version>
+ <groupId>com.azure</groupId>
+ <artifactId>azure-data-tables</artifactId>
+ <version>12.3.6</version>
</dependency>
+ <!-- https://mvnrepository.com/artifact/com.azure/azure-core -->
+ <dependency>
+ <groupId>com.azure</groupId>
+ <artifactId>azure-core</artifactId>
+ <version>1.34.0</version>
+ </dependency>
+
<dependency>
<groupId>com.azure.spring</groupId>
<artifactId>azure-spring-boot-starter-active-directory</artifactId>
diff --git a/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/di/CloudTableConfiguration.java b/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/di/DataTableConfiguration.java
similarity index 95%
rename from provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/di/CloudTableConfiguration.java
rename to provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/di/DataTableConfiguration.java
index 7258193b5383e37c98c53a21c3a69df52b0f812d..1c8f658cfdbc8682915aff76877708143bb07d95 100644
--- a/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/di/CloudTableConfiguration.java
+++ b/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/di/DataTableConfiguration.java
@@ -20,7 +20,7 @@ import org.springframework.context.annotation.Configuration;
@Configuration
@Getter
-public class CloudTableConfiguration {
+public class DataTableConfiguration {
@Value("${partition.cloud.table-name:PartitionInfo}")
private String cloudTableName;
diff --git a/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/di/TableStorageBootstrapConfig.java b/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/di/TableStorageBootstrapConfig.java
index d9ad0b19fff3bb27c70bd827f0a11e4c4d1679a1..35045fe0fb5f9b0ba2010cf6db1f6a42e589e35e 100644
--- a/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/di/TableStorageBootstrapConfig.java
+++ b/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/di/TableStorageBootstrapConfig.java
@@ -14,11 +14,11 @@
package org.opengroup.osdu.partition.provider.azure.di;
-import com.microsoft.azure.storage.CloudStorageAccount;
-import com.microsoft.azure.storage.RetryLinearRetry;
-import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.table.CloudTable;
-import com.microsoft.azure.storage.table.CloudTableClient;
+import com.azure.core.http.policy.FixedDelayOptions;
+import com.azure.core.http.policy.RetryOptions;
+import com.azure.data.tables.TableClient;
+import com.azure.data.tables.TableServiceClient;
+import com.azure.data.tables.TableServiceClientBuilder;
import lombok.Setter;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.common.Validators;
@@ -29,23 +29,22 @@ import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Lazy;
import javax.inject.Named;
-import java.net.URISyntaxException;
-import java.security.InvalidKeyException;
+import java.time.Duration;
@Configuration
@ConfigurationProperties(prefix = "azure.table-storage")
@Setter
public class TableStorageBootstrapConfig {
- private final static String CONNECTION_STRING = "DefaultEndpointsProtocol=https;AccountName=%s;AccountKey=%s;EndpointSuffix=core.windows.net";
-
private int maximumExecutionTimeMs;
private int retryDeltaBackoffMs;
private int retryMaxAttempts;
+ private final static String CONNECTION_STRING = "DefaultEndpointsProtocol=https;AccountName=%s;AccountKey=%s;EndpointSuffix=core.windows.net";
+
@Bean
@Lazy
- public CloudTableClient getCloudTableClient(
+ public TableServiceClient getTableServiceClient(
final @Named("TABLE_STORAGE_ACCOUNT_NAME") String storageAccountName,
final @Named("TABLE_STORAGE_ACCOUNT_KEY") String storageAccountKey) {
try {
@@ -53,30 +52,46 @@ public class TableStorageBootstrapConfig {
Validators.checkNotNullAndNotEmpty(storageAccountKey, "storageAccountKey");
final String storageConnectionString = String.format(CONNECTION_STRING, storageAccountName, storageAccountKey);
- CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString);
- CloudTableClient cloudTableClient = storageAccount.createCloudTableClient();
- cloudTableClient.getDefaultRequestOptions().setRetryPolicyFactory(new RetryLinearRetry(retryDeltaBackoffMs, retryMaxAttempts));
- cloudTableClient.getDefaultRequestOptions().setMaximumExecutionTimeInMs(maximumExecutionTimeMs);
- return cloudTableClient;
- } catch (URISyntaxException | InvalidKeyException e) {
+
+ //There was no substitute function available for setting the maximum execution time as in the previous
+ //version after my research, leaving that part for now. We would still like to know the replaceable code for below line:
+ //cloudTableClient.getDefaultRequestOptions().setMaximumExecutionTimeInMs(maximumExecutionTimeMs);
+ com.azure.core.http.policy.FixedDelayOptions fixedDelayOptions = new FixedDelayOptions(retryMaxAttempts, Duration.ofMillis(retryDeltaBackoffMs));
+ RetryOptions retryOptions = new RetryOptions(fixedDelayOptions);
+ TableServiceClient serviceClient = new TableServiceClientBuilder()
+ .connectionString(storageConnectionString)
+ .retryOptions(retryOptions)
+ .buildClient();
+
+ return serviceClient;
+ }
+ catch (Exception e){
throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error creating cloud table storage client", e.getMessage(), e);
}
}
@Bean
@Lazy
- public CloudTable getCloudTable(
- final CloudTableClient cloudTableClient,
- final CloudTableConfiguration tblConfiguration) {
+ public TableClient getTableClient(
+ TableServiceClient tableServiceClient,
+ final DataTableConfiguration tblConfiguration) {
try {
- Validators.checkNotNull(cloudTableClient, "cloudTableClient");
+ Validators.checkNotNull(tableServiceClient, "tableServiceClient");
Validators.checkNotNull(tblConfiguration, "tblConfiguration");
- CloudTable cloudTable = cloudTableClient.getTableReference(tblConfiguration.getCloudTableName());
- cloudTable.createIfNotExists();
- return cloudTable;
- } catch (URISyntaxException | StorageException e) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, String.format("Error querying cloud table: %s", tblConfiguration), e.getMessage(), e);
+ //Attempting to create the table first, since if the table is already existing we get a null tableClient.
+ //The behaviour of the API when a table is existing was not clearly documented here
+ //https://learn.microsoft.com/en-us/java/api/com.azure.data.tables.tableserviceclient?view=azure-java-stable#com-azure-data-tables-tableserviceclient-createtableifnotexists(java-lang-string).
+ TableClient tableClient = tableServiceClient.createTableIfNotExists(tblConfiguration.getCloudTableName());
+ if(tableClient == null){
+ //On the other hand, if we attempt to getTableClient for a non-existent table, it would not give us null
+ tableClient = tableServiceClient.getTableClient(tblConfiguration.getCloudTableName());
+ }
+ return tableClient;
}
+ catch (Exception e){
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, String.format("Error creating a Table Client for table: %s", tblConfiguration), e.getMessage(), e);
+ }
+
}
}
diff --git a/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/persistence/CloudTableStore.java b/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/persistence/CloudTableStore.java
deleted file mode 100644
index f9b29ecf3a04ee76089d872535617f13aff9d9f6..0000000000000000000000000000000000000000
--- a/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/persistence/CloudTableStore.java
+++ /dev/null
@@ -1,93 +0,0 @@
-// Copyright 2017-2020, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.partition.provider.azure.persistence;
-
-import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.table.*;
-import org.apache.http.HttpStatus;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-@Component
-public class CloudTableStore {
-
- @Autowired
- private CloudTable cloudTableClient;
-
- public boolean deleteCloudTableEntity(final Class<? extends TableEntity> clazzType, String partitionKey, String rowKey) {
-
- try {
- TableOperation retrievePartition = TableOperation.retrieve(partitionKey, rowKey, clazzType);
- TableEntity partitionEntity = this.cloudTableClient.execute(retrievePartition).getResultAsType();
- TableOperation deleteOperation = TableOperation.delete(partitionEntity);
- this.cloudTableClient.execute(deleteOperation);
- return true;
- } catch (StorageException e) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error querying cloud table", e.getMessage(), e);
- }
- }
-
- public Iterable<? extends TableEntity> queryByKey(final Class<? extends TableEntity> clazzType, final String key, final String value) {
-
- String partitionFilter = TableQuery.generateFilterCondition(
- key,
- TableQuery.QueryComparisons.EQUAL,
- value);
-
- TableQuery<? extends TableEntity> partitionQuery = TableQuery.from(clazzType)
- .where(partitionFilter);
-
- try {
- return this.cloudTableClient.execute(partitionQuery);
- } catch (Exception e) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "error getting partition", e.getMessage(), e);
- }
- }
-
- public Iterable<? extends TableEntity> queryByCompoundKey(final Class<? extends TableEntity> clazzType,
- final String rowKey, final String rowValue,
- final String valueKey, final String value) {
- String rowFilter = TableQuery.generateFilterCondition(
- rowKey,
- TableQuery.QueryComparisons.EQUAL,
- rowValue);
-
- String valueFilter = TableQuery.generateFilterCondition(
- valueKey,
- TableQuery.QueryComparisons.EQUAL,
- value);
-
- String combinedFilter = TableQuery.combineFilters(rowFilter,
- TableQuery.Operators.AND, valueFilter);
-
- TableQuery<? extends TableEntity> partitionQuery = TableQuery.from(clazzType)
- .where(combinedFilter);
-
- try {
- return this.cloudTableClient.execute(partitionQuery);
- } catch (Exception e) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "error getting partition", e.getMessage(), e);
- }
- }
-
- public void insertBatchEntities(TableBatchOperation batchOperation) {
- try {
- this.cloudTableClient.execute(batchOperation);
- } catch (StorageException e) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "error creating partition", e.getMessage(), e);
- }
- }
-}
diff --git a/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/persistence/DataTableStore.java b/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/persistence/DataTableStore.java
new file mode 100644
index 0000000000000000000000000000000000000000..a5007216bf2a58470980d8ba9a87ac13215add9b
--- /dev/null
+++ b/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/persistence/DataTableStore.java
@@ -0,0 +1,108 @@
+// Copyright 2017-2020, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.partition.provider.azure.persistence;
+
+import com.azure.data.tables.TableClient;
+import com.azure.data.tables.models.ListEntitiesOptions;
+import com.azure.data.tables.models.TableEntity;
+import com.azure.data.tables.models.TableTransactionAction;
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import javax.validation.ValidationException;
+
+@Component
+public class DataTableStore {
+ @Autowired
+ private TableClient tableClient;
+ private final String WHITELISTED_CHARACTERS = "[-_[A-Za-z0-9]]";
+ public boolean deleteCloudTableEntity(String partitionKey, String rowKey) {
+
+ try {
+ TableEntity tableEntity = new TableEntity(partitionKey, rowKey);
+ tableClient.deleteEntity(tableEntity);
+ return true;
+ } catch (Exception e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error querying cloud table", e.getMessage(), e);
+ }
+ }
+
+ public Iterable<TableEntity> queryByKey(final String key, final String value) {
+ /*** Addressing review comment: "We can add a validator for checking sql injection"
+ * There was no strict validation of 'value' in the upper layers,
+ * So, placing here a validation for allowed characters( alphanumeric, - and _)
+ */
+ if(checkValidString(value) == false){
+ throw new ValidationException("Invalid input parameters, value contains illegal character(s)");
+ }
+ String filter = key + " eq '" + value + "'";
+ //Create a filter condition key eq value
+ ListEntitiesOptions listEntitiesOptions = new ListEntitiesOptions().setFilter(filter);
+ try {
+ return this.tableClient.listEntities(listEntitiesOptions, null, null);
+ } catch (Exception e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR,
+ "error listing entities by filter: " + filter, e.getMessage(), e);
+ }
+ }
+
+ public Iterable<TableEntity> queryByCompoundKey(final String rowKey, final String rowValue,
+ final String valueKey, final String value) {
+ if(checkValidString(rowValue) == false || checkValidString(value) == false){
+ throw new ValidationException("Invalid input parameters, value contains illegal character(s)");
+ }
+
+ String rowFilter = rowKey + " eq '" + rowValue + "'";
+
+ String valueFilter = valueKey + " eq '" + value + "'";
+
+ String combinedFilter = rowFilter + " and " + valueFilter;
+ ListEntitiesOptions listEntitiesOptions = new ListEntitiesOptions().setFilter(combinedFilter);
+
+ try {
+ return tableClient.listEntities(listEntitiesOptions, null, null);
+ } catch (Exception e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "error getting partition", e.getMessage(), e);
+ }
+ }
+
+ public void insertBatchEntities(List<TableTransactionAction> transactionAction) {
+ try {
+ this.tableClient.submitTransaction(transactionAction);
+ } catch (Exception e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "error creating partition", e.getMessage(), e);
+ }
+ }
+
+ /**
+ * Helper function to check if the input string contains only the allowed characters
+ * Alphanumeric characters and - and underscore...
+ * We match the length of input string and the length of characters matched from the allowed list
+ * */
+ private Boolean checkValidString(String input){
+ Pattern pattern = Pattern.compile(WHITELISTED_CHARACTERS);
+ Matcher matcher = pattern.matcher(input);
+ int matchedCharacterCount = 0;
+ while (matcher.find()) {
+ matchedCharacterCount++;
+ }
+ return matchedCharacterCount == input.length();
+ }
+}
diff --git a/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/persistence/PartitionEntity.java b/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/persistence/PartitionEntity.java
deleted file mode 100644
index 21cc1f43c6885cd2af399eb054597fd62373137b..0000000000000000000000000000000000000000
--- a/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/persistence/PartitionEntity.java
+++ /dev/null
@@ -1,45 +0,0 @@
-// Copyright 2017-2020, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.partition.provider.azure.persistence;
-
-import com.microsoft.azure.storage.table.DynamicTableEntity;
-
-public class PartitionEntity extends DynamicTableEntity {
-
- private String partitionId;
-
- private String name;
-
- public PartitionEntity() {}
-
- public PartitionEntity(String partitionId, String name) {
- super(partitionId, name);
-
- this.partitionId = partitionId;
- this.name = name;
- }
-
- public String getName() {
- return this.name;
- }
-
- public void setName(String name) { this.name = name; }
-
- public String getPartitionId() {
- return this.partitionId;
- }
-
- public void setPartitionId(String partitionId) { this.partitionId = partitionId; }
-}
\ No newline at end of file
diff --git a/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/persistence/PartitionTableStore.java b/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/persistence/PartitionTableStore.java
index 4b101ddda2f25c58959e6b9b7e54a52696e71f5d..f44bd0e83ffdd27e6853f2ce32717ba1ba04414c 100644
--- a/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/persistence/PartitionTableStore.java
+++ b/provider/partition-azure/src/main/java/org/opengroup/osdu/partition/provider/azure/persistence/PartitionTableStore.java
@@ -14,8 +14,9 @@
package org.opengroup.osdu.partition.provider.azure.persistence;
-import com.microsoft.azure.storage.table.EntityProperty;
-import com.microsoft.azure.storage.table.TableBatchOperation;
+import com.azure.data.tables.models.TableEntity;
+import com.azure.data.tables.models.TableTransactionAction;
+import com.azure.data.tables.models.TableTransactionActionType;
import org.opengroup.osdu.partition.model.PartitionInfo;
import org.opengroup.osdu.partition.model.Property;
import org.springframework.beans.factory.annotation.Autowired;
@@ -34,54 +35,59 @@ public class PartitionTableStore {
private final static String ROW_KEY = "RowKey";
@Autowired
- private CloudTableStore cloudTableStore;
+ private DataTableStore dataTableStore;
public void addPartition(String partitionId, PartitionInfo partitionInfo) {
Map<String, Property> requestProperties = partitionInfo.getProperties();
requestProperties.put(ID, Property.builder().value(partitionId).build());
- TableBatchOperation batchOperation = new TableBatchOperation();
+ //create a list of transactions required
+ List<TableTransactionAction> actionsList = new ArrayList<>();
for (Map.Entry<String, Property> entry : requestProperties.entrySet()) {
String key = entry.getKey();
Property property = entry.getValue();
- PartitionEntity partitionEntity = new PartitionEntity(partitionId, key);
- HashMap<String, EntityProperty> properties = new HashMap<>();
+ TableEntity partitionEntity = new TableEntity(partitionId, key);
+ Map<String, Object> properties = new HashMap<>();
if (property.isSensitive()) {
property.setValue(this.getTenantSafeSecreteId(partitionId, String.valueOf(property.getValue())));
}
- properties.put(VALUE, new EntityProperty(String.valueOf(property.getValue())));
- properties.put(SENSITIVE, new EntityProperty(property.isSensitive()));
+ properties.put(VALUE, property.getValue());
+ properties.put(SENSITIVE, property.isSensitive());
partitionEntity.setProperties(properties);
- batchOperation.insertOrMerge(partitionEntity);
+
+ TableTransactionAction tableTransactionAction = new TableTransactionAction(TableTransactionActionType.UPSERT_MERGE,
+ partitionEntity);
+
+ actionsList.add(tableTransactionAction);
}
- this.cloudTableStore.insertBatchEntities(batchOperation);
+ this.dataTableStore.insertBatchEntities(actionsList);
}
public boolean partitionExists(String partitionId) {
- List<PartitionEntity> partitionEntities = this.queryById(partitionId);
+ List<TableEntity> partitionEntities = this.queryById(partitionId);
return partitionEntities.size() == 1;
}
public Map<String, Property> getPartition(String partitionId) {
Map<String, Property> out = new HashMap<>();
- List<PartitionEntity> partitionEntities = this.getAllByPartitionId(partitionId);
+ List<TableEntity> partitionEntities = this.getAllByPartitionId(partitionId);
if (partitionEntities.isEmpty()) {
return out;
}
- for (PartitionEntity pe : partitionEntities) {
+ for (TableEntity pe : partitionEntities) {
Property property = Property.builder().build();
- HashMap<String, EntityProperty> properties = pe.getProperties();
+ Map<String, Object> properties = pe.getProperties();
if (properties.containsKey(SENSITIVE)) {
- property.setSensitive(properties.get(SENSITIVE).getValueAsBoolean());
+ property.setSensitive((boolean) properties.get(SENSITIVE));
}
if (properties.containsKey(VALUE)) {
- property.setValue(properties.get(VALUE).getValueAsString());
+ property.setValue(properties.get(VALUE).toString());
}
out.put(pe.getRowKey(), property);
}
@@ -90,38 +96,30 @@ public class PartitionTableStore {
@SuppressWarnings("unchecked")
public void deletePartition(String partitionId) {
- Iterable<PartitionEntity> results = (Iterable<PartitionEntity>)
- this.cloudTableStore.queryByKey(PartitionEntity.class,
- PARTITION_KEY, partitionId);
- for (PartitionEntity tableEntity : results) {
- this.cloudTableStore.deleteCloudTableEntity(PartitionEntity.class, tableEntity.getPartitionKey(), tableEntity.getRowKey());
+ Iterable<TableEntity> results = this.dataTableStore.queryByKey(PARTITION_KEY, partitionId);
+ for (TableEntity tableEntity : results) {
+ this.dataTableStore.deleteCloudTableEntity(tableEntity.getPartitionKey(), tableEntity.getRowKey());
}
}
@SuppressWarnings("unchecked")
- private List<PartitionEntity> queryById(String partitionId) {
- List<PartitionEntity> out = new ArrayList<>();
- Iterable<PartitionEntity> results = (Iterable<PartitionEntity>)
- this.cloudTableStore.queryByCompoundKey(PartitionEntity.class,
+ private List<TableEntity> queryById(String partitionId) {
+ List<TableEntity> out = new ArrayList<>();
+ Iterable<TableEntity> results = this.dataTableStore.queryByCompoundKey(
ROW_KEY, ID,
VALUE, partitionId);
- for (PartitionEntity tableEntity : results) {
+ for (TableEntity tableEntity : results) {
out.add(tableEntity);
}
return out;
}
@SuppressWarnings("unchecked")
- private List<PartitionEntity> getAllByPartitionId(String partitionId) {
- List<PartitionEntity> out = new ArrayList<>();
- Iterable<PartitionEntity> results = (Iterable<PartitionEntity>)
- this.cloudTableStore.queryByKey(PartitionEntity.class,
- PARTITION_KEY, partitionId);
- for (PartitionEntity tableEntity : results) {
- tableEntity.setPartitionId(tableEntity.getPartitionKey());
- tableEntity.setName(tableEntity.getRowKey());
- out.add(tableEntity);
- }
+ private List<TableEntity> getAllByPartitionId(String partitionId) {
+ List<TableEntity> out = new ArrayList<>();
+ Iterable<TableEntity> results = (Iterable<TableEntity>)
+ this.dataTableStore.queryByKey(PARTITION_KEY, partitionId);
+ results.forEach(out::add);
return out;
}
@@ -131,10 +129,8 @@ public class PartitionTableStore {
public List<String> getAllPartitions() {
List<String> partitions = new ArrayList<>();
- Iterable<PartitionEntity> results = (Iterable<PartitionEntity>)
- this.cloudTableStore.queryByKey(PartitionEntity.class,
- ROW_KEY, ID);
- for (PartitionEntity tableEntity : results) {
+ Iterable<TableEntity> results = this.dataTableStore.queryByKey( ROW_KEY, ID);
+ for (TableEntity tableEntity : results) {
partitions.add(tableEntity.getPartitionKey());
}
return partitions;
diff --git a/provider/partition-azure/src/test/java/org/opengroup/osdu/partition/provider/azure/persistence/CloudTableStoreTest.java b/provider/partition-azure/src/test/java/org/opengroup/osdu/partition/provider/azure/persistence/CloudTableStoreTest.java
deleted file mode 100644
index 41fa1ffea7cee3dca90f8d200b4ebe849d87f578..0000000000000000000000000000000000000000
--- a/provider/partition-azure/src/test/java/org/opengroup/osdu/partition/provider/azure/persistence/CloudTableStoreTest.java
+++ /dev/null
@@ -1,90 +0,0 @@
-// Copyright 2017-2020, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, eitsher express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.partition.provider.azure.persistence;
-
-import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.table.*;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.powermock.modules.junit4.PowerMockRunner;
-
-
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.when;
-
-@RunWith(PowerMockRunner.class)
-public class CloudTableStoreTest {
-
- @Mock
- private CloudTable cloudTableClient;
-
- @InjectMocks
- private CloudTableStore sut;
-
- @Test
- public void should_empty_whenRecordNotExists() {
- Iterable<PartitionEntity> results = (Iterable<PartitionEntity>) sut.queryByKey(PartitionEntity.class,
- "partitionKey", "partitionId");
- assertNotNull(results);
- }
-
- @Test
- public void should_empty_whenRecordExists() throws StorageException {
- try {
- TableBatchOperation tbOp = new TableBatchOperation();
- when(cloudTableClient.execute(new TableBatchOperation())).thenThrow(new StorageException("Error", "Error", null));
- sut.insertBatchEntities(tbOp);
- fail("should not be here");
- } catch (AppException e) {
- assertEquals(500, e.getError().getCode());
- assertEquals("error creating partition", e.getError().getReason());
- }
- }
-
- @Test
- public void when_call_queryByCompoundKey() {
- Iterable<? extends TableEntity> result = sut.queryByCompoundKey(PartitionEntity.class, "RowKey", "id", "value", "partitionId");
- assertNotNull(result);
- }
-
- @Test
- public void when_wrongInput_ThrowException() {
- try {
- sut.queryByCompoundKey(PartitionEntity.class, null, null, null, null);
- fail("Should not be here");
- } catch (Exception e) {
- assertNotNull(e);
- }
- }
-
- @Test
- public void when_call_queryByKey() {
- Iterable<? extends TableEntity> result = sut.queryByKey(PartitionEntity.class, "PartitionKey", "partitionId");
- assertNotNull(result);
- }
-
- @Test
- public void when_call_queryByKey_wrongInput_ThrowException() {
- try {
- sut.queryByKey(PartitionEntity.class, null, null);
- fail("Should not be here");
- } catch (Exception e) {
- assertNotNull(e);
- }
- }
-}
diff --git a/provider/partition-azure/src/test/java/org/opengroup/osdu/partition/provider/azure/persistence/DataTableStoreTest.java b/provider/partition-azure/src/test/java/org/opengroup/osdu/partition/provider/azure/persistence/DataTableStoreTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..c64a92e3aa2a1818a81980abcd3f875fd2c3aa2d
--- /dev/null
+++ b/provider/partition-azure/src/test/java/org/opengroup/osdu/partition/provider/azure/persistence/DataTableStoreTest.java
@@ -0,0 +1,115 @@
+// Copyright 2017-2020, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, eitsher express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.partition.provider.azure.persistence;
+
+import com.azure.core.http.rest.PagedIterable;
+import com.azure.data.tables.models.TableEntity;
+import com.azure.data.tables.TableClient;
+import com.azure.data.tables.models.TableTransactionAction;
+import io.jsonwebtoken.lang.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import javax.validation.ValidationException;
+
+import static org.junit.Assert.*;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import java.util.*;
+
+@RunWith(PowerMockRunner.class)
+public class DataTableStoreTest {
+
+ @Mock
+ private TableClient tableClient;
+
+ @InjectMocks
+ private DataTableStore sut;
+
+ @Test
+ public void should_empty_whenRecordNotExists() {
+ PagedIterable<TableEntity> response = mock(PagedIterable.class);
+ when(tableClient.listEntities(any(), any(), any())).thenReturn(response);
+ Iterable<TableEntity> results = sut.queryByKey("partitionKey", "partitionId");
+ assertNotNull(results);
+ }
+
+
+ @Test
+ public void should_not_throw_whenEmptyRecordsUploaded() throws Exception {
+ List<TableTransactionAction> actionList = new ArrayList<>();
+ //valid input -> no exception should be thrown
+ //create a more logical test
+ sut.insertBatchEntities(actionList);
+ }
+
+ @Test
+ public void when_call_queryByCompoundKey() {
+ PagedIterable<TableEntity> response = mock(PagedIterable.class);
+ when(tableClient.listEntities(any(), any(), any())).thenReturn(response);
+ Iterable<TableEntity> result = sut.queryByCompoundKey("RowKey", "id", "value", "partitionId");
+ assertNotNull(result);
+ }
+
+ @Test
+ public void when_wrongInput_queryByCompoundKey_ReturnNull() {
+ try {
+ Iterable<TableEntity> outPut = sut.queryByCompoundKey( null, null, null, null);
+ Assert.isNull(outPut);
+ } catch (Exception e) {
+ assertNotNull(e);
+ }
+ }
+
+
+ @Test
+ public void when_call_queryByKey_wrongInput_ShouldReturnNull() {
+ try {
+ Iterable<TableEntity> outPut = sut.queryByKey(null, null);
+ assertNull(outPut);
+ } catch (Exception e) {
+ assertNotNull(e);
+ }
+ }
+
+ @Test
+ public void when_queryByKey_invalid_input_ExceptionIsThrown(){
+ try {
+ sut.queryByKey("partitionKey", "invalid '" );
+ }
+ catch (Exception e){
+ assertNotNull(e);
+ assertEquals(e.getClass(), ValidationException.class);
+ assertEquals(e.getMessage().toString(), "Invalid input parameters, value contains illegal character(s)");
+
+ }
+ }
+ @Test
+ public void when_queryByCompoundKey_invalid_input_ExceptionIsThrown(){
+ try {
+ sut.queryByCompoundKey("rowKey", "id","partitionKey", "invalid '" );
+ }
+ catch (Exception e){
+ assertNotNull(e);
+ assertEquals(e.getClass(), ValidationException.class);
+ assertEquals(e.getMessage().toString(), "Invalid input parameters, value contains illegal character(s)");
+
+ }
+ }
+}
diff --git a/provider/partition-azure/src/test/java/org/opengroup/osdu/partition/provider/azure/persistence/PartitionTableStoreTest.java b/provider/partition-azure/src/test/java/org/opengroup/osdu/partition/provider/azure/persistence/PartitionTableStoreTest.java
index 4ed1dc34a4e571a319002824a2db98e8eca966c8..f4378e4a098449b084d4585ef987088ea606c092 100644
--- a/provider/partition-azure/src/test/java/org/opengroup/osdu/partition/provider/azure/persistence/PartitionTableStoreTest.java
+++ b/provider/partition-azure/src/test/java/org/opengroup/osdu/partition/provider/azure/persistence/PartitionTableStoreTest.java
@@ -14,6 +14,7 @@
package org.opengroup.osdu.partition.provider.azure.persistence;
+import com.azure.data.tables.models.TableEntity;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
@@ -37,7 +38,7 @@ public class PartitionTableStoreTest {
private PartitionTableStore sut;
@Mock
- private CloudTableStore cloudTableStore;
+ private DataTableStore dataTableStore;
private static final String PARTITION_ID = "partitionId";
private static final String PARTITION_KEY = "PartitionKey";
@@ -50,10 +51,10 @@ public class PartitionTableStoreTest {
@Test
public void should_get_partitionInfo() {
- Collection<PartitionEntity> list = new ArrayList<>();
- PartitionEntity partitionEntity = new PartitionEntity(PARTITION_ID, "name");
- list.add(partitionEntity);
- when(cloudTableStore.queryByKey(PartitionEntity.class, PARTITION_KEY, PARTITION_ID)).thenReturn((Iterable) list);
+ Collection<TableEntity> list = new ArrayList<>();
+ TableEntity tableEntity = new TableEntity(PARTITION_ID, "name");
+ list.add(tableEntity);
+ when(dataTableStore.queryByKey(PARTITION_KEY, PARTITION_ID)).thenReturn((Iterable) list);
Map<String, Property> partition = sut.getPartition(PARTITION_ID);
assertNotNull(partition);
@@ -68,13 +69,13 @@ public class PartitionTableStoreTest {
}
@Test
- public void should_addPartiton_whenPartitionProvided() {
+ public void should_addPartition_whenPartitionProvided() {
sut.addPartition(PARTITION_ID, new PartitionInfo());
}
@Test
public void should_returnException_whenNoPartitionInfo() {
- doThrow(new AppException(500, "Error", "error creating partition")).when(cloudTableStore).insertBatchEntities(any());
+ doThrow(new AppException(500, "Error", "error creating partition")).when(dataTableStore).insertBatchEntities(any());
try {
sut.addPartition(PARTITION_ID, new PartitionInfo());
fail("Should not be here");
@@ -86,10 +87,10 @@ public class PartitionTableStoreTest {
@Test
public void should_getAll_partitions() {
- Collection<PartitionEntity> list = new ArrayList<>();
- PartitionEntity partitionEntity = new PartitionEntity(PARTITION_ID, "name");
- list.add(partitionEntity);
- when(cloudTableStore.queryByKey(PartitionEntity.class, "RowKey", "id")).thenReturn((Iterable) list);
+ Collection<TableEntity> list = new ArrayList<>();
+ TableEntity tableEntity = new TableEntity(PARTITION_ID, "name");
+ list.add(tableEntity);
+ when(dataTableStore.queryByKey("RowKey", "id")).thenReturn((Iterable) list);
List<String> partitions = sut.getAllPartitions();
assertNotNull(partitions);
@@ -98,10 +99,10 @@ public class PartitionTableStoreTest {
@Test
public void delete_partition() {
- Collection<PartitionEntity> list = new ArrayList<>();
- PartitionEntity partitionEntity = new PartitionEntity(PARTITION_ID, "name");
- list.add(partitionEntity);
- when(cloudTableStore.queryByKey(PartitionEntity.class, PARTITION_KEY, PARTITION_ID)).thenReturn((Iterable) list);
+ Collection<TableEntity> list = new ArrayList<>();
+ TableEntity tableEntity = new TableEntity(PARTITION_ID, "name");
+ list.add(tableEntity);
+ when(dataTableStore.queryByKey(PARTITION_KEY, PARTITION_ID)).thenReturn((Iterable) list);
sut.deletePartition(PARTITION_ID);
}
}
diff --git a/provider/partition-gcp/README.md b/provider/partition-gc/README.md
similarity index 85%
rename from provider/partition-gcp/README.md
rename to provider/partition-gc/README.md
index 801a4c87a843b6c7ce3d74fa35c651c99c0e4ebd..91e895845f6e5327b7128631701cf3e5d5a393d8 100644
--- a/provider/partition-gcp/README.md
+++ b/provider/partition-gc/README.md
@@ -1,17 +1,20 @@
# Partition Service
-os-partition-gcp is a [Spring Boot](https://spring.io/projects/spring-boot) service that is responsible for creating and retrieving partition specific properties on behalf of other services whether they are secret values or not.
+
+partition-gc is a [Spring Boot](https://spring.io/projects/spring-boot) service that is responsible for creating and retrieving partition specific properties on behalf of other services whether they are secret values or not.
## Features of implementation
+
This is a universal solution created using EPAM OSM mapper technology. It allows you to work with various
implementations of KV stores.
## Limitations of the current version
+
In the current version, the mappers have been equipped with several drivers to the stores:
OSM (mapper for KV-data): Google Datastore; Postgres
-
## Extensibility
+
To use any other store or message broker, implement a driver for it. With an extensible set of drivers, the solution is
unrestrictedly universal and portable without modification to the main code.
Mappers support "multitenancy" with flexibility in how it is implemented. They switch between datasources of different
@@ -21,11 +24,12 @@ tenants due to the work of a bunch of classes that implement the following inter
* DestinationResolver – accepts Destination, finds the resource, connects, and returns Resolution;
* DestinationResolution – contains a ready-made connection, the mapper uses it to get the data.
-
## Getting Started
+
These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system.
### Prerequisites
+
Pre-requisites
* GCloud SDK with java (latest version)
@@ -33,13 +37,16 @@ Pre-requisites
* Lombok 1.16 or later
* Maven
-### Anthos Service Configuration:
-[Anthos service configuration ](docs/anthos/README.md)
-### Google Cloud Service Configuration:
-[Google Cloud service configuration ](docs/gcp/README.md)
+### Anthos Service Configuration
+
+[Anthos service configuration](docs/anthos/README.md)
+
+### Google Cloud Service Configuration
+[Google Cloud service configuration](docs/gc/README.md)
### Run Locally
+
Check that maven is installed:
```bash
@@ -75,11 +82,13 @@ $ cat ~/.m2/settings.xml
</servers>
</settings>
```
+
* Update the Google cloud SDK to the latest version:
```bash
gcloud components update
```
+
* Set Google Project Id:
```bash
@@ -109,10 +118,11 @@ mvn clean install -DskipTests
After configuring your environment as specified above, you can follow these steps to build and run the application. These steps should be invoked from the *repository root.*
```bash
-cd provider/partition-gcp/ && mvn spring-boot:run
+cd provider/partition-gc/ && mvn spring-boot:run
```
## Testing
+
Navigate to partition service's root folder and run all the tests:
```bash
@@ -121,23 +131,29 @@ $ (cd testing/partition-test-core/ && mvn clean install)
```
### Running E2E Tests
+
This section describes how to run cloud OSDU E2E tests.
-### Anthos test configuration:
-[Anthos service configuration ](docs/anthos/README.md)
-### Google Cloud test configuration:
-[Google Cloud service configuration ](docs/gcp/README.md)
+### Anthos test configuration
+
+[Anthos service configuration](docs/anthos/README.md)
+
+### Google Cloud test configuration
+
+[Google Cloud service configuration](docs/gc/README.md)
## Deployment
+
Partition Service is compatible with App Engine Flexible Environment and Cloud Run.
* To deploy into Cloud run, please, use this documentation:
- https://cloud.google.com/run/docs/quickstarts/build-and-deploy
+ <https://cloud.google.com/run/docs/quickstarts/build-and-deploy>
* To deploy into App Engine, please, use this documentation:
- https://cloud.google.com/appengine/docs/flexible/java/quickstart
+ <https://cloud.google.com/appengine/docs/flexible/java/quickstart>
## License
+
Copyright © Google LLC
Copyright © EPAM Systems
diff --git a/provider/partition-gcp/bootstrap/Dockerfile b/provider/partition-gc/bootstrap/Dockerfile
similarity index 81%
rename from provider/partition-gcp/bootstrap/Dockerfile
rename to provider/partition-gc/bootstrap/Dockerfile
index da2812989e7d50efff3f8541895678255ddba9c6..b270902de1dab8a194328b3e0bf7884b085e1f93 100644
--- a/provider/partition-gcp/bootstrap/Dockerfile
+++ b/provider/partition-gc/bootstrap/Dockerfile
@@ -2,7 +2,7 @@ FROM alpine
WORKDIR /opt
-COPY ./provider/partition-gcp/bootstrap/ /opt/
+COPY ./provider/partition-gc/bootstrap/ /opt/
ENV PREFIX=$(pwd)
diff --git a/provider/partition-gcp/bootstrap/bootstrap_partition.sh b/provider/partition-gc/bootstrap/bootstrap_partition.sh
similarity index 91%
rename from provider/partition-gcp/bootstrap/bootstrap_partition.sh
rename to provider/partition-gc/bootstrap/bootstrap_partition.sh
index d821757c827bbdc098151ea524697389da50ee74..e469e74e7e2fcf2c2d7b2e2e0602da1bd88d7b22 100644
--- a/provider/partition-gcp/bootstrap/bootstrap_partition.sh
+++ b/provider/partition-gc/bootstrap/bootstrap_partition.sh
@@ -3,7 +3,7 @@
set -ex
source ./data_anthos.sh
-source ./data_gcp.sh
+source ./data_gc.sh
# Bootstrap Partition service on Anthos (on-prem)
bootstrap_anthos() {
@@ -34,7 +34,7 @@ bootstrap_anthos() {
}
# Bootstrap Partition service on Google Cloud
-bootstrap_gcp() {
+bootstrap_gc() {
echo "sleep to prevent 500 response from the Partition service, due to timeout of creation for Workload Identity"
sleep 20
@@ -45,7 +45,7 @@ bootstrap_gcp() {
status_code=$(curl -X POST \
--url "http://${PARTITION_NAME}/api/partition/v1/partitions/${DATA_PARTITION_ID}" --write-out "%{http_code}" --silent --output "/dev/null" \
-H "Content-Type: application/json" \
- --data-raw "$(generate_post_data_gcp)")
+ --data-raw "$(generate_post_data_gc)")
# shellcheck disable=SC2002
if [[ "${status_code}" == 201 ]]; then
@@ -55,7 +55,7 @@ bootstrap_gcp() {
patch_status_code=$(curl -X PATCH \
--url "http://${PARTITION_NAME}/api/partition/v1/partitions/${DATA_PARTITION_ID}" --write-out "%{http_code}" --silent --output "/dev/null" \
-H "Content-Type: application/json" \
- --data-raw "$(generate_post_data_gcp)")
+ --data-raw "$(generate_post_data_gc)")
echo "Partition was patched because Datastore had already had entities! Status code: ${patch_status_code}"
else
@@ -67,14 +67,14 @@ bootstrap_gcp() {
if [[ "${ENVIRONMENT}" == "anthos" && "${DATA_PARTITION_ID_LIST}" == "" ]]; then
bootstrap_anthos "${DATA_PARTITION_ID}" "${DATA_PARTITION_ID^^}"
elif [[ "${ENVIRONMENT}" == "gcp" && "${DATA_PARTITION_ID_LIST}" == "" ]]; then
- bootstrap_gcp "${DATA_PARTITION_ID}" "${DATA_PARTITION_ID^^}"
+ bootstrap_gc "${DATA_PARTITION_ID}" "${DATA_PARTITION_ID^^}"
elif [[ "${ENVIRONMENT}" == "gcp" && "${DATA_PARTITION_ID_LIST}" != "" ]]; then
IFS=',' read -ra PARTITIONS <<< "${DATA_PARTITION_ID_LIST}"
PARTITIONS=("${DATA_PARTITION_ID}" "${PARTITIONS[@]}")
for PARTITION in "${PARTITIONS[@]}"; do
- bootstrap_gcp "${PARTITION}" "${PARTITION^^}"
+ bootstrap_gc "${PARTITION}" "${PARTITION^^}"
done
fi
diff --git a/provider/partition-gcp/bootstrap/data_anthos.sh b/provider/partition-gc/bootstrap/data_anthos.sh
similarity index 100%
rename from provider/partition-gcp/bootstrap/data_anthos.sh
rename to provider/partition-gc/bootstrap/data_anthos.sh
diff --git a/provider/partition-gcp/bootstrap/data_gcp.sh b/provider/partition-gc/bootstrap/data_gc.sh
similarity index 98%
rename from provider/partition-gcp/bootstrap/data_gcp.sh
rename to provider/partition-gc/bootstrap/data_gc.sh
index 1408636bd34bc7352f3f7f3e41a02a6d51736dec..f79921893a710f49ee749df1e386123c481e543d 100644
--- a/provider/partition-gcp/bootstrap/data_gcp.sh
+++ b/provider/partition-gc/bootstrap/data_gc.sh
@@ -1,6 +1,6 @@
#!/usr/bin/env bash
-generate_post_data_gcp() {
+generate_post_data_gc() {
cat <<EOF
{
"properties": {
diff --git a/provider/partition-gcp/cloudbuild/Dockerfile.cloudbuild b/provider/partition-gc/cloudbuild/Dockerfile.cloudbuild
similarity index 100%
rename from provider/partition-gcp/cloudbuild/Dockerfile.cloudbuild
rename to provider/partition-gc/cloudbuild/Dockerfile.cloudbuild
diff --git a/provider/partition-gcp/cloudbuild/cloudbuild.yaml b/provider/partition-gc/cloudbuild/cloudbuild.yaml
similarity index 51%
rename from provider/partition-gcp/cloudbuild/cloudbuild.yaml
rename to provider/partition-gc/cloudbuild/cloudbuild.yaml
index c8c28a7154bfd2d29a6d03e4be14e0309c0dd46c..d8cde109c6bdf11a829c15238cf6757ebea9dfcc 100644
--- a/provider/partition-gcp/cloudbuild/cloudbuild.yaml
+++ b/provider/partition-gc/cloudbuild/cloudbuild.yaml
@@ -15,17 +15,22 @@
# limitations under the License.
steps:
- - name: 'gcr.io/cloud-builders/docker'
- args: [
- 'build',
- '--build-arg', 'PROVIDER_NAME=${_PROVIDER_NAME}',
- '--build-arg', 'PORT=${_PORT}',
- '-t', 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}:${_SHORT_SHA}',
- '-t', 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}:latest',
- '-f', 'provider/${_GCP_SERVICE}-${_PROVIDER_NAME}/cloudbuild/Dockerfile.cloudbuild',
- '.'
- ]
+ - name: "gcr.io/cloud-builders/docker"
+ args:
+ [
+ "build",
+ "--build-arg",
+ "PROVIDER_NAME=${_PROVIDER_NAME}",
+ "--build-arg",
+ "PORT=${_PORT}",
+ "-t",
+ "gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GC_SERVICE}-${_PROVIDER_NAME}:${_SHORT_SHA}",
+ "-t",
+ "gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GC_SERVICE}-${_PROVIDER_NAME}:latest",
+ "-f",
+ "provider/${_GC_SERVICE}-${_PROVIDER_NAME}/cloudbuild/Dockerfile.cloudbuild",
+ ".",
+ ]
images:
- - 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}'
-
+ - "gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GC_SERVICE}-${_PROVIDER_NAME}"
diff --git a/provider/partition-gcp/docs/anthos/README.md b/provider/partition-gc/docs/anthos/README.md
similarity index 100%
rename from provider/partition-gcp/docs/anthos/README.md
rename to provider/partition-gc/docs/anthos/README.md
diff --git a/provider/partition-gcp/docs/gcp/README.md b/provider/partition-gc/docs/gc/README.md
similarity index 80%
rename from provider/partition-gcp/docs/gcp/README.md
rename to provider/partition-gc/docs/gc/README.md
index 350f379544da8c14456ea4c5ab81fc0238bdc728..0db5faba2acc8ebee17ecd8e41195b3901569117 100644
--- a/provider/partition-gcp/docs/gcp/README.md
+++ b/provider/partition-gc/docs/gc/README.md
@@ -1,6 +1,6 @@
## Service Configuration for Google Cloud
-## Environment variables:
+## Environment variables
Define the following environment variables.
@@ -9,7 +9,7 @@ Must have:
| name | value | description | sensitive? | source |
| --- | --- | --- | --- | --- |
| `SPRING_PROFILES_ACTIVE` | ex `gcp` | Spring profile that activate default configuration for Google Cloud environment | false | - |
-| `GOOGLE_CLOUD_PROJECT` | ex `google` | Google Cloud Project Id| false | https://console.cloud.google.com/ |
+| `GOOGLE_CLOUD_PROJECT` | ex `google` | Google Cloud Project Id| false | <https://console.cloud.google.com/> |
Defined in default application property file but possible to override:
@@ -21,7 +21,7 @@ Defined in default application property file but possible to override:
| `PARTITION_PROPERTY_KIND` | ex `PartitionProperty` | Kind name to store the properties. | no | - |
| `PARTITION_NAMESPACE` | ex `partition` | Namespace for database. | no | - |
-These variables define service behavior, and are used to switch between `anthos` or `gcp` environments, their overriding and usage in mixed mode was not tested.
+These variables define service behavior, and are used to switch between `anthos` or `gc` environments, their overriding and usage in mixed mode was not tested.
Usage of spring profiles is preferred.
| `OSMDRIVER` | ex `postgres` or `datastore` | Osm driver mode that defines which storage will be used | no | - |
@@ -37,19 +37,19 @@ You will need to have the following environment variables defined.
| `PARTITION_BASE_URL` | ex `http://localhost:8080/` | service base URL | yes | |
| `CLIENT_TENANT` | ex `opendes` | name of the client partition | yes | |
| `MY_TENANT` | ex `opendes` | name of the OSDU partition | yes | |
-| `INTEGRATION_TESTER` | `********` | Service account for API calls. Note: this user must be `PARTITION_ADMIN_ACCOUNT` | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
-| `NO_DATA_ACCESS_TESTER` | `********` | Service account base64 encoded string without data access | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
-| `INTEGRATION_TEST_AUDIENCE` | `********` | client application ID | yes | https://console.cloud.google.com/apis/credentials |
+| `INTEGRATION_TESTER` | `********` | Service account for API calls. Note: this user must be `PARTITION_ADMIN_ACCOUNT` | yes | <https://console.cloud.google.com/iam-admin/serviceaccounts> |
+| `NO_DATA_ACCESS_TESTER` | `********` | Service account base64 encoded string without data access | yes | <https://console.cloud.google.com/iam-admin/serviceaccounts> |
Execute following command to build code and run all the integration tests:
```bash
# Note: this assumes that the environment variables for integration tests as outlined
# above are already exported in your environment.
-$ (cd testing/partition-test-gcp/ && mvn clean test)
+$ (cd testing/partition-test-gc/ && mvn clean test)
```
-## Google Cloud service account configuration :
+## Google Cloud service account configuration
+
TBD
| Required roles |
@@ -57,6 +57,7 @@ TBD
| - |
## License
+
Copyright © Google LLC
Copyright © EPAM Systems
@@ -70,4 +71,4 @@ Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
-limitations under the License.
\ No newline at end of file
+limitations under the License.
diff --git a/provider/partition-gcp/pom.xml b/provider/partition-gc/pom.xml
similarity index 97%
rename from provider/partition-gcp/pom.xml
rename to provider/partition-gc/pom.xml
index 5f810a4a8411e5100ec16155e4c80eafd52c42e6..a759f59c7a5428620b9660c34068d94bee7324b0 100644
--- a/provider/partition-gcp/pom.xml
+++ b/provider/partition-gc/pom.xml
@@ -9,8 +9,8 @@
<relativePath>../../pom.xml</relativePath>
</parent>
- <artifactId>partition-gcp</artifactId>
- <description>Partition service on GCP</description>
+ <artifactId>partition-gc</artifactId>
+ <description>Partition service on Google Cloud</description>
<packaging>jar</packaging>
<dependencies>
@@ -28,7 +28,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-gcp</artifactId>
- <version>0.16.0</version>
+ <version>0.19.0-rc3</version>
</dependency>
<dependency>
diff --git a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/PartitionGcpApplication.java b/provider/partition-gc/src/main/java/org/opengroup/osdu/partition/provider/gcp/PartitionGcpApplication.java
similarity index 100%
rename from provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/PartitionGcpApplication.java
rename to provider/partition-gc/src/main/java/org/opengroup/osdu/partition/provider/gcp/PartitionGcpApplication.java
diff --git a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/cache/VmCacheConfiguration.java b/provider/partition-gc/src/main/java/org/opengroup/osdu/partition/provider/gcp/cache/VmCacheConfiguration.java
similarity index 100%
rename from provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/cache/VmCacheConfiguration.java
rename to provider/partition-gc/src/main/java/org/opengroup/osdu/partition/provider/gcp/cache/VmCacheConfiguration.java
diff --git a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/config/IDTokenVerifierConfiguration.java b/provider/partition-gc/src/main/java/org/opengroup/osdu/partition/provider/gcp/config/IDTokenVerifierConfiguration.java
similarity index 100%
rename from provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/config/IDTokenVerifierConfiguration.java
rename to provider/partition-gc/src/main/java/org/opengroup/osdu/partition/provider/gcp/config/IDTokenVerifierConfiguration.java
diff --git a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/config/PostgresOsmConfiguration.java b/provider/partition-gc/src/main/java/org/opengroup/osdu/partition/provider/gcp/config/PostgresOsmConfiguration.java
similarity index 100%
rename from provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/config/PostgresOsmConfiguration.java
rename to provider/partition-gc/src/main/java/org/opengroup/osdu/partition/provider/gcp/config/PostgresOsmConfiguration.java
diff --git a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/config/PropertiesConfiguration.java b/provider/partition-gc/src/main/java/org/opengroup/osdu/partition/provider/gcp/config/PropertiesConfiguration.java
similarity index 100%
rename from provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/config/PropertiesConfiguration.java
rename to provider/partition-gc/src/main/java/org/opengroup/osdu/partition/provider/gcp/config/PropertiesConfiguration.java
diff --git a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/middleware/GcpExceptionMapper.java b/provider/partition-gc/src/main/java/org/opengroup/osdu/partition/provider/gcp/middleware/GcpExceptionMapper.java
similarity index 100%
rename from provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/middleware/GcpExceptionMapper.java
rename to provider/partition-gc/src/main/java/org/opengroup/osdu/partition/provider/gcp/middleware/GcpExceptionMapper.java
diff --git a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/model/PartitionPropertyEntity.java b/provider/partition-gc/src/main/java/org/opengroup/osdu/partition/provider/gcp/model/PartitionPropertyEntity.java
similarity index 100%
rename from provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/model/PartitionPropertyEntity.java
rename to provider/partition-gc/src/main/java/org/opengroup/osdu/partition/provider/gcp/model/PartitionPropertyEntity.java
diff --git a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/mapper/TypeMapperImpl.java b/provider/partition-gc/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/mapper/TypeMapperImpl.java
similarity index 100%
rename from provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/mapper/TypeMapperImpl.java
rename to provider/partition-gc/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/mapper/TypeMapperImpl.java
diff --git a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/provider/OsmPartitionDestinationProvider.java b/provider/partition-gc/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/provider/OsmPartitionDestinationProvider.java
similarity index 100%
rename from provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/provider/OsmPartitionDestinationProvider.java
rename to provider/partition-gc/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/provider/OsmPartitionDestinationProvider.java
diff --git a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/resolver/OsmDatastoreDestinationResolver.java b/provider/partition-gc/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/resolver/OsmDatastoreDestinationResolver.java
similarity index 100%
rename from provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/resolver/OsmDatastoreDestinationResolver.java
rename to provider/partition-gc/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/resolver/OsmDatastoreDestinationResolver.java
diff --git a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/resolver/OsmPostgresDestinationResolver.java b/provider/partition-gc/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/resolver/OsmPostgresDestinationResolver.java
similarity index 100%
rename from provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/resolver/OsmPostgresDestinationResolver.java
rename to provider/partition-gc/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/config/resolver/OsmPostgresDestinationResolver.java
diff --git a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/repository/OsmPartitionPropertyRepository.java b/provider/partition-gc/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/repository/OsmPartitionPropertyRepository.java
similarity index 100%
rename from provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/repository/OsmPartitionPropertyRepository.java
rename to provider/partition-gc/src/main/java/org/opengroup/osdu/partition/provider/gcp/osm/repository/OsmPartitionPropertyRepository.java
diff --git a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/security/AnthosAuthorizationService.java b/provider/partition-gc/src/main/java/org/opengroup/osdu/partition/provider/gcp/security/AnthosAuthorizationService.java
similarity index 100%
rename from provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/security/AnthosAuthorizationService.java
rename to provider/partition-gc/src/main/java/org/opengroup/osdu/partition/provider/gcp/security/AnthosAuthorizationService.java
diff --git a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/security/GcpAuthorizationService.java b/provider/partition-gc/src/main/java/org/opengroup/osdu/partition/provider/gcp/security/GcpAuthorizationService.java
similarity index 100%
rename from provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/security/GcpAuthorizationService.java
rename to provider/partition-gc/src/main/java/org/opengroup/osdu/partition/provider/gcp/security/GcpAuthorizationService.java
diff --git a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/security/SecurityConfigGcp.java b/provider/partition-gc/src/main/java/org/opengroup/osdu/partition/provider/gcp/security/SecurityConfigGcp.java
similarity index 100%
rename from provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/security/SecurityConfigGcp.java
rename to provider/partition-gc/src/main/java/org/opengroup/osdu/partition/provider/gcp/security/SecurityConfigGcp.java
diff --git a/provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImpl.java b/provider/partition-gc/src/main/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImpl.java
similarity index 100%
rename from provider/partition-gcp/src/main/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImpl.java
rename to provider/partition-gc/src/main/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImpl.java
diff --git a/provider/partition-gcp/src/main/resources/application-anthos.properties b/provider/partition-gc/src/main/resources/application-anthos.properties
similarity index 100%
rename from provider/partition-gcp/src/main/resources/application-anthos.properties
rename to provider/partition-gc/src/main/resources/application-anthos.properties
diff --git a/provider/partition-gcp/src/main/resources/application-gcp.properties b/provider/partition-gc/src/main/resources/application-gcp.properties
similarity index 100%
rename from provider/partition-gcp/src/main/resources/application-gcp.properties
rename to provider/partition-gc/src/main/resources/application-gcp.properties
diff --git a/provider/partition-gcp/src/main/resources/application.properties b/provider/partition-gc/src/main/resources/application.properties
similarity index 100%
rename from provider/partition-gcp/src/main/resources/application.properties
rename to provider/partition-gc/src/main/resources/application.properties
diff --git a/provider/partition-gcp/src/main/resources/logback.xml b/provider/partition-gc/src/main/resources/logback.xml
similarity index 100%
rename from provider/partition-gcp/src/main/resources/logback.xml
rename to provider/partition-gc/src/main/resources/logback.xml
diff --git a/provider/partition-gcp/src/test/java/org/opengroup/osdu/partition/provider/gcp/security/GcpAuthorizationServiceTest.java b/provider/partition-gc/src/test/java/org/opengroup/osdu/partition/provider/gcp/security/GcpAuthorizationServiceTest.java
similarity index 100%
rename from provider/partition-gcp/src/test/java/org/opengroup/osdu/partition/provider/gcp/security/GcpAuthorizationServiceTest.java
rename to provider/partition-gc/src/test/java/org/opengroup/osdu/partition/provider/gcp/security/GcpAuthorizationServiceTest.java
diff --git a/provider/partition-gcp/src/test/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImplCacheTest.java b/provider/partition-gc/src/test/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImplCacheTest.java
similarity index 100%
rename from provider/partition-gcp/src/test/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImplCacheTest.java
rename to provider/partition-gc/src/test/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImplCacheTest.java
diff --git a/provider/partition-gcp/src/test/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImplTest.java b/provider/partition-gc/src/test/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImplTest.java
similarity index 100%
rename from provider/partition-gcp/src/test/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImplTest.java
rename to provider/partition-gc/src/test/java/org/opengroup/osdu/partition/provider/gcp/service/PartitionServiceImplTest.java
diff --git a/testing/partition-test-gcp/pom.xml b/testing/partition-test-gc/pom.xml
similarity index 96%
rename from testing/partition-test-gcp/pom.xml
rename to testing/partition-test-gc/pom.xml
index fbc05f92fa6edd78aafbf09ba09ea449ad4202f2..7fd48b41feb09d582f7887779093f0f6ef784160 100644
--- a/testing/partition-test-gcp/pom.xml
+++ b/testing/partition-test-gc/pom.xml
@@ -11,7 +11,7 @@
</parent>
<groupId>org.opengroup.osdu.partition</groupId>
- <artifactId>partition-test-gcp</artifactId>
+ <artifactId>partition-test-gc</artifactId>
<version>0.19.0-SNAPSHOT</version>
<packaging>jar</packaging>
@@ -26,7 +26,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-gcp</artifactId>
- <version>0.3.25</version>
+ <version>0.19.0-rc3</version>
</dependency>
<dependency>
diff --git a/testing/partition-test-gcp/src/test/java/org/opengroup/osdu/partition/api/TestCreatePartition.java b/testing/partition-test-gc/src/test/java/org/opengroup/osdu/partition/api/TestCreatePartition.java
similarity index 100%
rename from testing/partition-test-gcp/src/test/java/org/opengroup/osdu/partition/api/TestCreatePartition.java
rename to testing/partition-test-gc/src/test/java/org/opengroup/osdu/partition/api/TestCreatePartition.java
diff --git a/testing/partition-test-gcp/src/test/java/org/opengroup/osdu/partition/api/TestDeletePartition.java b/testing/partition-test-gc/src/test/java/org/opengroup/osdu/partition/api/TestDeletePartition.java
similarity index 100%
rename from testing/partition-test-gcp/src/test/java/org/opengroup/osdu/partition/api/TestDeletePartition.java
rename to testing/partition-test-gc/src/test/java/org/opengroup/osdu/partition/api/TestDeletePartition.java
diff --git a/testing/partition-test-gcp/src/test/java/org/opengroup/osdu/partition/api/TestGetPartitionById.java b/testing/partition-test-gc/src/test/java/org/opengroup/osdu/partition/api/TestGetPartitionById.java
similarity index 100%
rename from testing/partition-test-gcp/src/test/java/org/opengroup/osdu/partition/api/TestGetPartitionById.java
rename to testing/partition-test-gc/src/test/java/org/opengroup/osdu/partition/api/TestGetPartitionById.java
diff --git a/testing/partition-test-gcp/src/test/java/org/opengroup/osdu/partition/api/TestGetVersionInfo.java b/testing/partition-test-gc/src/test/java/org/opengroup/osdu/partition/api/TestGetVersionInfo.java
similarity index 100%
rename from testing/partition-test-gcp/src/test/java/org/opengroup/osdu/partition/api/TestGetVersionInfo.java
rename to testing/partition-test-gc/src/test/java/org/opengroup/osdu/partition/api/TestGetVersionInfo.java
diff --git a/testing/partition-test-gcp/src/test/java/org/opengroup/osdu/partition/api/TestListPartitions.java b/testing/partition-test-gc/src/test/java/org/opengroup/osdu/partition/api/TestListPartitions.java
similarity index 100%
rename from testing/partition-test-gcp/src/test/java/org/opengroup/osdu/partition/api/TestListPartitions.java
rename to testing/partition-test-gc/src/test/java/org/opengroup/osdu/partition/api/TestListPartitions.java
diff --git a/testing/partition-test-gcp/src/test/java/org/opengroup/osdu/partition/api/TestUpdatePartition.java b/testing/partition-test-gc/src/test/java/org/opengroup/osdu/partition/api/TestUpdatePartition.java
similarity index 100%
rename from testing/partition-test-gcp/src/test/java/org/opengroup/osdu/partition/api/TestUpdatePartition.java
rename to testing/partition-test-gc/src/test/java/org/opengroup/osdu/partition/api/TestUpdatePartition.java
diff --git a/testing/partition-test-gcp/src/test/java/org/opengroup/osdu/partition/util/GCPTestUtils.java b/testing/partition-test-gc/src/test/java/org/opengroup/osdu/partition/util/GCPTestUtils.java
similarity index 70%
rename from testing/partition-test-gcp/src/test/java/org/opengroup/osdu/partition/util/GCPTestUtils.java
rename to testing/partition-test-gc/src/test/java/org/opengroup/osdu/partition/util/GCPTestUtils.java
index be505489d999e4a8dc6e41bfe4001b7ca33e6c98..a00df6c558de638f319add793550ebbac478047f 100644
--- a/testing/partition-test-gcp/src/test/java/org/opengroup/osdu/partition/util/GCPTestUtils.java
+++ b/testing/partition-test-gc/src/test/java/org/opengroup/osdu/partition/util/GCPTestUtils.java
@@ -26,12 +26,7 @@ public class GCPTestUtils extends TestUtils {
if (Strings.isNullOrEmpty(token)) {
String serviceAccountFile = System
.getProperty("INTEGRATION_TESTER", System.getenv("INTEGRATION_TESTER"));
- String audience = System.getProperty("INTEGRATION_TEST_AUDIENCE",
- System.getenv("INTEGRATION_TEST_AUDIENCE"));
- if (Strings.isNullOrEmpty(audience)) {
- audience = "245464679631-ktfdfpl147m1mjpbutl00b3cmffissgq.apps.googleusercontent.com";
- }
- token = new GoogleServiceAccount(serviceAccountFile).getAuthToken(audience);
+ token = new GoogleServiceAccount(serviceAccountFile).getAuthToken();
}
return "Bearer " + token;
}
@@ -41,12 +36,7 @@ public class GCPTestUtils extends TestUtils {
if (Strings.isNullOrEmpty(noAccessToken)) {
String serviceAccountFile = System.getProperty("NO_DATA_ACCESS_TESTER",
System.getenv("NO_DATA_ACCESS_TESTER"));
- String audience = System.getProperty("INTEGRATION_TEST_AUDIENCE",
- System.getenv("INTEGRATION_TEST_AUDIENCE"));
- if (Strings.isNullOrEmpty(audience)) {
- audience = "245464679631-ktfdfpl147m1mjpbutl00b3cmffissgq.apps.googleusercontent.com";
- }
- noAccessToken = new GoogleServiceAccount(serviceAccountFile).getAuthToken(audience);
+ noAccessToken = new GoogleServiceAccount(serviceAccountFile).getAuthToken();
}
return "Bearer " + noAccessToken;
}
diff --git a/testing/partition-test-gcp/src/test/java/org/opengroup/osdu/partition/util/GoogleServiceAccount.java b/testing/partition-test-gc/src/test/java/org/opengroup/osdu/partition/util/GoogleServiceAccount.java
similarity index 96%
rename from testing/partition-test-gcp/src/test/java/org/opengroup/osdu/partition/util/GoogleServiceAccount.java
rename to testing/partition-test-gc/src/test/java/org/opengroup/osdu/partition/util/GoogleServiceAccount.java
index 84e430c012fb785d2e9e252fdf032453cc798869..2082595a000bff7638432d35086e6ed1fb360dfa 100644
--- a/testing/partition-test-gcp/src/test/java/org/opengroup/osdu/partition/util/GoogleServiceAccount.java
+++ b/testing/partition-test-gc/src/test/java/org/opengroup/osdu/partition/util/GoogleServiceAccount.java
@@ -58,7 +58,7 @@ public class GoogleServiceAccount {
return this.serviceAccount.getClientEmail();
}
- public String getAuthToken(String audience) throws IOException {
+ public String getAuthToken() throws IOException {
JwtBuilder jwtBuilder = Jwts.builder();
Map<String, Object> header = new HashMap<>();
@@ -67,7 +67,7 @@ public class GoogleServiceAccount {
jwtBuilder.setHeader(header);
Map<String, Object> claims = new HashMap<>();
- claims.put("target_audience", audience);
+ claims.put("target_audience", "osdu");
claims.put("exp", System.currentTimeMillis() / 1000 + 3600);
claims.put("iat", System.currentTimeMillis() / 1000);
claims.put("iss", this.getEmail());
diff --git a/testing/partition-test-gcp/src/test/resources/logback-test.xml b/testing/partition-test-gc/src/test/resources/logback-test.xml
similarity index 100%
rename from testing/partition-test-gcp/src/test/resources/logback-test.xml
rename to testing/partition-test-gc/src/test/resources/logback-test.xml
diff --git a/testing/pom.xml b/testing/pom.xml
index 2ecf052a1c224c0812f018eda13aa0df72dd0768..6b5ad7df7e7ea26ff177c2cfc797d00f6b80e0ab 100644
--- a/testing/pom.xml
+++ b/testing/pom.xml
@@ -34,7 +34,7 @@
<module>partition-test-azure</module>
<module>partition-test-aws</module>
<module>partition-test-ibm</module>
- <module>partition-test-gcp</module>
+ <module>partition-test-gc</module>
<module>partition-test-anthos</module>
</modules>