Commit f39fe1dc authored by Rustam Lotsmanenko (EPAM)'s avatar Rustam Lotsmanenko (EPAM)
Browse files

Gcp push subscriber mode, tenant & iam migration(GONRG-2742)

parent 720be4f5
...@@ -7,9 +7,11 @@ variables: ...@@ -7,9 +7,11 @@ variables:
OSDU_GCP_SERVICE: wks OSDU_GCP_SERVICE: wks
OSDU_GCP_VENDOR: gcp OSDU_GCP_VENDOR: gcp
OSDU_GCP_APPLICATION_NAME: os-wks OSDU_GCP_APPLICATION_NAME: os-wks
OSDU_GCP_OS_TARGET_SCHEMA_KIND_TENANT: osdu:wks:testWellbore:1.3.1
OSDU_GCP_WKS_MAPPING: WksMapping OSDU_GCP_WKS_MAPPING: WksMapping
OSDU_GCP_WKS_BUCKET: nice-etching-277309-wks-mapping-definitions OSDU_GCP_WKS_BUCKET: nice-etching-277309-wks-mapping-definitions
OSDU_GCP_ENV_VARS: GOOGLE_AUDIENCES=$GOOGLE_AUDIENCE,AUTHORIZE_API=$OSDU_GCP_ENTITLEMENTS_V2_URL,GOOGLE_CLOUD_PROJECT=$OSDU_GCP_PROJECT,WKS_GCP_DATASTORE_MAPPINGINFOKIND=$OSDU_GCP_WKS_MAPPING,WKS_GCP_TENANTNAME=$TENANT_NAME,WKS_GCP_STORAGE_BUCKETNAME=$OSDU_GCP_WKS_BUCKET,PARTITION_API=$OSDU_GCP_PARTITION_API,GOOGLE_AUDIENCES=$GOOGLE_AUDIENCE OSDU_GCP_ENV_VARS: WKS_GCP_AUDIENCES=$GOOGLE_AUDIENCE,WKS_GCP_ENTITLEMENTS_URL=$OSDU_GCP_ENTITLEMENTS_V2_URL,GOOGLE_CLOUD_PROJECT=$OSDU_GCP_PROJECT,WKS_GCP_DATASTORE_MAPPING_INFO_KIND=$OSDU_GCP_WKS_MAPPING,WKS_GCP_TENANT_NAME=$TENANT_NAME,WKS_GCP_STORAGE_BUCKET_NAME=$OSDU_GCP_WKS_BUCKET,PARTITION_API=$OSDU_GCP_PARTITION_API,SEARCH_API=$OSDU_GCP_WKS_SEARCH_API,STORAGE_API=$OSDU_GCP_WKS_STORAGE_API,SCHEMA_API=$OSDU_GCP_WKS_SCHEMA_API,WKS_GCP_SUBSCRIBER_MODE=push,WKS_GCP_REDIS_HOST=$REDIS_SEARCH_HOST --vpc-connector=$OSDU_GCP_VPC_CONNECTOR
OSDU_GCP_DEPLOYMENTS_SUBDIR: deployments/scripts/gcp
include: include:
...@@ -38,7 +40,9 @@ include: ...@@ -38,7 +40,9 @@ include:
file: 'cloud-providers/osdu-gcp-cloudrun.yml' file: 'cloud-providers/osdu-gcp-cloudrun.yml'
- local: "/devops/azure/bootstrap.yml" - local: "/devops/azure/bootstrap.yml"
- local: "/devops/gcp/bootstrap.yml"
osdu-gcp-test: osdu-gcp-test:
allow_failure: true variables:
OS_TARGET_SCHEMA_KIND_TENANT: $OSDU_GCP_OS_TARGET_SCHEMA_KIND_TENANT
...@@ -129,10 +129,10 @@ The following software have components provided under the terms of this license: ...@@ -129,10 +129,10 @@ The following software have components provided under the terms of this license:
- Lucene Highlighter (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-highlighter) - Lucene Highlighter (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-highlighter)
- Lucene Join (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-join) - Lucene Join (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-join)
- Lucene Join (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-join) - Lucene Join (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-join)
- Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-memory)
- Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-backward-codecs) - Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-backward-codecs)
- Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-backward-codecs) - Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-backward-codecs)
- Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-memory) - Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-memory)
- Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-memory)
- Lucene Miscellaneous (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-misc) - Lucene Miscellaneous (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-misc)
- Lucene Miscellaneous (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-misc) - Lucene Miscellaneous (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-misc)
- Lucene Queries (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-queries) - Lucene Queries (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-queries)
...@@ -157,7 +157,6 @@ The following software have components provided under the terms of this license: ...@@ -157,7 +157,6 @@ The following software have components provided under the terms of this license:
- Microsoft Azure Java Core Library (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure Java Core Library (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java)
- Mockito (from http://www.mockito.org)
- Netty/Buffer (from https://repo1.maven.org/maven2/io/netty/netty-buffer) - Netty/Buffer (from https://repo1.maven.org/maven2/io/netty/netty-buffer)
- Netty/Codec (from https://repo1.maven.org/maven2/io/netty/netty-codec) - Netty/Codec (from https://repo1.maven.org/maven2/io/netty/netty-codec)
- Netty/Codec/DNS (from https://repo1.maven.org/maven2/io/netty/netty-codec-dns) - Netty/Codec/DNS (from https://repo1.maven.org/maven2/io/netty/netty-codec-dns)
...@@ -243,6 +242,7 @@ The following software have components provided under the terms of this license: ...@@ -243,6 +242,7 @@ The following software have components provided under the terms of this license:
- elasticsearch-x-content (from https://github.com/elastic/elasticsearch) - elasticsearch-x-content (from https://github.com/elastic/elasticsearch)
- elasticsearch-x-content (from https://github.com/elastic/elasticsearch) - elasticsearch-x-content (from https://github.com/elastic/elasticsearch)
- error-prone annotations (from https://repo1.maven.org/maven2/com/google/errorprone/error_prone_annotations) - error-prone annotations (from https://repo1.maven.org/maven2/com/google/errorprone/error_prone_annotations)
- error-prone annotations (from https://repo1.maven.org/maven2/com/google/errorprone/error_prone_annotations)
- io.grpc:grpc-alts (from https://github.com/grpc/grpc-java) - io.grpc:grpc-alts (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-api (from https://github.com/grpc/grpc-java) - io.grpc:grpc-api (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-auth (from https://github.com/grpc/grpc-java) - io.grpc:grpc-auth (from https://github.com/grpc/grpc-java)
...@@ -267,6 +267,7 @@ The following software have components provided under the terms of this license: ...@@ -267,6 +267,7 @@ The following software have components provided under the terms of this license:
- micrometer-registry-azure-monitor (from https://github.com/micrometer-metrics/micrometer) - micrometer-registry-azure-monitor (from https://github.com/micrometer-metrics/micrometer)
- mockito-core (from https://github.com/mockito/mockito) - mockito-core (from https://github.com/mockito/mockito)
- mockito-core (from https://github.com/mockito/mockito) - mockito-core (from https://github.com/mockito/mockito)
- mockito-core (from https://github.com/mockito/mockito)
- org.apiguardian:apiguardian-api (from https://github.com/apiguardian-team/apiguardian) - org.apiguardian:apiguardian-api (from https://github.com/apiguardian-team/apiguardian)
- org.conscrypt:conscrypt-openjdk-uber (from https://conscrypt.org/) - org.conscrypt:conscrypt-openjdk-uber (from https://conscrypt.org/)
- org.opentest4j:opentest4j (from https://github.com/ota4j-team/opentest4j) - org.opentest4j:opentest4j (from https://github.com/ota4j-team/opentest4j)
...@@ -306,6 +307,7 @@ The following software have components provided under the terms of this license: ...@@ -306,6 +307,7 @@ The following software have components provided under the terms of this license:
- spring-boot-starter-log4j2 (from https://spring.io/projects/spring-boot) - spring-boot-starter-log4j2 (from https://spring.io/projects/spring-boot)
- spring-boot-starter-logging (from https://spring.io/projects/spring-boot) - spring-boot-starter-logging (from https://spring.io/projects/spring-boot)
- spring-boot-starter-security (from https://spring.io/projects/spring-boot) - spring-boot-starter-security (from https://spring.io/projects/spring-boot)
- spring-boot-starter-security (from https://spring.io/projects/spring-boot)
- spring-boot-starter-test (from https://spring.io/projects/spring-boot) - spring-boot-starter-test (from https://spring.io/projects/spring-boot)
- spring-boot-starter-test (from https://spring.io/projects/spring-boot) - spring-boot-starter-test (from https://spring.io/projects/spring-boot)
- spring-boot-starter-tomcat (from https://spring.io/projects/spring-boot) - spring-boot-starter-tomcat (from https://spring.io/projects/spring-boot)
...@@ -319,8 +321,11 @@ The following software have components provided under the terms of this license: ...@@ -319,8 +321,11 @@ The following software have components provided under the terms of this license:
- spring-boot-test-autoconfigure (from https://spring.io/projects/spring-boot) - spring-boot-test-autoconfigure (from https://spring.io/projects/spring-boot)
- spring-boot-test-autoconfigure (from https://spring.io/projects/spring-boot) - spring-boot-test-autoconfigure (from https://spring.io/projects/spring-boot)
- spring-security-config (from http://spring.io/spring-security) - spring-security-config (from http://spring.io/spring-security)
- spring-security-config (from http://spring.io/spring-security)
- spring-security-core (from https://repo1.maven.org/maven2/org/springframework/security/spring-security-core)
- spring-security-core (from https://repo1.maven.org/maven2/org/springframework/security/spring-security-core) - spring-security-core (from https://repo1.maven.org/maven2/org/springframework/security/spring-security-core)
- spring-security-web (from https://repo1.maven.org/maven2/org/springframework/security/spring-security-web) - spring-security-web (from https://repo1.maven.org/maven2/org/springframework/security/spring-security-web)
- spring-security-web (from https://repo1.maven.org/maven2/org/springframework/security/spring-security-web)
- swagger-annotations (from https://repo1.maven.org/maven2/io/swagger/swagger-annotations) - swagger-annotations (from https://repo1.maven.org/maven2/io/swagger/swagger-annotations)
- swagger-jaxrs (from ) - swagger-jaxrs (from )
- tomcat-embed-core (from http://tomcat.apache.org/) - tomcat-embed-core (from http://tomcat.apache.org/)
...@@ -378,7 +383,6 @@ The following software have components provided under the terms of this license: ...@@ -378,7 +383,6 @@ The following software have components provided under the terms of this license:
- Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Java SDK Web with Auto Registration Module (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Web with Auto Registration Module (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
- Mockito (from http://www.mockito.org)
- Netty/Codec/HTTP (from https://repo1.maven.org/maven2/io/netty/netty-codec-http) - Netty/Codec/HTTP (from https://repo1.maven.org/maven2/io/netty/netty-codec-http)
- Protocol Buffers [Core] (from https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java) - Protocol Buffers [Core] (from https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java)
- Protocol Buffers [Util] (from https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java-util) - Protocol Buffers [Util] (from https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java-util)
...@@ -634,7 +638,6 @@ The following software have components provided under the terms of this license: ...@@ -634,7 +638,6 @@ The following software have components provided under the terms of this license:
- Microsoft Azure client library for KeyVault Secrets (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure client library for KeyVault Secrets (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure common module for Storage (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure common module for Storage (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure internal Avro module for Storage (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure internal Avro module for Storage (from https://github.com/Azure/azure-sdk-for-java)
- Mockito (from http://www.mockito.org)
- Netty/Codec/HTTP (from https://repo1.maven.org/maven2/io/netty/netty-codec-http) - Netty/Codec/HTTP (from https://repo1.maven.org/maven2/io/netty/netty-codec-http)
- Netty/Common (from https://repo1.maven.org/maven2/io/netty/netty-common) - Netty/Common (from https://repo1.maven.org/maven2/io/netty/netty-common)
- Project Lombok (from https://projectlombok.org) - Project Lombok (from https://projectlombok.org)
...@@ -652,11 +655,14 @@ The following software have components provided under the terms of this license: ...@@ -652,11 +655,14 @@ The following software have components provided under the terms of this license:
- micrometer-core (from https://github.com/micrometer-metrics/micrometer) - micrometer-core (from https://github.com/micrometer-metrics/micrometer)
- mockito-core (from https://github.com/mockito/mockito) - mockito-core (from https://github.com/mockito/mockito)
- mockito-core (from https://github.com/mockito/mockito) - mockito-core (from https://github.com/mockito/mockito)
- mockito-core (from https://github.com/mockito/mockito)
- mockito-inline (from https://github.com/mockito/mockito)
- mockito-junit-jupiter (from https://github.com/mockito/mockito) - mockito-junit-jupiter (from https://github.com/mockito/mockito)
- mockito-junit-jupiter (from https://github.com/mockito/mockito) - mockito-junit-jupiter (from https://github.com/mockito/mockito)
- msal4j (from https://github.com/AzureAD/microsoft-authentication-library-for-java) - msal4j (from https://github.com/AzureAD/microsoft-authentication-library-for-java)
- msal4j-persistence-extension (from https://github.com/AzureAD/microsoft-authentication-extensions-for-java) - msal4j-persistence-extension (from https://github.com/AzureAD/microsoft-authentication-extensions-for-java)
- spring-security-core (from https://repo1.maven.org/maven2/org/springframework/security/spring-security-core) - spring-security-core (from https://repo1.maven.org/maven2/org/springframework/security/spring-security-core)
- spring-security-core (from https://repo1.maven.org/maven2/org/springframework/security/spring-security-core)
======================================================================== ========================================================================
MPL-1.1 MPL-1.1
......
# Copyright 2021 Google LLC
# Copyright 2021 EPAM Systems, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os
from os import listdir
from os.path import isfile, join
from google.cloud import datastore
local_folder = os.path.join(os.path.dirname(__file__),"../../mapping_info_records/opendes")
wks_namespace = os.environ.get("WKS_NAMESPACE")
kind = os.environ.get("WKS_KIND")
tenant = os.environ.get("WKS_TENANT")
datastore_client = datastore.Client(namespace=wks_namespace)
def upload_records():
files = [f for f in listdir(local_folder) if isfile(join(local_folder, f))]
for file in files:
local_file = join(local_folder, file)
with open(local_file) as f:
data = json.load(f)
data['kind'] = data['kind'].replace('opendes', tenant)
data['sourceSchemaAuthority'] = data['sourceSchemaAuthority'].replace('opendes', tenant)
data['sourceSchemaKind'] = data['sourceSchemaKind'].replace('opendes', tenant)
complete_key = datastore_client.key(kind, data['id'])
mapping = datastore.Entity(key=complete_key)
mapping.update(data)
datastore_client.put(mapping)
if __name__ == "__main__":
upload_records()
# Copyright 2021 Google LLC
# Copyright 2021 EPAM Systems, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from os import listdir
from os.path import isfile, join
from google.cloud import storage
local_folder = os.path.join(os.path.dirname(__file__), "../../mappings/opendes")
storage_client = storage.Client()
bucket_name = os.environ.get("WKS_BUCKET")
tenant = os.environ.get("WKS_TENANT")
bucket = storage_client.get_bucket(bucket_name)
def upload_files():
files = [f for f in listdir(local_folder) if isfile(join(local_folder, f))]
for file in files:
local_file = join(local_folder, file)
fin = open(local_file, 'rt')
data = fin.read()
data = data.replace("opendes", tenant)
fin.close()
fin = open(local_file, 'wt')
fin.write(data)
fin.close()
blob = bucket.blob(file)
blob.upload_from_filename(local_file)
if __name__ == "__main__":
upload_files()
google-cloud-datastore==2.1.3
google-cloud-storage==1.40.0
\ No newline at end of file
osdu_gcp_bootstrap:
stage: bootstrap
image: gcr.io/google.com/cloudsdktool/cloud-sdk
needs: ["osdu-gcp-deploy"]
extends:
- .osdu-gcp-variables
variables:
WKS_KIND: $OSDU_GCP_WKS_MAPPING
WKS_NAMESPACE: $OSDU_TENANT
WKS_TENANT: $OSDU_TENANT
WKS_BUCKET: $OSDU_GCP_WKS_BUCKET
script:
- gcloud auth activate-service-account --key-file $OSDU_GCP_DEPLOY_FILE
- gcloud config set project $OSDU_GCP_PROJECT
- export GOOGLE_APPLICATION_CREDENTIALS=$OSDU_GCP_DEPLOY_FILE
- pip install -r $OSDU_GCP_DEPLOYMENTS_SUBDIR/requirements.txt
- python3 $OSDU_GCP_DEPLOYMENTS_SUBDIR/deploy_mappings.py
- python3 $OSDU_GCP_DEPLOYMENTS_SUBDIR/deploy_mapping_info_records.py
only:
variables:
- $OSDU_GCP == 'true'
osdu-gcp-test:
needs: ["osdu_gcp_bootstrap"]
only:
variables:
- $OSDU_GCP == 'true'
...@@ -17,13 +17,22 @@ In order to run the service locally, you will need to have the following environ ...@@ -17,13 +17,22 @@ In order to run the service locally, you will need to have the following environ
| name | value | description | sensitive? | source | | name | value | description | sensitive? | source |
| --- | --- | --- | --- | --- | | --- | --- | --- | --- | --- |
| `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials |
| `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts | | `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
| `AUTHORIZE_API` | ex `https://entitlements.com/entitlements/v1` | Entitlements API endpoint | yes | output of infrastructure deployment |
| `GOOGLE_CLOUD_PROJECT` | ex `osdu-cicd-epam` | Google Cloud Project Id | yes | - | | `GOOGLE_CLOUD_PROJECT` | ex `osdu-cicd-epam` | Google Cloud Project Id | yes | - |
| `wks.gcp.datastore-mappingInfoKind` | WksMapping | kind for Datastore | yes | - | | `WKS_GCP_AUDIENCES` | ex `1234.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials |
| `wks.gcp.tenantName` | ex `opendes` | Tenant name | yes | - | | `WKS_GCP_DATASTORE_MAPPING_INFO_KIND` | WksMapping | kind for Datastore | yes | - |
| `wks.gcp.storage-bucketName` | ex `osdu-cicd-epam-wks-mapping-definitions` | Storage bucket name for mapping definitions | yes | - | | `WKS_GCP_TENANT_NAME` | ex `opendes` | Tenant name | yes | - |
| `WKS_GCP_STORAGE_BUCKET_NAME` | ex `osdu-cicd-epam-wks-mapping-definitions` | Storage bucket name for mapping definitions | yes | - |
| `PARTITION_API` | ex `https://dev.osdu-gcp.go3-nrg.projects.epam.com/api/partition/v1/` | Partition service endpoint | no | - |
| `SEARCH_API` | ex `https://os-search-jvmvia5dea-uc.a.run.app/api/search/v2` | Search service endpoint | no | - |
| `STORAGE_API` | ex `https://os-storage-jvmvia5dea-uc.a.run.app/api/storage/v2` | Storage service endpoint | no | - |
| `SCHEMA_API` | ex `https://os-schema-jvmvia5dea-uc.a.run.app/api/schema-service/v1` | Schema service endpoint | no | - |
| `WKS_GCP_ENTITLEMENTS_URL` | ex `https://dev.osdu-gcp.go3-nrg.projects.epam.com/api/entitlements/v2/` | Entitlements service endpoint | no | - |
| `WKS_GCP_REDIS_HOST` | ex `127.0.0.1` | Redis host | no | - |
| `WKS_GCP_SUBSCRIBER_MODE` | `push` OR `pull` | Pubsub subscriber mode, `push` mode turn on http endpoint in WKS service and allow to use it in envs that require from deployed service to serve HTTP requests (Cloud Run for example), `pull` mode turn off endpoint and WKS listen events from PubSub by itself | no | - |
Depending on which subscriber mode chosen, the environment must have configured PubSub subscription for records-changed topic, for push mode there must be push subscription with configured endpoint `https://wks-service/api/wks-service/v1/_ah/push-handlers/enqueue` and Service account that can be authorized at entitlements.
For pull mode in PubSub must be present subscriptions for each partition in following format `records-changed-sub-<partition>-wks`.
### Run Locally ### Run Locally
Check that maven is installed: Check that maven is installed:
...@@ -118,6 +127,21 @@ You will need to have the following environment variables defined. ...@@ -118,6 +127,21 @@ You will need to have the following environment variables defined.
| `STORAGE_URL` | ex `********/api/storage/v2/`| storage service URL | yes | - | | `STORAGE_URL` | ex `********/api/storage/v2/`| storage service URL | yes | - |
| `HOST` | `` | schema service URL | yes | - | | `HOST` | `` | schema service URL | yes | - |
The mapping files must be added to the Storage and Datastore for the tests to run successfully, they can be found in [deployments](/deployments).
Scripts for bootstrapping files can be found in [gcp scripts](/deployments/scripts/gcp).
```bash
pip install -r requrments.txt
```
You will need to have the following environment variables defined to run scripts.
| name | value | description | sensitive? | source |
| --- | --- | --- | --- | --- |
| `WKS_KIND` | ex `WksMapping`| Kind in Datastore used by WKS service | no | - |
| `WKS_NAMESPACE` | ex `osdu`| Namespace in Datastore used by WKS service | no | - |
| `WKS_TENANT` | ex `osdu`| WKS service tenant | no | - |
| `WKS_BUCKET` | ex `nice-etching-277309-wks-mapping-definitions`| Bucket used by WKS service to store mapping files | no | - |
| `GOOGLE_APPLICATION_CREDENTIALS` | ex`usr/key.json` | Google Service account credentials with write access to Google Storage and Datastore | yes | - |
Execute following command to build code and run all the integration tests: Execute following command to build code and run all the integration tests:
```bash ```bash
......
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd"> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
<modelVersion>4.0.0</modelVersion> xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent> <modelVersion>4.0.0</modelVersion>
<groupId>org.opengroup.osdu</groupId> <parent>
<artifactId>os-wks</artifactId> <groupId>org.opengroup.osdu</groupId>
<version>0.11.0-SNAPSHOT</version> <artifactId>os-wks</artifactId>
<relativePath>../../pom.xml</relativePath> <version>0.11.0-SNAPSHOT</version>
</parent> <relativePath>../../pom.xml</relativePath>
<artifactId>os-wks-gcp</artifactId> </parent>
<description>Google cloud related implementation stuff.</description> <artifactId>os-wks-gcp</artifactId>
<packaging>jar</packaging> <description>Google cloud related implementation stuff.</description>
<packaging>jar</packaging>
<dependencies> <dependencies>
<dependency> <dependency>
<groupId>org.opengroup.osdu</groupId> <groupId>org.opengroup.osdu</groupId>
<artifactId>os-wks-core</artifactId> <artifactId>os-wks-core</artifactId>
<version>0.11.0-SNAPSHOT</version> <version>0.11.0-SNAPSHOT</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.opengroup.osdu</groupId> <groupId>org.springframework.boot</groupId>
<artifactId>os-core-common</artifactId> <artifactId>spring-boot-starter-security</artifactId>
<version>0.10.0</version> </dependency>
</dependency> <dependency>
<dependency> <groupId>org.opengroup.osdu</groupId>
<groupId>org.opengroup.osdu</groupId> <artifactId>os-core-common</artifactId>
<artifactId>core-lib-gcp</artifactId> <version>0.10.0</version>
<version>0.10.0</version> </dependency>
</dependency> <dependency>
<dependency> <groupId>org.opengroup.osdu</groupId>
<groupId>com.google.cloud</groupId> <artifactId>core-lib-gcp</artifactId>
<artifactId>google-cloud-storage</artifactId> <version>0.10.0</version>
<version>1.108.0</version> </dependency>
</dependency> <dependency>
<dependency> <groupId>io.grpc</groupId>
<groupId>com.google.cloud</groupId> <artifactId>grpc-core</artifactId>
<artifactId>google-cloud-datastore</artifactId> <version>1.38.1</version>
<version>1.102.2</version> </dependency>
</dependency> <dependency>
<dependency> <groupId>com.google.cloud</groupId>
<groupId>org.springframework.boot</groupId> <artifactId>google-cloud-storage</artifactId>
<artifactId>spring-boot-starter-test</artifactId> <version>1.108.0</version>
<scope>test</scope> </dependency>
</dependency> <dependency>
<dependency> <groupId>com.google.cloud</groupId>
<groupId>org.mockito</groupId> <artifactId>google-cloud-datastore</artifactId>
<artifactId>mockito-all</artifactId> <version>1.102.2</version>
<version>1.10.19</version> </dependency>
<scope>test</scope> <dependency>
</dependency> <groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</dependency>
<dependency> <dependency>
<groupId>com.google.apis</groupId> <groupId>com.google.inject</groupId>
<artifactId>google-api-services-iam</artifactId> <artifactId>guice</artifactId>
<version>v1-rev310-1.25.0</version> <version>4.2.0</version>
</dependency> <scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>3.11.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-inline</artifactId>
<version>3.11.2</version>
<scope>test</scope>
</dependency>
<dependency> </dependencies>
<groupId>io.cucumber</groupId>
<artifactId>cucumber-java8</artifactId>
<version>5.4.0</version>
<scope>test</scope>
</dependency>
<dependency> <build>
<groupId>io.cucumber</groupId> <plugins>
<artifactId>cucumber-junit</artifactId> <plugin>
<version>5.4.0</version> <groupId>org.springframework.boot</groupId>
<scope>test</scope> <artifactId>spring-boot-maven-plugin</artifactId>
</dependency> <executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
<configuration>
<classifier>spring-boot</classifier>
<mainClass>
org.opengroup.osdu.wks.WksServiceApplicationGcp
</mainClass>
</configuration>
</execution>
</executions>
</plugin>
<dependency> <plugin>
<groupId>io.cucumber</groupId> <artifactId>maven-war-plugin</artifactId>
<artifactId>cucumber-guice</artifactId> <configuration>
<version>5.4.0</version> <failOnMissingWebXml>false</failOnMissingWebXml>
<scope>test</scope> </configuration>
</dependency> </plugin>
<dependency> <plugin>
<groupId>com.google.inject</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>guice</artifactId> <artifactId>maven-surefire-plugin</artifactId>
<version>4.2.0</version> <version>3.0.0-M3</version>
<scope>test</scope> <configuration>
</dependency> <skipTests>${skipUnitTests}</skipTests>
<dependency> <forkCount>3</forkCount>
<groupId>org.projectlombok</groupId> <reuseForks>true</reuseForks>
<artifactId>lombok</artifactId> <argLine>-Xmx1024m -XX:MaxPermSize=256m</argLine>
</dependency> </configuration>
</plugin>
</dependencies> </plugins>
</build>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
<configuration>
<classifier>spring-boot</classifier>
<mainClass>
org.opengroup.osdu.wks.WksServiceApplication
</mainClass>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-war-plugin</artifactId>
<configuration>
<failOnMissingWebXml>false</failOnMissingWebXml>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>3.0.0-M3</version>
<configuration>
<skipTests>${skipUnitTests}</skipTests>
<forkCount>3</forkCount>
<reuseForks>true</reuseForks>
<argLine>-Xmx1024m -XX:MaxPermSize=256m</argLine>
</configuration>
</plugin>
</plugins>
</build>
</project> </project>
/* /*
Copyright 2021 Google LLC * Copyright 2021 Google LLC
Copyright 2021 EPAM Systems, Inc * Copyright 2021 EPAM Systems, Inc
*
Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
You may obtain a copy of the License at * You may obtain a copy of the License at
*
http://www.apache.org/licenses/LICENSE-2.0 * https://www.apache.org/licenses/LICENSE-2.0
*
Unless required by applicable law or agreed to in writing, software * Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, * distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
limitations under the License. * limitations under the License.
*/