Commit f39fe1dc authored by Rustam Lotsmanenko (EPAM)'s avatar Rustam Lotsmanenko (EPAM)
Browse files

Gcp push subscriber mode, tenant & iam migration(GONRG-2742)

parent 720be4f5
......@@ -7,9 +7,11 @@ variables:
OSDU_GCP_SERVICE: wks
OSDU_GCP_VENDOR: gcp
OSDU_GCP_APPLICATION_NAME: os-wks
OSDU_GCP_OS_TARGET_SCHEMA_KIND_TENANT: osdu:wks:testWellbore:1.3.1
OSDU_GCP_WKS_MAPPING: WksMapping
OSDU_GCP_WKS_BUCKET: nice-etching-277309-wks-mapping-definitions
OSDU_GCP_ENV_VARS: GOOGLE_AUDIENCES=$GOOGLE_AUDIENCE,AUTHORIZE_API=$OSDU_GCP_ENTITLEMENTS_V2_URL,GOOGLE_CLOUD_PROJECT=$OSDU_GCP_PROJECT,WKS_GCP_DATASTORE_MAPPINGINFOKIND=$OSDU_GCP_WKS_MAPPING,WKS_GCP_TENANTNAME=$TENANT_NAME,WKS_GCP_STORAGE_BUCKETNAME=$OSDU_GCP_WKS_BUCKET,PARTITION_API=$OSDU_GCP_PARTITION_API,GOOGLE_AUDIENCES=$GOOGLE_AUDIENCE
OSDU_GCP_ENV_VARS: WKS_GCP_AUDIENCES=$GOOGLE_AUDIENCE,WKS_GCP_ENTITLEMENTS_URL=$OSDU_GCP_ENTITLEMENTS_V2_URL,GOOGLE_CLOUD_PROJECT=$OSDU_GCP_PROJECT,WKS_GCP_DATASTORE_MAPPING_INFO_KIND=$OSDU_GCP_WKS_MAPPING,WKS_GCP_TENANT_NAME=$TENANT_NAME,WKS_GCP_STORAGE_BUCKET_NAME=$OSDU_GCP_WKS_BUCKET,PARTITION_API=$OSDU_GCP_PARTITION_API,SEARCH_API=$OSDU_GCP_WKS_SEARCH_API,STORAGE_API=$OSDU_GCP_WKS_STORAGE_API,SCHEMA_API=$OSDU_GCP_WKS_SCHEMA_API,WKS_GCP_SUBSCRIBER_MODE=push,WKS_GCP_REDIS_HOST=$REDIS_SEARCH_HOST --vpc-connector=$OSDU_GCP_VPC_CONNECTOR
OSDU_GCP_DEPLOYMENTS_SUBDIR: deployments/scripts/gcp
include:
......@@ -38,7 +40,9 @@ include:
file: 'cloud-providers/osdu-gcp-cloudrun.yml'
- local: "/devops/azure/bootstrap.yml"
- local: "/devops/gcp/bootstrap.yml"
osdu-gcp-test:
allow_failure: true
variables:
OS_TARGET_SCHEMA_KIND_TENANT: $OSDU_GCP_OS_TARGET_SCHEMA_KIND_TENANT
......@@ -129,10 +129,10 @@ The following software have components provided under the terms of this license:
- Lucene Highlighter (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-highlighter)
- Lucene Join (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-join)
- Lucene Join (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-join)
- Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-memory)
- Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-backward-codecs)
- Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-backward-codecs)
- Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-memory)
- Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-memory)
- Lucene Miscellaneous (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-misc)
- Lucene Miscellaneous (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-misc)
- Lucene Queries (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-queries)
......@@ -157,7 +157,6 @@ The following software have components provided under the terms of this license:
- Microsoft Azure Java Core Library (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java)
- Mockito (from http://www.mockito.org)
- Netty/Buffer (from https://repo1.maven.org/maven2/io/netty/netty-buffer)
- Netty/Codec (from https://repo1.maven.org/maven2/io/netty/netty-codec)
- Netty/Codec/DNS (from https://repo1.maven.org/maven2/io/netty/netty-codec-dns)
......@@ -243,6 +242,7 @@ The following software have components provided under the terms of this license:
- elasticsearch-x-content (from https://github.com/elastic/elasticsearch)
- elasticsearch-x-content (from https://github.com/elastic/elasticsearch)
- error-prone annotations (from https://repo1.maven.org/maven2/com/google/errorprone/error_prone_annotations)
- error-prone annotations (from https://repo1.maven.org/maven2/com/google/errorprone/error_prone_annotations)
- io.grpc:grpc-alts (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-api (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-auth (from https://github.com/grpc/grpc-java)
......@@ -267,6 +267,7 @@ The following software have components provided under the terms of this license:
- micrometer-registry-azure-monitor (from https://github.com/micrometer-metrics/micrometer)
- mockito-core (from https://github.com/mockito/mockito)
- mockito-core (from https://github.com/mockito/mockito)
- mockito-core (from https://github.com/mockito/mockito)
- org.apiguardian:apiguardian-api (from https://github.com/apiguardian-team/apiguardian)
- org.conscrypt:conscrypt-openjdk-uber (from https://conscrypt.org/)
- org.opentest4j:opentest4j (from https://github.com/ota4j-team/opentest4j)
......@@ -306,6 +307,7 @@ The following software have components provided under the terms of this license:
- spring-boot-starter-log4j2 (from https://spring.io/projects/spring-boot)
- spring-boot-starter-logging (from https://spring.io/projects/spring-boot)
- spring-boot-starter-security (from https://spring.io/projects/spring-boot)
- spring-boot-starter-security (from https://spring.io/projects/spring-boot)
- spring-boot-starter-test (from https://spring.io/projects/spring-boot)
- spring-boot-starter-test (from https://spring.io/projects/spring-boot)
- spring-boot-starter-tomcat (from https://spring.io/projects/spring-boot)
......@@ -319,8 +321,11 @@ The following software have components provided under the terms of this license:
- spring-boot-test-autoconfigure (from https://spring.io/projects/spring-boot)
- spring-boot-test-autoconfigure (from https://spring.io/projects/spring-boot)
- spring-security-config (from http://spring.io/spring-security)
- spring-security-config (from http://spring.io/spring-security)
- spring-security-core (from https://repo1.maven.org/maven2/org/springframework/security/spring-security-core)
- spring-security-core (from https://repo1.maven.org/maven2/org/springframework/security/spring-security-core)
- spring-security-web (from https://repo1.maven.org/maven2/org/springframework/security/spring-security-web)
- spring-security-web (from https://repo1.maven.org/maven2/org/springframework/security/spring-security-web)
- swagger-annotations (from https://repo1.maven.org/maven2/io/swagger/swagger-annotations)
- swagger-jaxrs (from )
- tomcat-embed-core (from http://tomcat.apache.org/)
......@@ -378,7 +383,6 @@ The following software have components provided under the terms of this license:
- Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Java SDK Web with Auto Registration Module (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
- Mockito (from http://www.mockito.org)
- Netty/Codec/HTTP (from https://repo1.maven.org/maven2/io/netty/netty-codec-http)
- Protocol Buffers [Core] (from https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java)
- Protocol Buffers [Util] (from https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java-util)
......@@ -634,7 +638,6 @@ The following software have components provided under the terms of this license:
- Microsoft Azure client library for KeyVault Secrets (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure common module for Storage (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure internal Avro module for Storage (from https://github.com/Azure/azure-sdk-for-java)
- Mockito (from http://www.mockito.org)
- Netty/Codec/HTTP (from https://repo1.maven.org/maven2/io/netty/netty-codec-http)
- Netty/Common (from https://repo1.maven.org/maven2/io/netty/netty-common)
- Project Lombok (from https://projectlombok.org)
......@@ -652,11 +655,14 @@ The following software have components provided under the terms of this license:
- micrometer-core (from https://github.com/micrometer-metrics/micrometer)
- mockito-core (from https://github.com/mockito/mockito)
- mockito-core (from https://github.com/mockito/mockito)
- mockito-core (from https://github.com/mockito/mockito)
- mockito-inline (from https://github.com/mockito/mockito)
- mockito-junit-jupiter (from https://github.com/mockito/mockito)
- mockito-junit-jupiter (from https://github.com/mockito/mockito)
- msal4j (from https://github.com/AzureAD/microsoft-authentication-library-for-java)
- msal4j-persistence-extension (from https://github.com/AzureAD/microsoft-authentication-extensions-for-java)
- spring-security-core (from https://repo1.maven.org/maven2/org/springframework/security/spring-security-core)
- spring-security-core (from https://repo1.maven.org/maven2/org/springframework/security/spring-security-core)
========================================================================
MPL-1.1
......
# Copyright 2021 Google LLC
# Copyright 2021 EPAM Systems, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os
from os import listdir
from os.path import isfile, join
from google.cloud import datastore
local_folder = os.path.join(os.path.dirname(__file__),"../../mapping_info_records/opendes")
wks_namespace = os.environ.get("WKS_NAMESPACE")
kind = os.environ.get("WKS_KIND")
tenant = os.environ.get("WKS_TENANT")
datastore_client = datastore.Client(namespace=wks_namespace)
def upload_records():
files = [f for f in listdir(local_folder) if isfile(join(local_folder, f))]
for file in files:
local_file = join(local_folder, file)
with open(local_file) as f:
data = json.load(f)
data['kind'] = data['kind'].replace('opendes', tenant)
data['sourceSchemaAuthority'] = data['sourceSchemaAuthority'].replace('opendes', tenant)
data['sourceSchemaKind'] = data['sourceSchemaKind'].replace('opendes', tenant)
complete_key = datastore_client.key(kind, data['id'])
mapping = datastore.Entity(key=complete_key)
mapping.update(data)
datastore_client.put(mapping)
if __name__ == "__main__":
upload_records()
# Copyright 2021 Google LLC
# Copyright 2021 EPAM Systems, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from os import listdir
from os.path import isfile, join
from google.cloud import storage
local_folder = os.path.join(os.path.dirname(__file__), "../../mappings/opendes")
storage_client = storage.Client()
bucket_name = os.environ.get("WKS_BUCKET")
tenant = os.environ.get("WKS_TENANT")
bucket = storage_client.get_bucket(bucket_name)
def upload_files():
files = [f for f in listdir(local_folder) if isfile(join(local_folder, f))]
for file in files:
local_file = join(local_folder, file)
fin = open(local_file, 'rt')
data = fin.read()
data = data.replace("opendes", tenant)
fin.close()
fin = open(local_file, 'wt')
fin.write(data)
fin.close()
blob = bucket.blob(file)
blob.upload_from_filename(local_file)
if __name__ == "__main__":
upload_files()
google-cloud-datastore==2.1.3
google-cloud-storage==1.40.0
\ No newline at end of file
osdu_gcp_bootstrap:
stage: bootstrap
image: gcr.io/google.com/cloudsdktool/cloud-sdk
needs: ["osdu-gcp-deploy"]
extends:
- .osdu-gcp-variables
variables:
WKS_KIND: $OSDU_GCP_WKS_MAPPING
WKS_NAMESPACE: $OSDU_TENANT
WKS_TENANT: $OSDU_TENANT
WKS_BUCKET: $OSDU_GCP_WKS_BUCKET
script:
- gcloud auth activate-service-account --key-file $OSDU_GCP_DEPLOY_FILE
- gcloud config set project $OSDU_GCP_PROJECT
- export GOOGLE_APPLICATION_CREDENTIALS=$OSDU_GCP_DEPLOY_FILE
- pip install -r $OSDU_GCP_DEPLOYMENTS_SUBDIR/requirements.txt
- python3 $OSDU_GCP_DEPLOYMENTS_SUBDIR/deploy_mappings.py
- python3 $OSDU_GCP_DEPLOYMENTS_SUBDIR/deploy_mapping_info_records.py
only:
variables:
- $OSDU_GCP == 'true'
osdu-gcp-test:
needs: ["osdu_gcp_bootstrap"]
only:
variables:
- $OSDU_GCP == 'true'
......@@ -17,13 +17,22 @@ In order to run the service locally, you will need to have the following environ
| name | value | description | sensitive? | source |
| --- | --- | --- | --- | --- |
| `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials |
| `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
| `AUTHORIZE_API` | ex `https://entitlements.com/entitlements/v1` | Entitlements API endpoint | yes | output of infrastructure deployment |
| `GOOGLE_CLOUD_PROJECT` | ex `osdu-cicd-epam` | Google Cloud Project Id | yes | - |
| `wks.gcp.datastore-mappingInfoKind` | WksMapping | kind for Datastore | yes | - |
| `wks.gcp.tenantName` | ex `opendes` | Tenant name | yes | - |
| `wks.gcp.storage-bucketName` | ex `osdu-cicd-epam-wks-mapping-definitions` | Storage bucket name for mapping definitions | yes | - |
| `WKS_GCP_AUDIENCES` | ex `1234.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials |
| `WKS_GCP_DATASTORE_MAPPING_INFO_KIND` | WksMapping | kind for Datastore | yes | - |
| `WKS_GCP_TENANT_NAME` | ex `opendes` | Tenant name | yes | - |
| `WKS_GCP_STORAGE_BUCKET_NAME` | ex `osdu-cicd-epam-wks-mapping-definitions` | Storage bucket name for mapping definitions | yes | - |
| `PARTITION_API` | ex `https://dev.osdu-gcp.go3-nrg.projects.epam.com/api/partition/v1/` | Partition service endpoint | no | - |
| `SEARCH_API` | ex `https://os-search-jvmvia5dea-uc.a.run.app/api/search/v2` | Search service endpoint | no | - |
| `STORAGE_API` | ex `https://os-storage-jvmvia5dea-uc.a.run.app/api/storage/v2` | Storage service endpoint | no | - |
| `SCHEMA_API` | ex `https://os-schema-jvmvia5dea-uc.a.run.app/api/schema-service/v1` | Schema service endpoint | no | - |
| `WKS_GCP_ENTITLEMENTS_URL` | ex `https://dev.osdu-gcp.go3-nrg.projects.epam.com/api/entitlements/v2/` | Entitlements service endpoint | no | - |
| `WKS_GCP_REDIS_HOST` | ex `127.0.0.1` | Redis host | no | - |
| `WKS_GCP_SUBSCRIBER_MODE` | `push` OR `pull` | Pubsub subscriber mode, `push` mode turn on http endpoint in WKS service and allow to use it in envs that require from deployed service to serve HTTP requests (Cloud Run for example), `pull` mode turn off endpoint and WKS listen events from PubSub by itself | no | - |
Depending on which subscriber mode chosen, the environment must have configured PubSub subscription for records-changed topic, for push mode there must be push subscription with configured endpoint `https://wks-service/api/wks-service/v1/_ah/push-handlers/enqueue` and Service account that can be authorized at entitlements.
For pull mode in PubSub must be present subscriptions for each partition in following format `records-changed-sub-<partition>-wks`.
### Run Locally
Check that maven is installed:
......@@ -118,6 +127,21 @@ You will need to have the following environment variables defined.
| `STORAGE_URL` | ex `********/api/storage/v2/`| storage service URL | yes | - |
| `HOST` | `` | schema service URL | yes | - |
The mapping files must be added to the Storage and Datastore for the tests to run successfully, they can be found in [deployments](/deployments).
Scripts for bootstrapping files can be found in [gcp scripts](/deployments/scripts/gcp).
```bash
pip install -r requrments.txt
```
You will need to have the following environment variables defined to run scripts.
| name | value | description | sensitive? | source |
| --- | --- | --- | --- | --- |
| `WKS_KIND` | ex `WksMapping`| Kind in Datastore used by WKS service | no | - |
| `WKS_NAMESPACE` | ex `osdu`| Namespace in Datastore used by WKS service | no | - |
| `WKS_TENANT` | ex `osdu`| WKS service tenant | no | - |
| `WKS_BUCKET` | ex `nice-etching-277309-wks-mapping-definitions`| Bucket used by WKS service to store mapping files | no | - |
| `GOOGLE_APPLICATION_CREDENTIALS` | ex`usr/key.json` | Google Service account credentials with write access to Google Storage and Datastore | yes | - |
Execute following command to build code and run all the integration tests:
```bash
......
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.opengroup.osdu</groupId>
......@@ -16,6 +17,10 @@
<artifactId>os-wks-core</artifactId>
<version>0.11.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-security</artifactId>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
......@@ -26,6 +31,11 @@
<artifactId>core-lib-gcp</artifactId>
<version>0.10.0</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-core</artifactId>
<version>1.38.1</version>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-storage</artifactId>
......@@ -37,54 +47,33 @@
<version>1.102.2</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>1.10.19</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.google.apis</groupId>
<artifactId>google-api-services-iam</artifactId>
<version>v1-rev310-1.25.0</version>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</dependency>
<dependency>
<groupId>io.cucumber</groupId>
<artifactId>cucumber-java8</artifactId>
<version>5.4.0</version>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
<version>4.2.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.cucumber</groupId>
<artifactId>cucumber-junit</artifactId>
<version>5.4.0</version>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.cucumber</groupId>
<artifactId>cucumber-guice</artifactId>
<version>5.4.0</version>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>3.11.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
<version>4.2.0</version>
<groupId>org.mockito</groupId>
<artifactId>mockito-inline</artifactId>
<version>3.11.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</dependency>
</dependencies>
......@@ -101,7 +90,7 @@
<configuration>
<classifier>spring-boot</classifier>
<mainClass>
org.opengroup.osdu.wks.WksServiceApplication
org.opengroup.osdu.wks.WksServiceApplicationGcp
</mainClass>
</configuration>
</execution>
......
/*
Copyright 2021 Google LLC
Copyright 2021 EPAM Systems, Inc
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Copyright 2021 Google LLC
* Copyright 2021 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.wks;
import org.opengroup.osdu.wks.provider.interfaces.SubscriptionManager;
import org.opengroup.osdu.core.gcp.di.PartitionTenantInfoFactoryBean;
import org.opengroup.osdu.core.gcp.multitenancy.StorageFactory;
import org.opengroup.osdu.core.gcp.multitenancy.TenantFactory;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.FilterType;
@SpringBootApplication
@ComponentScan(value = {"org.opengroup.osdu"})
@ComponentScan(value = {
"org.opengroup.osdu"
}, excludeFilters = {
@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = {
PartitionTenantInfoFactoryBean.class, WksServiceApplication.class, TenantFactory.class, StorageFactory.class})})
public class WksServiceApplicationGcp {
public static void main(String[] args) {
ApplicationContext context = SpringApplication
.run(WksServiceApplicationGcp.class, args);
SubscriptionManager subscriptionManager = context.getBean(SubscriptionManager.class);
subscriptionManager.subscribeRecordsChangeEvent();
SpringApplication.run(WksServiceApplicationGcp.class, args);
}
}
/*
* Copyright 2021 Google LLC
* Copyright 2021 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.wks.api;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
import org.opengroup.osdu.core.common.model.search.SearchServiceRole;
import org.opengroup.osdu.wks.pubsub.PubSubPushMessageReceiver;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@ConditionalOnProperty(value = "wks.gcp.subscriber_mode", havingValue = "push")
@Slf4j
@RestController
@RequiredArgsConstructor
@RequestMapping("/_ah/push-handlers")
public class EnqueueApi {
private final PubSubPushMessageReceiver messageReceiver;
private final RecordChangedMessages message;
@PostMapping(value = "/enqueue", produces = MediaType.APPLICATION_JSON_VALUE)
@PreAuthorize("@authorizationFilter.pubSubTaskHasRole('" + SearchServiceRole.ADMIN + "')")
public ResponseEntity<String> enqueueTask() {
return messageReceiver.receiveMessage(message);
}
@ExceptionHandler(AppException.class)
public ResponseEntity<Object> handleAppExceptions(AppException e) {
return new ResponseEntity<>(e.getError(), HttpStatus.valueOf(e.getError().getCode()));
}
}
/*
* Copyright 2021 Google LLC
* Copyright 2021 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.wks.cache;
import org.opengroup.osdu.core.common.cache.RedisCache;
import org.opengroup.osdu.core.common.model.search.IdToken;
import org.opengroup.osdu.wks.config.PropertiesConfiguration;
import org.springframework.stereotype.Component;
@Component
public class JwtCache extends RedisCache<String, IdToken> {
private static final int EXPIRED_AFTER = 59;
public JwtCache(PropertiesConfiguration configuration) {
super(configuration.getRedisHost(), configuration.getRedisPort(), EXPIRED_AFTER * 60, String.class, IdToken.class);
}
}
/*
Copyright 2021 Google LLC
Copyright 2021 EPAM Systems, Inc
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package org.opengroup.osdu.wks.config;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
@Getter
@Setter
@Configuration
@ConfigurationProperties("wks.gcp")
public class DatastorePropertiesConfiguration {
private String datastoreMappingInfoKind;
private String tenantName;
private String storageBucketName;
}
/*
* Copyright 2021 Google LLC
* Copyright 2021 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.wks.config;
import org.springframework.context.annotation.Configuration;
import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity;
import org.springframework.security.config.annotation.web.builders.HttpSecurity;
import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter;
@Configuration