Commit 1a5fb54b authored by Rustam Lotsmanenko (EPAM)'s avatar Rustam Lotsmanenko (EPAM) Committed by Riabokon Stanislav(EPAM)[GCP]
Browse files

gcp-untie-messaging-from-http(GONRG-4640)

parent 08bd2cbb
......@@ -14,6 +14,7 @@ data:
PARTITION_HOST: "{{ .Values.data.partition_host }}"
REDIS_GROUP_HOST: "{{ .Values.data.redis_group_host }}"
REDIS_STORAGE_HOST: "{{ .Values.data.redis_storage_host }}"
STORAGE_SERVICE_ACCOUNT_EMAIL: "{{ .Values.data.storage_service_account_email }}"
SPRING_PROFILES_ACTIVE: "{{ .Values.data.spring_profiles_active }}"
{{- if not .Values.conf.on_prem_enabled }}
GOOGLE_AUDIENCES: "{{ .Values.data.google_audiences }}"
......
......@@ -12,6 +12,7 @@ data:
partition_host: "http://partition"
redis_group_host: ""
redis_storage_host: ""
storage_service_account_email: "workload-storage-gcp@nice-etching-277309.iam.gserviceaccount.com"
spring_profiles_active: "gcp"
# gcp
google_audiences: ""
......
......@@ -192,6 +192,7 @@ Storage Service is compatible with App Engine Flexible Environment and Cloud Run
## Entitlements groups
Storage service account should have entitlements groups listed below:
- service.entitlements.user
- service.legal.user
## License
......
......@@ -33,6 +33,7 @@ Must have:
| `<MINIO_SECRETKEY_ENV_VARIABLE_NAME>` | ex `MINIO_SECRET_OSDU` | Minio secret env name, name of that variable not defined at the service level, the name will be received through partition service. Each tenant can have it's own ENV name value, and it must be present in ENV of Storage service| yes | - |
| `<AMQP_PASSWORD_ENV_VARIABLE_NAME>` | ex `AMQP_PASS_OSDU` | Amqp password env name, name of that variable not defined at the service level, the name will be received through partition service. Each tenant can have it's own ENV name value, and it must be present in ENV of Storage service | yes | - |
| `<AMQP_ADMIN_PASSWORD_ENV_VARIABLE_NAME>` | ex `AMQP_ADMIN_PASS_OSDU` | Amqp admin password env name, name of that variable not defined at the service level, the name will be received through partition service. Each tenant can have it's own ENV name value, and it must be present in ENV of Storage service | yes | - |
| `STORAGE_SERVICE_ACCOUNT_EMAIL` | `workload-storage@keycloak.com` | Storage service account email, used during OQM events processing |no| - |
| name | value | description | sensitive? | source |
| --- | --- | --- | --- | --- |
......
......@@ -19,6 +19,7 @@ Must have:
| --- | --- | --- | --- | --- |
| `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials |
| `SPRING_PROFILES_ACTIVE` | ex `gcp` | Spring profile that activate default configuration for GCP environment | false | - |
| `STORAGE_SERVICE_ACCOUNT_EMAIL` | `workload-storage-gcp@nice-etching-277309.iam.gserviceaccount.com` | Storage service account email, used during OQM events processing |no| - |
### Common properties for all environments
......
......@@ -14,33 +14,34 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.opengroup.osdu</groupId>
<artifactId>storage-gcp</artifactId>
<version>0.15.0-SNAPSHOT</version>
<description>Google cloud related implementation staff.</description>
<packaging>jar</packaging>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.opengroup.osdu</groupId>
<artifactId>storage-gcp</artifactId>
<version>0.15.0-SNAPSHOT</version>
<description>Google cloud related implementation staff.</description>
<packaging>jar</packaging>
<properties>
<java.version>1.8</java.version>
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
<log4j.version>2.17.1</log4j.version>
<log4j.version>2.17.1</log4j.version>
</properties>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>${log4j.version}</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-to-slf4j</artifactId>
<version>${log4j.version}</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>${log4j.version}</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-to-slf4j</artifactId>
<version>${log4j.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-dependencies</artifactId>
......@@ -84,18 +85,14 @@
</dependencies>
</dependencyManagement>
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-storage</artifactId>
<version>0.15.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-storage</artifactId>
<version>0.15.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<dependencies>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-iamcredentials</artifactId>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-gcp</artifactId>
......@@ -106,22 +103,26 @@
<artifactId>os-core-common</artifactId>
<version>${os-core-common.version}</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>storage-core</artifactId>
<version>0.15.0-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
</exclusion>
<exclusion>
<groupId>org.powermock</groupId>
<artifactId>powermock-api-mockito</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>ch.qos.logback.contrib</groupId>
<artifactId>logback-json-classic</artifactId>
<version>0.1.5</version>
</dependency>
<dependency>
<groupId>com.google.apis</groupId>
<artifactId>google-api-services-iam</artifactId>
<version>v1-rev289-1.25.0</version>
</dependency>
<dependency>
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-config</artifactId>
......@@ -138,7 +139,12 @@
<version>4.12</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-api-mockito2</artifactId>
<version>2.0.9</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
......@@ -173,7 +179,7 @@
<configuration>
<classifier>spring-boot</classifier>
<mainClass>
org.opengroup.osdu.storage.StorageApplication
org.opengroup.osdu.storage.provider.gcp.StorageApplicationGCP
</mainClass>
</configuration>
</execution>
......
/*
* Copyright 2020-2022 Google LLC
* Copyright 2020-2022 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.storage.provider.gcp;
import org.opengroup.osdu.storage.provider.gcp.messaging.config.MessagingCustomContextConfiguration;
import org.opengroup.osdu.storage.provider.gcp.web.config.WebAppMainContextConfiguration;
import org.springframework.boot.SpringBootConfiguration;
import org.springframework.boot.WebApplicationType;
import org.springframework.boot.builder.SpringApplicationBuilder;
@SpringBootConfiguration
public class StorageApplicationGCP {
/**
* Storage application starts 2 application contexts at once, one for asynchronous message receiving via OQM that has its own context configuration that is
* unbounded from request bean configurations and the second one for serving API that has request oriented bean configuration from common code.
*/
public static void main(String[] args) {
new SpringApplicationBuilder(StorageApplicationGCP.class)
.sources(StorageApplicationGCP.class).web(WebApplicationType.NONE)
.child(MessagingCustomContextConfiguration.class).web(WebApplicationType.NONE)
.sibling(WebAppMainContextConfiguration.class).web(WebApplicationType.SERVLET)
.run(args);
}
}
// Copyright 2017-2019, Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.storage.provider.gcp.cache;
import org.opengroup.osdu.core.common.model.entitlements.Groups;
import org.opengroup.osdu.core.common.cache.RedisCache;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@Component
public class GroupCache extends RedisCache<String, Groups> {
public GroupCache(@Value("${REDIS_GROUP_HOST}") final String REDIS_GROUP_HOST,@Value("${REDIS_GROUP_PORT}") final String REDIS_GROUP_PORT)
{
super(REDIS_GROUP_HOST, Integer.parseInt(REDIS_GROUP_PORT), 30, String.class,
Groups.class);
}
}
\ No newline at end of file
// Copyright 2017-2019, Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.storage.provider.gcp.cache;
import org.opengroup.osdu.core.common.cache.MultiTenantCache;
import org.opengroup.osdu.core.common.cache.RedisCache;
import org.opengroup.osdu.core.common.cache.ICache;
import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@Component("LegalTagCache")
public class LegalTagCache implements ICache<String, String> {
@Autowired
private TenantInfo tenant;
private final MultiTenantCache<String> caches;
public LegalTagCache(@Value("${REDIS_STORAGE_HOST}") final String REDIS_STORAGE_HOST,@Value("${REDIS_STORAGE_PORT}") final String REDIS_STORAGE_PORT) {
this.caches = new MultiTenantCache<>(new RedisCache<>(REDIS_STORAGE_HOST,Integer.parseInt(REDIS_STORAGE_PORT),
60 * 60,
String.class,
String.class));
}
@Override
public void put(String key, String val) {
this.partitionCache().put(key, val);
}
@Override
public String get(String key) {
return this.partitionCache().get(key);
}
@Override
public void delete(String key) {
this.partitionCache().delete(key);
}
@Override
public void clearAll() {
this.partitionCache().clearAll();
}
private ICache<String, String> partitionCache() {
return this.caches.get(String.format("%s:legalTag", this.tenant));
}
}
\ No newline at end of file
// Copyright 2017-2019, Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.storage.provider.gcp.cache;
import org.opengroup.osdu.core.common.model.storage.Schema;
import org.opengroup.osdu.core.common.cache.RedisCache;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@Component
public class SchemaCache extends RedisCache<String, Schema> {
public SchemaCache(@Value("${REDIS_STORAGE_HOST}") final String REDIS_STORAGE_HOST, @Value("${REDIS_STORAGE_PORT}") final String REDIS_STORAGE_PORT) {
super(REDIS_STORAGE_HOST, Integer.parseInt(REDIS_STORAGE_PORT), 60 * 60, String.class,
Schema.class);
}
}
\ No newline at end of file
/*
* Copyright 2021 Google LLC
* Copyright 2021 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.storage.provider.gcp.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
@Configuration
@ConfigurationProperties
@Data
public class GcpAppServiceConfig {
private String pubsubSearchTopic = "records-changed";
}
/*
* Copyright 2020-2022 Google LLC
* Copyright 2020-2022 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.storage.provider.gcp.messaging.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
@Configuration
@ConfigurationProperties
@Data
public class MessagingConfigurationProperties {
private String legalTagsChangedTopicName;
private String legalTagsChangedSubscriptionName;
private String storageServiceAccountEmail;
}
/*
* Copyright 2020-2022 Google LLC
* Copyright 2020-2022 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.storage.provider.gcp.messaging.config;
import java.util.Arrays;
import javax.annotation.PostConstruct;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.opengroup.osdu.core.common.multitenancy.TenantInfoFactory;
import org.opengroup.osdu.storage.StorageApplication;
import org.opengroup.osdu.storage.provider.gcp.web.config.WebAppMainContextConfiguration;
import org.springframework.beans.factory.config.BeanFactoryPostProcessor;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.FilterType;
import org.springframework.context.annotation.PropertySource;
/**
* This context configuration serves for message receiving configuration not bounded to request scope.
* Also, configured context does not serve incoming requests.
*/
@Slf4j
@Configuration
@EnableConfigurationProperties
@PropertySource("classpath:application.properties")
@RequiredArgsConstructor
@ComponentScan(value = {
"org.opengroup.osdu"
},
excludeFilters = {
@ComponentScan.Filter(
type = FilterType.ASSIGNABLE_TYPE,
value = {
StorageApplication.class,
WebAppMainContextConfiguration.class,
TenantInfoFactory.class
}
)
}
)
public class MessagingCustomContextConfiguration {
private final ApplicationContext applicationContext;
@PostConstruct
public void setUp() {
log.debug("Messaging context initialized with id: {} ", applicationContext.getId());
log.debug("Messaging context status: {} ", applicationContext);
String[] allBeansNames = applicationContext.getBeanDefinitionNames();
log.debug("Messaging context beans definitions: {} ", Arrays.toString(allBeansNames));
}
@Bean
public static BeanFactoryPostProcessor beanFactoryPostProcessor() {
return new ThreadBeanFactoryPostProcessor();
}
}
/*
* Copyright 2020-2022 Google LLC
* Copyright 2020-2022 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.storage.provider.gcp.messaging.config;
import org.opengroup.osdu.storage.provider.gcp.messaging.thread.ThreadScope;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.config.BeanFactoryPostProcessor;
import org.springframework.beans.factory.config.ConfigurableListableBeanFactory;
public class ThreadBeanFactoryPostProcessor implements BeanFactoryPostProcessor {
public static final String SCOPE_THREAD = "scope_thread";
@Override
public void postProcessBeanFactory(ConfigurableListableBeanFactory configurableListableBeanFactory) throws BeansException {
configurableListableBeanFactory.registerScope(SCOPE_THREAD, new ThreadScope());
}
}
// Copyright 2017-2019, Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.storage.provider.gcp.jobs;
import static java.util.Collections.singletonList;
import java.util.AbstractMap;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.opengroup.osdu.core.common.model.legal.jobs.*;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.storage.provider.gcp.cache.LegalTagCache;
import org.opengroup.osdu.storage.provider.interfaces.IMessageBus;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.springframework.beans.factory.annotation.Autowired;
import com.google.cloud.datastore.Cursor;
import org.opengroup.osdu.storage.logging.StorageAuditLogger;
import org.opengroup.osdu.core.common.model.legal.LegalCompliance;
import org.opengroup.osdu.core.common.model.indexer.OperationType;
import org.opengroup.osdu.core.common.model.storage.PubSubInfo;
import org.opengroup.osdu.core.common.model.storage.RecordMetadata;
import org.opengroup.osdu.core.common.model.storage.RecordState;
import org.opengroup.osdu.storage.provider.interfaces.IRecordsMetadataRepository;
import org.springframework.stereotype.Service;
import org.opengroup.osdu.core.common.model.legal.jobs.ComplianceUpdateStoppedException;
@Service
public class LegalComplianceChangeServiceGcpImpl implements ILegalComplianceChangeService {
@Autowired
private IRecordsMetadataRepository recordsRepo;