Merge branch 'anthos-master' into 'integration-master'

GONRG-1423 Merge Anthos PoCs into (master) codebase

See merge request go3-nrg/platform/System/search!41

pom.xml

@@ -62,6 +62,7 @@
         <module>provider/search-aws</module>
         <module>provider/search-azure</module>
         <module>provider/search-ibm</module>
+        <module>provider/search-reference</module>
     </modules>
     <dependencyManagement>
         <dependencies>

provider/search-reference/docker/Dockerfile

+FROM openjdk:8-slim
+WORKDIR /app
+COPY ../target/search-reference-0.0.5-SNAPSHOT-spring-boot.jar search-reference.jar
+# Run the web service on container startup.
+CMD java -Djava.security.egd=file:/dev/./urandom -Dserver.port=8080 -jar /app/search-reference.jar
\ No newline at end of file

provider/search-reference/docker/docker-compose.yml

+version: "3"
+services:
+  os-search-app:
+    build:
+      args:
+        JAR_FILE: target/search-reference-0.0.5-SNAPSHOT-spring-boot.jar
+      context: ""
+      dockerfile: ../Dockerfile
+    image: gcr.io/osdu-anthos-02/os-search/search-reference
+    ports:
+      - "8080:8080"

provider/search-reference/kubernetes/deployments/deployment-os-search-service.yml

+apiVersion: v1
+data:
+  MONGO_DB_URL: ${MONGO_DB_URL}
+  MONGO_DB_USER: ${MONGO_DB_USER}
+  ELASTIC_DATASTORE_KIND: ${ELASTIC_DATASTORE_KIND}
+  ELASTIC_DATASTORE_ID: ${ELASTIC_DATASTORE_ID}
+  REDIS_SEARCH_HOST: ${REDIS_SEARCH_HOST}
+  REDIS_SEARCH_PORT: ${REDIS_SEARCH_PORT}
+  REDIS_GROUP_HOST: ${REDIS_GROUP_HOST}
+  spring.jackson.deserialization.fail-on-unknown-properties: "true"
+  security.https.certificate.trust: "false"
+  SCHEMA_CACHE_EXPIRATION: ${SCHEMA_CACHE_EXPIRATION}
+  INDEX_CACHE_EXPIRATION: ${INDEX_CACHE_EXPIRATION}
+  ELASTIC_CACHE_EXPIRATION: ${ELASTIC_CACHE_EXPIRATION}
+  CURSOR_CACHE_EXPIRATION: ${CURSOR_CACHE_EXPIRATION}
+  KINDS_CACHE_EXPIRATION: ${KINDS_CACHE_EXPIRATION}
+  ATTRIBUTES_CACHE_EXPIRATION: ${ATTRIBUTES_CACHE_EXPIRATION}
+  KINDS_REDIS_DATABASE: ${KINDS_REDIS_DATABASE}
+  CRON_INDEX_CLEANUP_THRESHOLD_DAYS: ${CRON_INDEX_CLEANUP_THRESHOLD_DAYS}
+  CRON_EMPTY_INDEX_CLEANUP_THRESHOLD_DAYS: ${CRON_EMPTY_INDEX_CLEANUP_THRESHOLD_DAYS}
+  QUERY_DEFAULT_LIMIT: ${QUERY_DEFAULT_LIMIT}
+  QUERY_LIMIT_MAXIMUM: ${QUERY_LIMIT_MAXIMUM}
+  AGGREGATION_SIZE: ${AGGREGATION_SIZE}
+  ENVIRONMENT: ${ENVIRONMENT}
+kind: ConfigMap
+metadata:
+  labels:
+    app: search-reference
+  name: search-config
+  namespace: default
+---
+apiVersion: "apps/v1"
+kind: "Deployment"
+metadata:
+  name: "search-reference"
+  namespace: "default"
+  labels:
+    app: "search-reference"
+spec:
+  selector:
+    matchLabels:
+      app: "search-reference"
+  template:
+    metadata:
+      labels:
+        app: "search-reference"
+    spec:
+      containers:
+       - env:
+          - name: MONGO_DB_URL
+            valueFrom:
+              configMapKeyRef:
+                key: MONGO_DB_URL
+                name: search-config
+          - name: MONGO_DB_USER
+            valueFrom:
+              configMapKeyRef:
+                key: MONGO_DB_USER
+                name: search-config
+          - name: ELASTIC_DATASTORE_KIND
+            valueFrom:
+              configMapKeyRef:
+                key: ELASTIC_DATASTORE_KIND
+                name: search-config
+          - name: ELASTIC_DATASTORE_ID
+            valueFrom:
+              configMapKeyRef:
+                key: ELASTIC_DATASTORE_ID
+                name: search-config
+          - name: MONGO_DB_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: search-secret
+                key: mongo.db.password
+          - name: ELASTIC_HOST
+            valueFrom:
+              secretKeyRef:
+                name: search-secret
+                key: reference.elastic.host
+          - name: ELASTIC_PORT
+            valueFrom:
+              secretKeyRef:
+                name: search-secret
+                key: reference.elastic.port
+          - name: ELASTIC_USER_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: search-secret
+                key: reference.elastic.user.password
+          - name: REDIS_GROUP_HOST
+            valueFrom:
+              configMapKeyRef:
+                key: REDIS_GROUP_HOST
+                name: search-config
+          - name: REDIS_SEARCH_HOST
+            valueFrom:
+              configMapKeyRef:
+                key: REDIS_SEARCH_HOST
+                name: search-config
+          - name: REDIS_SEARCH_PORT
+            valueFrom:
+              configMapKeyRef:
+                key: REDIS_SEARCH_PORT
+                name: search-config
+          - name: spring.jackson.deserialization.fail-on-unknown-properties
+            valueFrom:
+              configMapKeyRef:
+                key: spring.jackson.deserialization.fail-on-unknown-properties
+                name: search-config
+          - name: SECURITY_HTTPS_CERTIFICATE_TRUST
+            valueFrom:
+              configMapKeyRef:
+                key: security.https.certificate.trust
+                name: search-config
+          - name: SCHEMA_CACHE_EXPIRATION
+            valueFrom:
+              configMapKeyRef:
+                key: SCHEMA_CACHE_EXPIRATION
+                name: search-config
+          - name: INDEX_CACHE_EXPIRATION
+            valueFrom:
+              configMapKeyRef:
+                key: INDEX_CACHE_EXPIRATION
+                name: search-config
+          - name: ELASTIC_CACHE_EXPIRATION
+            valueFrom:
+              configMapKeyRef:
+                key: ELASTIC_CACHE_EXPIRATION
+                name: search-config
+          - name: CURSOR_CACHE_EXPIRATION
+            valueFrom:
+              configMapKeyRef:
+                key: CURSOR_CACHE_EXPIRATION
+                name: search-config
+          - name: KINDS_CACHE_EXPIRATION
+            valueFrom:
+              configMapKeyRef:
+                key: KINDS_CACHE_EXPIRATION
+                name: search-config
+          - name: ATTRIBUTES_CACHE_EXPIRATION
+            valueFrom:
+              configMapKeyRef:
+                key: ATTRIBUTES_CACHE_EXPIRATION
+                name: search-config
+          - name: KINDS_REDIS_DATABASE
+            valueFrom:
+              configMapKeyRef:
+                key: KINDS_REDIS_DATABASE
+                name: search-config
+          - name: CRON_INDEX_CLEANUP_THRESHOLD_DAYS
+            valueFrom:
+              configMapKeyRef:
+                key: CRON_INDEX_CLEANUP_THRESHOLD_DAYS
+                name: search-config
+          - name: CRON_EMPTY_INDEX_CLEANUP_THRESHOLD_DAYS
+            valueFrom:
+              configMapKeyRef:
+                key: CRON_EMPTY_INDEX_CLEANUP_THRESHOLD_DAYS
+                name: search-config
+          - name: QUERY_DEFAULT_LIMIT
+            valueFrom:
+              configMapKeyRef:
+                key: QUERY_DEFAULT_LIMIT
+                name: search-config
+          - name: QUERY_LIMIT_MAXIMUM
+            valueFrom:
+              configMapKeyRef:
+                key: QUERY_LIMIT_MAXIMUM
+                name: search-config
+          - name: AGGREGATION_SIZE
+            valueFrom:
+              configMapKeyRef:
+                key: AGGREGATION_SIZE
+                name: search-config
+          - name: ENVIRONMENT
+            valueFrom:
+              configMapKeyRef:
+                key: ENVIRONMENT
+                name: search-config
+         name: "search-reference-sha256-1"
+         image: "gcr.io/osdu-anthos-02/os-search/search-reference@sha256:57448c37d7716d44f32db7c17b201d685b6d1843058fd956eb22c174d0f7fcb6"
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: search-reference
+  namespace: default
+spec:
+  ports:
+    -   protocol: TCP
+        port: 80
+        targetPort: 8080
+  selector:
+    app: search-reference
+  type: LoadBalancer
+---
+apiVersion: v1
+kind: Secret
+type: Opaque
+data:
+  mongo.db.password: ${MONGO_DB_PASSWORD}
+  reference.elastic.user.password: ${ELASTIC_USER_PASSWORD}
+  reference.elastic.host: ${ELASTIC_HOST}
+  reference.elastic.port: ${ELASTIC_PORT}
+metadata:
+  name: search-secret
+  namespace: default

provider/search-reference/kubernetes/deployments/gke-deploy.yml

+steps:
+# deploy container image to GKE
+  - name: "gcr.io/cloud-builders/gke-deploy"
+    args:
+      - run
+      - --filename=deployment-os-search-service.yml
+      - --location=us-central1-c
+      - --cluster=anthos-test
\ No newline at end of file

provider/search-reference/pom.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.opengroup.osdu</groupId>
+        <artifactId>os-search</artifactId>
+        <version>0.0.5-SNAPSHOT</version>
+        <relativePath>../../pom.xml</relativePath>
+    </parent>
+
+    <groupId>org.opengroup.osdu</groupId>
+    <artifactId>search-reference</artifactId>
+    <version>0.0.5-SNAPSHOT</version>
+    <description>MongoDB implementation of Search service APIs</description>
+    <packaging>jar</packaging>
+
+    <properties>
+        <maven.compiler.target>1.8</maven.compiler.target>
+        <maven.compiler.source>1.8</maven.compiler.source>
+        <maven.war.plugin>2.6</maven.war.plugin>
+        <appengine.maven.plugin>1.0.0</appengine.maven.plugin>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <failOnMissingWebXml>false</failOnMissingWebXml>
+        <project.main.basedir>${project.parent.basedir}</project.main.basedir>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.opengroup.osdu</groupId>
+            <artifactId>search-core</artifactId>
+            <version>0.0.5-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>org.opengroup.osdu</groupId>
+            <artifactId>os-core-common</artifactId>
+            <version>0.3.8</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-data-mongodb</artifactId>
+            <version>2.1.7.RELEASE</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-test</artifactId>
+            <version>2.1.7.RELEASE</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.powermock</groupId>
+            <artifactId>powermock-api-mockito2</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.powermock</groupId>
+            <artifactId>powermock-module-junit4</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-all</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <!--Elasticsearch-->
+        <dependency>
+            <groupId>org.elasticsearch</groupId>
+            <artifactId>elasticsearch</artifactId>
+            <version>6.6.2</version>
+        </dependency>
+        <dependency>
+            <groupId>org.elasticsearch.client</groupId>
+            <artifactId>elasticsearch-rest-client</artifactId>
+            <version>6.6.2</version>
+        </dependency>
+        <dependency>
+            <groupId>org.elasticsearch.client</groupId>
+            <artifactId>elasticsearch-rest-high-level-client</artifactId>
+            <version>6.6.2</version>
+        </dependency>
+        <dependency>
+            <groupId>org.locationtech.jts.io</groupId>
+            <artifactId>jts-io-common</artifactId>
+            <version>1.15.0</version>
+        </dependency>
+      <dependency>
+        <groupId>org.projectlombok</groupId>
+        <artifactId>lombok</artifactId>
+        <version>1.18.16</version>
+      </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+                <configuration>
+                    <classifier>spring-boot</classifier>
+                    <mainClass>
+                        org.opengroup.osdu.search.SearchApplication
+                    </mainClass>
+                    <profiles>
+                        <profile>
+                            <id>local</id>
+                            <activation>
+                                <activeByDefault>true</activeByDefault>
+                            </activation>
+                            <properties>
+                                <spring.profiles.active>local</spring.profiles.active>
+                            </properties>
+                        </profile>
+                        <profile>
+                            <id>dev</id>
+                            <properties>
+                                <spring.profiles.active>dev</spring.profiles.active>
+                            </properties>
+                        </profile>
+                    </profiles>
+                </configuration>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>repackage</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>
\ No newline at end of file

provider/search-reference/src/main/java/org/opengroup/osdu/search/provider/reference/SearchReferenceApplication.java

+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.search.provider.reference;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.context.annotation.ComponentScan;
+
+@ComponentScan({"org.opengroup.osdu"})
+@SpringBootApplication(
+    exclude = {
+        org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration.class,
+        org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration.class})
+public class SearchReferenceApplication {
+  public static void main(String[] args) {
+    SpringApplication.run(SearchReferenceApplication.class, args);
+  }
+}

provider/search-reference/src/main/java/org/opengroup/osdu/search/provider/reference/cache/CursorCacheImpl.java

+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.search.provider.reference.cache;
+
+import org.opengroup.osdu.core.common.cache.RedisCache;
+import org.opengroup.osdu.core.common.model.search.CursorSettings;
+import org.opengroup.osdu.search.cache.CursorCache;
+import org.opengroup.osdu.search.config.SearchConfigurationProperties;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class CursorCacheImpl extends RedisCache<String, CursorSettings> implements CursorCache {
+
+  @Autowired
+  public CursorCacheImpl(SearchConfigurationProperties searchConfigurationProperties) {
+    super(searchConfigurationProperties.getSearchHost(),
+        Integer.parseInt(searchConfigurationProperties.getRedisSearchPort()),
+        searchConfigurationProperties.getCursorCacheExpiration(), String.class,
+        CursorSettings.class);
+  }
+}

provider/search-reference/src/main/java/org/opengroup/osdu/search/provider/reference/cache/ElasticCredentialsCache.java

+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.search.provider.reference.cache;
+
+import org.opengroup.osdu.core.common.cache.RedisCache;
+import org.opengroup.osdu.core.common.model.search.ClusterSettings;
+import org.opengroup.osdu.search.config.SearchConfigurationProperties;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class ElasticCredentialsCache extends RedisCache<String, ClusterSettings> {
+
+  @Autowired
+  public ElasticCredentialsCache(final SearchConfigurationProperties configurationProperties) {
+    super(configurationProperties.getRedisSearchHost(),
+        Integer.parseInt(configurationProperties.getRedisSearchPort()),
+        configurationProperties.getElasticCacheExpiration() * 60,
+        String.class,
+        ClusterSettings.class);
+  }
+}

provider/search-reference/src/main/java/org/opengroup/osdu/search/provider/reference/cache/FieldTypeMappingCache.java

+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.search.provider.reference.cache;
+
+import java.util.HashSet;
+import org.opengroup.osdu.core.common.cache.RedisCache;
+import org.opengroup.osdu.search.config.SearchConfigurationProperties;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class FieldTypeMappingCache extends RedisCache<String, HashSet> {
+
+  @Autowired
+  public FieldTypeMappingCache(final SearchConfigurationProperties configurationProperties) {
+    super(configurationProperties.getRedisSearchHost(),
+        Integer.parseInt(configurationProperties.getRedisSearchPort()),
+        1440 * 60,
+        String.class, HashSet.class);
+  }
+}

provider/search-reference/src/main/java/org/opengroup/osdu/search/provider/reference/config/MongoDBConfig.java

+package org.opengroup.osdu.search.provider.reference.config;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@ConfigurationProperties
+@Getter
+@Setter
+public class MongoDBConfig {
+
+  private String mongoDbUrl;
+  private String mongoDbUser;
+  private String mongoDbPassword;
+
+}

provider/search-reference/src/main/java/org/opengroup/osdu/search/provider/reference/di/EntitlementsClientFactory.java

+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.search.provider.reference.di;
+
+import org.opengroup.osdu.core.common.entitlements.EntitlementsAPIConfig;
+import org.opengroup.osdu.core.common.entitlements.EntitlementsFactory;