Commit a2d8b251 authored by David Diederich's avatar David Diederich
Browse files

Merge remote-tracking branch 'origin/master' into release/0.4

 Conflicts:
	.gitlab-ci.yml
parents ad13291e ec76c2a0
Pipeline #18799 failed
......@@ -14,13 +14,14 @@ variables:
GCP_DOMAIN: cloud.slb-ds.com
GCP_STORAGE_URL: https://osdu-indexer-dot-opendes.appspot.com/api/storage/v2/
OSDU_GCP_BUILD_SUBDIR: provider/indexer-gcp
OSDU_GCP_INT_TEST_SUBDIR: testing/indexer-test-gcp
OSDU_GCP_APPLICATION_NAME: os-indexer
OSDU_GCP_PROJECT: nice-etching-277309
OSDU_GCP_TENANT_NAME: osdu
OSDU_GCP_STORAGE_SCHEMA_HOST: https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/schemas
OSDU_GCP_SERVICE: indexer
OSDU_GCP_VENDOR: gcp
OSDU_GCP_SERVICE_ACCOUNT: osdu-gcp-sa@nice-etching-277309.iam.gserviceaccount.com
OSDU_SECURITY_HTTPS_CERTIFICATE_TRUST: 'true'
OSDU_GCP_STORAGE_RECORDS_BATCH_SIZE: 20
OSDU_GCP_DATA_GROUP: osdu
OSDU_GCP_ENV_VARS: AUTHORIZE_API=$OSDU_GCP_ENTITLEMENTS_URL,GOOGLE_CLOUD_PROJECT=$OSDU_GCP_PROJECT,REDIS_SEARCH_HOST=$REDIS_SEARCH_HOST,REDIS_GROUP_HOST=$REDIS_GROUP_HOST,SECURITY_HTTPS_CERTIFICATE_TRUST=$OSDU_SECURITY_HTTPS_CERTIFICATE_TRUST,INDEXER_HOST=$OSDU_GCP_INDEXER_HOST,STORAGE_QUERY_RECORD_HOST=$OSDU_GCP_STORAGE_QUERY_RECORD_HOST,STORAGE_SCHEMA_HOST=$OSDU_GCP_STORAGE_SCHEMA_HOST,STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=$OSDU_GCP_STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST,STORAGE_HOSTNAME=$OSDU_GCP_STORAGE_HOSTNAME,STORAGE_RECORDS_BATCH_SIZE=$OSDU_GCP_STORAGE_RECORDS_BATCH_SIZE,INDEXER_QUEUE_HOST=$OSDU_GCP_INDEXER_QUEUE_HOST,LEGALTAG_API=$OSDU_GCP_LEGALTAG_API,CRS_API=$OSDU_GCP_CRS_API,DATA_GROUP=$OSDU_GCP_DATA_GROUP,GOOGLE_AUDIENCES=$GOOGLE_AUDIENCE,INDEXER_QUE_SERVICE_MAIL=$OSDU_GCP_SERVICE_ACCOUNT --vpc-connector=$OSDU_GCP_VPC_CONNECTOR
IBM_BUILD_SUBDIR: provider/indexer-ibm
IBM_INT_TEST_SUBDIR: testing/indexer-test-ibm
......@@ -40,7 +41,7 @@ include:
- project: "osdu/platform/ci-cd-pipelines"
ref: "v0.4.2"
file: "scanners/fossa.yml"
file: "scanners/fossa-maven.yml"
- project: "osdu/platform/ci-cd-pipelines"
ref: "v0.4.2"
......@@ -64,7 +65,7 @@ include:
- project: 'osdu/platform/ci-cd-pipelines'
ref: "v0.4.2"
file: 'cloud-providers/osdu-gcp.yml'
file: 'cloud-providers/osdu-gcp-cloudrun.yml'
aws-test-java:
......
......@@ -7,14 +7,20 @@
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<gitlab-server>community-maven-via-job-token</gitlab-server>
<repo.releases.id>community-maven-repo</repo.releases.id>
<publish.snapshots.id>community-maven-via-job-token</publish.snapshots.id>
<publish.releases.id>community-maven-via-job-token</publish.releases.id>
<repo.releases.url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</repo.releases.url>
<publish.snapshots.url>https://community.opengroup.org/api/v4/projects/25/packages/maven</publish.snapshots.url>
<publish.releases.url>https://community.opengroup.org/api/v4/projects/25/packages/maven</publish.releases.url>
</properties>
</profile>
<profile>
<!-- This profile uses a personal token to authenticate, which is useful for local or manual runs. The
presence of the COMMUNITY_MAVEN_TOKEN variable triggers this and overrides the CI Token
based authentication -->
presence of the COMMUNITY_MAVEN_TOKEN variable triggers this and overrides the CI Token
based authentication -->
<id>GitLab-Authenticate-With-Private-Token</id>
<activation>
<property>
......@@ -22,7 +28,13 @@
</property>
</activation>
<properties>
<gitlab-server>community-maven-via-private-token</gitlab-server>
<repo.releases.id>community-maven-repo</repo.releases.id>
<publish.snapshots.id>community-maven-via-private-token</publish.snapshots.id>
<publish.releases.id>community-maven-via-private-token</publish.releases.id>
<repo.releases.url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</repo.releases.url>
<publish.snapshots.url>https://community.opengroup.org/api/v4/projects/25/packages/maven</publish.snapshots.url>
<publish.releases.url>https://community.opengroup.org/api/v4/projects/25/packages/maven</publish.releases.url>
</properties>
</profile>
</profiles>
......@@ -62,4 +74,5 @@
</configuration>
</server>
</servers>
</settings>
......@@ -11,6 +11,7 @@ The following software have components provided under the terms of this license:
- Cobertura code coverage (from http://cobertura.sourceforge.net)
- Plexus :: Default Container (from )
- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils)
- StAX (from http://stax.codehaus.org/)
- oro (from )
========================================================================
......@@ -165,11 +166,8 @@ The following software have components provided under the terms of this license:
- Apache Commons BeanUtils (from http://commons.apache.org/proper/commons-beanutils/)
- Apache Commons CLI (from http://commons.apache.org/proper/commons-cli/)
- Apache Commons Codec (from http://commons.apache.org/proper/commons-codec/)
- Apache Commons Collections (from http://commons.apache.org/proper/commons-collections/)
- Apache Commons Lang (from http://commons.apache.org/proper/commons-lang/)
- Apache Commons Logging (from http://commons.apache.org/proper/commons-logging/)
- Apache Commons Text (from http://commons.apache.org/proper/commons-text/)
- Apache Commons Validator (from http://commons.apache.org/proper/commons-validator/)
- Apache Commons Validator (from http://commons.apache.org/proper/commons-validator/)
- Apache Geronimo JMS Spec 2.0 (from http://geronimo.apache.org/maven/${siteId}/${version})
- Apache HttpAsyncClient (from http://hc.apache.org/httpcomponents-asyncclient)
......@@ -196,6 +194,8 @@ The following software have components provided under the terms of this license:
- Asynchronous Http Client (from )
- Asynchronous Http Client Netty Utils (from )
- AutoValue (from )
- Azure AD Spring Security Integration Spring Boot Starter (from https://github.com/Microsoft/azure-spring-boot)
- Azure Metrics Spring Boot Starter (from https://github.com/Microsoft/azure-spring-boot)
- Bean Validation API (from http://beanvalidation.org)
- Byte Buddy (without dependencies) (from )
- Byte Buddy Java agent (from )
......@@ -204,8 +204,6 @@ The following software have components provided under the terms of this license:
- CloudWatch Metrics for AWS Java SDK (from https://aws.amazon.com/sdkforjava)
- Cobertura code coverage (from http://cobertura.sourceforge.net)
- Commons Digester (from http://commons.apache.org/digester/)
- Commons Digester (from http://commons.apache.org/digester/)
- Commons IO (from http://commons.apache.org/io/)
- Commons IO (from http://commons.apache.org/io/)
- Commons Lang (from http://commons.apache.org/lang/)
- Converter: Jackson (from )
......@@ -248,6 +246,7 @@ The following software have components provided under the terms of this license:
- IBM COS SDK For Java (from https://github.com/ibm/ibm-cos-sdk-java)
- IBM COS SDK for Java - Core (from https://github.com/ibm/ibm-cos-sdk-java)
- Identity and Access Management (IAM) API v1-rev247-1.23.0 (from )
- IntelliJ IDEA Annotations (from http://www.jetbrains.org)
- J2ObjC Annotations (from https://github.com/google/j2objc/)
- J2ObjC Annotations (from https://github.com/google/j2objc/)
- JBoss Logging 3 (from http://www.jboss.org)
......@@ -263,29 +262,43 @@ The following software have components provided under the terms of this license:
- Jackson (from http://jackson.codehaus.org)
- Jackson 2 extensions to the Google HTTP Client Library for Java. (from https://github.com/google/google-http-java-client.git/google-http-client-jackson2)
- Jackson dataformat: CBOR (from http://github.com/FasterXML/jackson-dataformats-binary)
- Jackson dataformat: CBOR (from http://github.com/FasterXML/jackson-dataformats-binary)
- Jackson datatype: JSR310 (from http://wiki.fasterxml.com/JacksonModuleJSR310)
- Jackson datatype: JSR310 (from http://wiki.fasterxml.com/JacksonModuleJSR310)
- Jackson extensions to the Google HTTP Client Library for Java. (from )
- Jackson-annotations (from http://github.com/FasterXML/jackson)
- Jackson-annotations (from http://github.com/FasterXML/jackson)
- Jackson-core (from https://github.com/FasterXML/jackson-core)
- Jackson-dataformat-Smile (from http://github.com/FasterXML/jackson-dataformat-smile)
- Jackson-dataformat-Smile (from http://github.com/FasterXML/jackson-dataformat-smile)
- Jackson-dataformat-XML (from http://wiki.fasterxml.com/JacksonExtensionXmlDataBinding)
- Jackson-dataformat-YAML (from https://github.com/FasterXML/jackson)
- Jackson-dataformat-YAML (from https://github.com/FasterXML/jackson)
- Jackson-datatype-Joda (from http://wiki.fasterxml.com/JacksonModuleJoda)
- Jackson-datatype-jdk8 (from )
- Jackson-datatype-jdk8 (from )
- Jackson-module-Afterburner (from http://wiki.fasterxml.com/JacksonHome)
- Jackson-module-JAXB-annotations (from http://wiki.fasterxml.com/JacksonJAXBAnnotations)
- Jackson-module-JAXB-annotations (from http://wiki.fasterxml.com/JacksonJAXBAnnotations)
- Jackson-module-parameter-names (from )
- Jackson-module-parameter-names (from )
- Jakarta Bean Validation API (from https://beanvalidation.org)
- Java Libraries for Amazon Simple WorkFlow (from https://aws.amazon.com/sdkforjava)
- Java Native Access (from https://github.com/java-native-access/jna)
- Java Native Access Platform (from https://github.com/java-native-access/jna)
- Java Servlet API (from http://servlet-spec.java.net)
- Java UUID Generator (from http://wiki.fasterxml.com/JugHome)
- Javassist (from http://www.javassist.org/)
- Javassist (from http://www.javassist.org/)
- Jetty Server (from )
- Jetty Utilities (from )
- Joda-Time (from http://www.joda.org/joda-time/)
- Json Path (from https://github.com/jayway/JsonPath)
- KeePassJava2 :: All (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2)
- KeePassJava2 :: DOM (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2-dom)
- KeePassJava2 :: JAXB (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2-jaxb)
- KeePassJava2 :: KDB (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2-kdb)
- KeePassJava2 :: KDBX (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2-kdbx)
- KeePassJava2 :: Simple (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2-simple)
- Lucene Common Analyzers (from )
- Lucene Common Analyzers (from )
- Lucene Core (from )
......@@ -338,15 +351,15 @@ The following software have components provided under the terms of this license:
- Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java)
- Mockito (from http://mockito.org)
- Mockito (from http://www.mockito.org)
- Mockito (from http://mockito.org)
- Mockito (from http://mockito.org)
- Mockito (from http://www.mockito.org)
- Mojo's Maven plugin for Cobertura (from http://mojo.codehaus.org/cobertura-maven-plugin/)
- Netty Reactive Streams Implementation (from )
- Netty/Buffer (from http://netty.io/)
- Netty/Codec (from )
- Netty/Codec/DNS (from )
- Netty/Codec/HTTP (from )
- Netty/Codec/HTTP2 (from )
- Netty/Codec/Socks (from )
......@@ -354,7 +367,6 @@ The following software have components provided under the terms of this license:
- Netty/Handler (from )
- Netty/Handler/Proxy (from )
- Netty/Resolver (from )
- Netty/Resolver/DNS (from )
- Netty/TomcatNative [BoringSSL - Static] (from )
- Netty/Transport (from http://netty.io/)
- Netty/Transport/Native/Unix/Common (from )
......@@ -376,6 +388,7 @@ The following software have components provided under the terms of this license:
- OpenCensus (from https://github.com/census-instrumentation/opencensus-java)
- OpenCensus (from https://github.com/census-instrumentation/opencensus-java)
- OpenCensus (from https://github.com/census-instrumentation/opencensus-java)
- PWDB :: Database (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/database)
- Plexus :: Default Container (from )
- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils)
- Plexus Default Interactivity Handler (from )
......@@ -387,6 +400,7 @@ The following software have components provided under the terms of this license:
- Reactive Streams Netty driver (from https://github.com/reactor/reactor-netty)
- Retrofit (from )
- Servlet Specification 2.5 API (from )
- Simple XML (from http://simple.sourceforge.net)
- SnakeYAML (from http://www.snakeyaml.org)
- Spatial4J (from http://www.locationtech.org/projects/locationtech.spatial4j)
- Spring AOP (from https://github.com/spring-projects/spring-framework)
......@@ -426,9 +440,12 @@ The following software have components provided under the terms of this license:
- Spring Transaction (from https://github.com/spring-projects/spring-framework)
- Spring Web (from https://github.com/spring-projects/spring-framework)
- Spring Web MVC (from https://github.com/spring-projects/spring-framework)
- StAX (from http://stax.codehaus.org/)
- StAX API (from http://stax.codehaus.org/)
- T-Digest (from https://github.com/tdunning/t-digest)
- Woodstox (from https://github.com/FasterXML/woodstox)
- Xerces2-j (from https://xerces.apache.org/xerces2-j/)
- aalto-xml (from )
- aggs-matrix-stats (from https://github.com/elastic/elasticsearch)
- aggs-matrix-stats (from https://github.com/elastic/elasticsearch)
- cli (from https://github.com/elastic/elasticsearch)
......@@ -460,6 +477,7 @@ The following software have components provided under the terms of this license:
- jackson-databind (from http://github.com/FasterXML/jackson)
- java-cloudant (from https://cloudant.com)
- java-cloudant (from https://cloudant.com)
- javatuples (from http://www.javatuples.org)
- javax.inject (from http://code.google.com/p/atinject/)
- javax.ws.rs-api (from http://jax-rs-spec.java.net)
- jersey-container-servlet (from git://java.net/jersey~code/project/jersey-container-servlet)
......@@ -490,7 +508,6 @@ The following software have components provided under the terms of this license:
- powermock-reflect (from )
- proto-google-cloud-datastore-v1 (from https://github.com/googleapis/api-client-staging)
- proton-j (from )
- proton-j (from )
- rank-eval (from https://github.com/elastic/elasticsearch)
- rank-eval (from https://github.com/elastic/elasticsearch)
- rest (from https://github.com/elastic/elasticsearch)
......@@ -498,7 +515,6 @@ The following software have components provided under the terms of this license:
- rest-high-level (from https://github.com/elastic/elasticsearch)
- rest-high-level (from https://github.com/elastic/elasticsearch)
- rxjava (from https://github.com/ReactiveX/RxJava)
- rxjava (from https://github.com/ReactiveX/RxJava)
- secure-sm (from https://github.com/elastic/elasticsearch)
- secure-sm (from https://github.com/elastic/elasticsearch)
- spring-security-config (from http://spring.io/spring-security)
......@@ -541,10 +557,10 @@ The following software have components provided under the terms of this license:
- Lucene Common Analyzers (from )
- Plexus :: Default Container (from )
- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils)
- StAX (from http://stax.codehaus.org/)
- Stax2 API (from http://github.com/FasterXML/stax2-api)
- jersey-ext-bean-validation (from )
- jersey-spring4 (from )
- oro (from )
========================================================================
BSD-3-Clause
......@@ -608,12 +624,6 @@ CC-BY-2.5
The following software have components provided under the terms of this license:
- Checker Qual (from https://checkerframework.org)
========================================================================
CC-BY-3.0
========================================================================
The following software have components provided under the terms of this license:
- FindBugs-jsr305 (from http://findbugs.sourceforge.net/)
========================================================================
......@@ -648,7 +658,6 @@ The following software have components provided under the terms of this license:
- HK2 core module (from )
- HK2 module of HK2 itself (from )
- JavaBeans Activation Framework API jar (from )
- JavaBeans(TM) Activation Framework (from http://java.sun.com/javase/technologies/desktop/javabeans/jaf/index.jsp)
- Run Level Service (from )
- ServiceLocator Default Implementation (from git://java.net/hk2~git/hk2-locator)
- Servlet Specification 2.5 API (from )
......@@ -673,6 +682,7 @@ The following software have components provided under the terms of this license:
- Java Architecture For XML Binding (from )
- Java Servlet API (from http://servlet-spec.java.net)
- JavaBeans Activation Framework (from )
- JavaBeans(TM) Activation Framework (from http://java.sun.com/javase/technologies/desktop/javabeans/jaf/index.jsp)
- JavaMail API (from )
- OSGi resource locator (from )
- jersey-ext-bean-validation (from )
......@@ -817,7 +827,6 @@ LGPL-2.1-or-later
========================================================================
The following software have components provided under the terms of this license:
- Java Native Access (from https://github.com/java-native-access/jna)
- Java Native Access Platform (from https://github.com/java-native-access/jna)
- Javassist (from http://www.javassist.org/)
- SnakeYAML (from http://www.snakeyaml.org)
......@@ -842,7 +851,6 @@ The following software have components provided under the terms of this license:
- Azure Java Client Runtime for AutoRest (from https://github.com/Azure/autorest-clientruntime-for-java)
- Azure Metrics Spring Boot Starter (from https://github.com/Microsoft/azure-spring-boot)
- Azure Spring Boot AutoConfigure (from https://github.com/Microsoft/azure-spring-boot)
- Azure Spring Boot Starter (from https://github.com/Microsoft/azure-spring-boot)
- Bouncy Castle Provider (from http://www.bouncycastle.org/java.html)
- Bouncy Castle Provider (from http://www.bouncycastle.org/java.html)
- Checker Qual (from https://checkerframework.org)
......@@ -870,9 +878,10 @@ The following software have components provided under the terms of this license:
- Microsoft Azure client library for Identity (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure client library for KeyVault Secrets (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure common module for Storage (from https://github.com/Azure/azure-sdk-for-java)
- Mockito (from http://www.mockito.org)
- Microsoft Azure internal Avro module for Storage (from https://github.com/Azure/azure-sdk-for-java)
- Mockito (from http://mockito.org)
- Mockito (from http://mockito.org)
- Mockito (from http://www.mockito.org)
- Mockito (from http://mockito.org)
- Netty/Codec/HTTP (from )
- Netty/Common (from )
......@@ -881,9 +890,12 @@ The following software have components provided under the terms of this license:
- Project Lombok (from https://projectlombok.org)
- Project Lombok (from https://projectlombok.org)
- SLF4J API Module (from http://www.slf4j.org)
- Spongy Castle (from http://rtyley.github.io/spongycastle/)
- Spring Data for Azure Cosmos DB SQL API (from https://github.com/Azure/azure-sdk-for-java/tree/master/sdk/cosmos/azure-spring-data-cosmos)
- adal4j (from https://github.com/AzureAD/azure-activedirectory-library-for-java)
- micrometer-core (from https://github.com/micrometer-metrics/micrometer)
- msal4j (from https://github.com/AzureAD/microsoft-authentication-library-for-java)
- msal4j-persistence-extension (from https://github.com/AzureAD/microsoft-authentication-extensions-for-java)
- spring-security-core (from http://spring.io/spring-security)
========================================================================
......@@ -920,10 +932,10 @@ The following software have components provided under the terms of this license:
- JTidy (from http://jtidy.sourceforge.net)
- LatencyUtils (from http://latencyutils.github.io/LatencyUtils/)
- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils)
- Spongy Castle (from http://rtyley.github.io/spongycastle/)
- jersey-core-common (from )
- jersey-core-server (from git://java.net/jersey~code/jersey-server)
- jts-core (from )
- xml-apis (from )
========================================================================
SISSL-1.2
......@@ -981,7 +993,10 @@ The following software have components provided under the terms of this license:
- Spring Security JWT Library (from http://github.com/spring-projects/spring-security-oauth)
- Spring Security JWT Library (from http://github.com/spring-projects/spring-security-oauth)
- Spring Web (from https://github.com/spring-projects/spring-framework)
- StAX API (from http://stax.codehaus.org/)
- msal4j (from https://github.com/AzureAD/microsoft-authentication-library-for-java)
- reactive-streams (from http://www.reactive-streams.org/)
- xml-apis (from )
========================================================================
unknown
......@@ -995,6 +1010,7 @@ The following software have components provided under the terms of this license:
- Byte Buddy (without dependencies) (from )
- JavaBeans Activation Framework API jar (from )
- Servlet Specification 2.5 API (from )
- Spongy Castle (from http://rtyley.github.io/spongycastle/)
- System Rules (from http://stefanbirkner.github.io/system-rules/)
- jakarta.xml.bind-api (from )
- jaxen (from http://jaxen.codehaus.org/)
......
......@@ -68,21 +68,6 @@ spec:
configMapKeyRef:
name: osdu-svc-properties
key: ENV_KEYVAULT
- name: AZURE_CLIENT_ID
valueFrom:
secretKeyRef:
name: active-directory
key: principal-clientid
- name: AZURE_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: active-directory
key: principal-clientpassword
- name: AZURE_TENANT_ID
valueFrom:
secretKeyRef:
name: active-directory
key: tenantid
- name: aad_client_id
valueFrom:
secretKeyRef:
......
......@@ -6,6 +6,7 @@
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
<version>1.0.4-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>indexer-core</artifactId>
......@@ -197,7 +198,6 @@
<scope>test</scope>
</dependency>
<!-- swagger dependencies -->
<dependency>
<groupId>io.springfox</groupId>
......@@ -212,23 +212,6 @@
</dependencies>
<repositories>
<repository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</url>
</repository>
</repositories>
<distributionManagement>
<repository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/projects/25/packages/maven</url>
</repository>
<snapshotRepository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/projects/25/packages/maven</url>
</snapshotRepository>
</distributionManagement>
<build>
<defaultGoal>test</defaultGoal>
<plugins>
......
package org.opengroup.osdu.indexer.config;
import com.google.common.base.Strings;
import java.util.regex.PatternSyntaxException;
import lombok.Getter;
import lombok.Setter;
import org.opengroup.osdu.core.common.model.search.DeploymentEnvironment;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
@Configuration
@ConfigurationProperties
@Getter
@Setter
public class IndexerConfigurationProperties {
//Search query properties
private Integer queryDefaultLimit = 10;
private Integer queryLimitMaximum = 1000;
private Integer aggregationSize = 1000;
private String elasticDatastoreKind;
private String elasticDatastoreId;
//Default Cache Settings
private Integer schemaCacheExpiration = 60;
private Integer indexCacheExpiration = 60;
private Integer elasticCacheExpiration = 1440;
private Integer cursorCacheExpiration = 60;
//Kinds Cache expiration 2*24*60
private Integer kindsCacheExpiration = 2880;
//Attributes Cache expiration 2*24*60
private Integer attributesCacheExpiration = 2880;
private Integer kindsRedisDatabase = 1;
private Integer cronIndexCleanupThresholdDays = 3;
private Integer cronEmptyIndexCleanupThresholdDays = 7;
private String deploymentEnvironment = DeploymentEnvironment.CLOUD.name();
private String environment;
private String indexerHost;
private String searchHost;
private String storageQueryRecordForConversionHost;
private String storageQueryRecordHost;
private Integer storageRecordsBatchSize;
private String storageSchemaHost;
private String entitlementsHost;
private String entitlementTargetAudience;
private String indexerQueueHost;
private String redisSearchHost;
private String redisSearchPort;
private String elasticHost;
private String elasticClusterName;
private String keyRing;
private String kmsKey;
private String cronIndexCleanupPattern;
private String cronIndexCleanupTenants;
private String smartSearchCcsDisabled;
private String gaeService;
private String gaeVersion;
private String googleCloudProject;
private String googleCloudProjectRegion;
private String googleAudiences;
public DeploymentEnvironment getDeploymentEnvironment(){
return DeploymentEnvironment.valueOf(deploymentEnvironment);
}
public String getDeploymentLocation() {
return googleCloudProjectRegion;
}
public String getDeployedServiceId() {
return gaeService;
}
public String getDeployedVersionId() {
return gaeVersion;
}
public boolean isLocalEnvironment() {
return "local".equalsIgnoreCase(environment);
}
public boolean isPreP4d() {
return isLocalEnvironment() || "evd".equalsIgnoreCase(environment) || "evt".equalsIgnoreCase(environment);
}
public boolean isPreDemo() {
return isPreP4d() || "p4d".equalsIgnoreCase(environment);
}
public String[] getIndexCleanupPattern() {
if (!Strings.isNullOrEmpty(cronIndexCleanupPattern)) {
try {
return cronIndexCleanupPattern.split(",");
} catch (PatternSyntaxException var2) {
}
}
return new String[0];
}
public String[] getIndexCleanupTenants() {
if (!Strings.isNullOrEmpty(cronIndexCleanupTenants)) {
try {
return cronIndexCleanupTenants.split(",");
} catch (PatternSyntaxException var2) {
}
}
return new String[0];
}
public final Boolean isSmartSearchCcsDisabled() {
return Boolean.TRUE.toString().equalsIgnoreCase(smartSearchCcsDisabled);
}
}
......@@ -23,6 +23,7 @@ import org.opengroup.osdu.core.common.model.search.IndexInfo;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
import org.opengroup.osdu.core.common.search.IndicesService;
import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
......@@ -44,16 +45,12 @@ public class CronServiceImpl implements CronService{
private IndicesService indicesService;
@Inject
private JaxRsDpsLog log;
@Value("${CRON_INDEX_CLEANUP_THRESHOLD_DAYS}")
private String CRON_INDEX_CLEANUP_THRESHOLD_DAYS;
@Value("${CRON_EMPTY_INDEX_CLEANUP_THRESHOLD_DAYS}")
private String CRON_EMPTY_INDEX_CLEANUP_THRESHOLD_DAYS;
@Inject
private IndexerConfigurationProperties configurationProperties;
@Override
public boolean cleanupIndices(String indexPattern) throws IOException {
long threshHoldTime = Instant.now().minus(Integer.parseInt(CRON_INDEX_CLEANUP_THRESHOLD_DAYS), ChronoUnit.DAYS).toEpochMilli();
long threshHoldTime = Instant.now().minus(configurationProperties.getCronIndexCleanupThresholdDays(), ChronoUnit.DAYS).toEpochMilli();
try {
try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
......@@ -73,7 +70,7 @@ public class CronServiceImpl implements CronService{
@Override
public boolean cleanupEmptyStaleIndices() throws IOException {
long threshHoldTime = Instant.now().minus(Integer.parseInt(CRON_EMPTY_INDEX_CLEANUP_THRESHOLD_DAYS), ChronoUnit.DAYS).toEpochMilli();
long threshHoldTime = Instant.now().minus(configurationProperties.getCronEmptyIndexCleanupThresholdDays(), ChronoUnit.DAYS).toEpochMilli();
try {
try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
......
......@@ -22,7 +22,7 @@ import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository;
import org.opengroup.osdu.core.common.provider.interfaces.IElasticCredentialsCache;
import org.opengroup.osdu.core.common.model.indexer.IElasticSettingService;
import org.springframework.beans.factory.annotation.Value;
import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
import org.springframework.stereotype.Service;
import javax.inject.Inject;
......@@ -38,14 +38,13 @@ public class ElasticSettingServiceImpl implements IElasticSettingService {
private IElasticCredentialsCache elasticCredentialCache;
@Inject
private JaxRsDpsLog log;
@Value("${GAE_SERVICE}")
public String GAE_SERVICE;
@Inject
private IndexerConfigurationProperties configurationProperties;
@Override
public ClusterSettings getElasticClusterInformation() {
String cacheKey = String.format("%s-%s", GAE_SERVICE, tenantInfo.getName());
String cacheKey = String.format("%s-%s", configurationProperties.getGaeService(), tenantInfo.getName());
ClusterSettings clusterInfo = (ClusterSettings) this.elasticCredentialCache.get(cacheKey);
if (clusterInfo != null) {
return clusterInfo;
......
......@@ -46,15 +46,16 @@ import org.opengroup.osdu.core.common.search.Preconditions;
import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
import org.opengroup.osdu.core.common.model.indexer.Records;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
import org.springframework.stereotype.Service;
import javax.inject.Inject;
import static org.opengroup.osdu.core.common.search.Config.isPreDemo;
@Service
public class IndexerMappingServiceImpl extends MappingServiceImpl implements IndexerMappingService {
@Inject
private IndexerConfigurationProperties configurationProperties;
@Inject
private JaxRsDpsLog log;