diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b95ee08e180d940ef98d9a0f144340684005faa3..d54920bd08a34894ec6a7c775e117167ca4f8175 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,7 +1,9 @@
variables:
- AWS_BUILD_SUBDIR: provider/indexer-aws
- AWS_APPLICATION_NAME: os-indexer
+ AWS_BUILD_SUBDIR: provider/indexer-aws/build-aws
+ AWS_TEST_SUBDIR: testing/indexer-test-aws
+ AWS_SERVICE: indexer
AWS_ENVIRONMENT: dev
+
GCP_BUILD_SUBDIR: provider/indexer-gcp
GCP_INT_TEST_SUBDIR: testing/indexer-test-gcp
GCP_APPLICATION_NAME: osdu-indexer
@@ -11,25 +13,35 @@ variables:
GCP_DEPLOY_ENV: p4d
GCP_DOMAIN: cloud.slb-ds.com
GCP_STORAGE_URL: https://osdu-indexer-dot-opendes.appspot.com/api/storage/v2/
+
IBM_BUILD_SUBDIR: provider/indexer-ibm
IBM_INT_TEST_SUBDIR: testing/indexer-test-ibm
- IBM_SKIP_BUILD: 'false'
- IBM_SKIP_DEPLOY: 'false'
- IBM_SKIP_TEST: 'false'
+
+ AZURE_SERVICE: indexer
+ AZURE_BUILD_SUBDIR: provider/indexer-azure
+ AZURE_TEST_SUBDIR: testing/indexer-test-azure
include:
- project: "osdu/platform/ci-cd-pipelines"
- ref: "ibm"
file: "standard-setup.yml"
+
- project: "osdu/platform/ci-cd-pipelines"
- ref: "ibm"
file: "build/maven.yml"
+
+ - project: "osdu/platform/ci-cd-pipelines"
+ file: "scanners/fossa.yml"
+
+ - project: "osdu/platform/ci-cd-pipelines"
+ file: "scanners/gitlab-ultimate.yml"
+
+ - project: "osdu/platform/ci-cd-pipelines"
+ file: "cloud-providers/aws.yml"
+
- project: "osdu/platform/ci-cd-pipelines"
- ref: "ibm"
file: "cloud-providers/ibm.yml"
-# - project: "osdu/platform/ci-cd-pipelines"
-# ref: "ibm"
-# file: "scanners/fossa.yml"
-# - project: "osdu/platform/ci-cd-pipelines"
-# ref: "ibm"
-# file: "scanners/gitlab-ultimate.yml"
+
+ - project: "osdu/platform/ci-cd-pipelines"
+ file: "publishing/pages.yml"
+
+aws-test:
+ tags: ['aws-internal-test']
diff --git a/.mvn/community-maven.settings.xml b/.mvn/community-maven.settings.xml
index 75718a6d51b789d04f859ab8906b094bfb76b825..3af866cf08df445892d0239db6d6d35235a153df 100644
--- a/.mvn/community-maven.settings.xml
+++ b/.mvn/community-maven.settings.xml
@@ -1,55 +1,65 @@
<settings>
- <profiles>
- <profile>
- <!-- This profile uses the CI-Token to authenticate with the server, and is the default case -->
- <id>GitLab-Authenticate-With-Job-Token</id>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
- <properties>
- <gitlab-server>community-maven-via-job-token</gitlab-server>
- </properties>
- </profile>
+ <profiles>
+ <profile>
+ <!-- This profile uses the CI-Token to authenticate with the server, and is the default case -->
+ <id>GitLab-Authenticate-With-Job-Token</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
+ <properties>
+ <gitlab-server>community-maven-via-job-token</gitlab-server>
+ </properties>
+ </profile>
- <profile>
- <!-- This profile uses a personal token to authenticate, which is useful for local or manual runs. The
+ <profile>
+ <!-- This profile uses a personal token to authenticate, which is useful for local or manual runs. The
presence of the COMMUNITY_MAVEN_TOKEN variable triggers this and overrides the CI Token
based authentication -->
- <id>GitLab-Authenticate-With-Private-Token</id>
- <activation>
- <property>
- <name>env.COMMUNITY_MAVEN_TOKEN</name>
- </property>
- </activation>
- <properties>
- <gitlab-server>community-maven-via-private-token</gitlab-server>
- </properties>
- </profile>
- </profiles>
+ <id>GitLab-Authenticate-With-Private-Token</id>
+ <activation>
+ <property>
+ <name>env.COMMUNITY_MAVEN_TOKEN</name>
+ </property>
+ </activation>
+ <properties>
+ <gitlab-server>community-maven-via-private-token</gitlab-server>
+ </properties>
+ </profile>
+ </profiles>
- <servers>
- <server>
- <id>community-maven-via-job-token</id>
- <configuration>
- <httpHeaders>
- <property>
- <name>Job-Token</name>
- <value>${env.CI_JOB_TOKEN}</value>
- </property>
- </httpHeaders>
- </configuration>
- </server>
+ <servers>
+ <server>
+ <id>community-maven-via-job-token</id>
+ <configuration>
+ <httpHeaders>
+ <property>
+ <name>Job-Token</name>
+ <value>${env.CI_JOB_TOKEN}</value>
+ </property>
+ </httpHeaders>
+ </configuration>
+ </server>
- <server>
- <id>community-maven-via-private-token</id>
- <configuration>
- <httpHeaders>
- <property>
- <name>Private-Token</name>
- <value>${env.COMMUNITY_MAVEN_TOKEN}</value>
- </property>
- </httpHeaders>
- </configuration>
- </server>
- </servers>
+ <server>
+ <id>community-maven-via-private-token</id>
+ <configuration>
+ <httpHeaders>
+ <property>
+ <name>Private-Token</name>
+ <value>${env.COMMUNITY_MAVEN_TOKEN}</value>
+ </property>
+ </httpHeaders>
+ </configuration>
+ </server>
+
+ <server>
+ <id>azure-auth</id>
+ <configuration>
+ <tenant>${env.AZURE_TENANT_ID}</tenant>
+ <client>${env.AZURE_CLIENT_ID}</client>
+ <key>${env.AZURE_CLIENT_SECRET}</key>
+ <environment>AZURE</environment>
+ </configuration>
+ </server>
+ </servers>
</settings>
diff --git a/NOTICE b/NOTICE
index c2b2df630a852d6da44e3325304ee0e356bf8c2d..78ffe18c36f108b999403a21aca5561da549c439 100644
--- a/NOTICE
+++ b/NOTICE
@@ -2,12 +2,28 @@
Generated by fossa-cli (https://github.com/fossas/fossa-cli).
This software includes the following software and licenses:
+========================================================================
+Apache-1.1
+========================================================================
+The following software have components provided under the terms of this license:
+
+- Apache Commons CLI (from http://commons.apache.org/proper/commons-cli/)
+- Cobertura code coverage (from http://cobertura.sourceforge.net)
+- Plexus :: Default Container (from )
+- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils)
+- oro (from )
+
========================================================================
Apache-2.0
========================================================================
The following software have components provided under the terms of this license:
+- ASM Analysis (from )
+- ASM Commons (from )
+- ASM Core (from )
- ASM Core (from )
+- ASM Tree (from )
+- ASM Util (from )
- ASM based accessors helper used by json-smart (from )
- AWS Java SDK for AWS AppSync (from https://aws.amazon.com/sdkforjava)
- AWS Java SDK for AWS Application Auto Scaling (from https://aws.amazon.com/sdkforjava)
@@ -140,35 +156,80 @@ The following software have components provided under the terms of this license:
- AWS SDK for Java - BOM (from https://aws.amazon.com/sdkforjava)
- AWS SDK for Java - Core (from https://aws.amazon.com/sdkforjava)
- AWS SDK for Java - Models (from https://aws.amazon.com/sdkforjava)
+- Adapter: RxJava (from )
+- Apache Ant + JUnit (from http://ant.apache.org/)
+- Apache Ant Core (from http://ant.apache.org/)
+- Apache Ant Launcher (from http://ant.apache.org/)
- Apache Commons BeanUtils (from http://commons.apache.org/proper/commons-beanutils/)
+- Apache Commons CLI (from http://commons.apache.org/proper/commons-cli/)
- Apache Commons Codec (from http://commons.apache.org/proper/commons-codec/)
- Apache Commons Codec (from http://commons.apache.org/proper/commons-codec/)
+- Apache Commons Collections (from http://commons.apache.org/proper/commons-collections/)
- Apache Commons Lang (from http://commons.apache.org/proper/commons-lang/)
- Apache Commons Logging (from http://commons.apache.org/proper/commons-logging/)
+- Apache Commons Logging (from http://commons.apache.org/proper/commons-logging/)
+- Apache Commons Text (from http://commons.apache.org/proper/commons-text/)
+- Apache Commons Validator (from http://commons.apache.org/proper/commons-validator/)
+- Apache Commons Validator (from http://commons.apache.org/proper/commons-validator/)
- Apache HttpAsyncClient (from http://hc.apache.org/httpcomponents-asyncclient)
- Apache HttpClient (from http://hc.apache.org/httpcomponents-client)
- Apache HttpCore (from http://hc.apache.org/httpcomponents-core-ga)
- Apache HttpCore NIO (from http://hc.apache.org/httpcomponents-core-ga)
- Apache Log4j API (from )
+- Apache Log4j Core (from )
+- Apache Log4j JUL Adapter (from )
- Apache Log4j to SLF4J Adapter (from )
+- Apache Maven Invoker (from )
+- Apache Maven Reporting Implementation (from )
+- Apache Maven Wagon :: API (from )
+- Apache Maven Wagon :: Providers :: File Provider (from )
+- Apache Maven Wagon :: Providers :: HTTP Shared Library (from )
+- Apache Maven Wagon :: Providers :: Lightweight HTTP Provider (from )
+- Apache Maven Wagon :: Providers :: SSH Common Library (from )
+- Apache Maven Wagon :: Providers :: SSH External Provider (from )
+- Apache Maven Wagon :: Providers :: SSH Provider (from )
+- Apache Velocity (from http://velocity.apache.org/engine/devel/)
- AssertJ fluent assertions (from )
+- Asynchronous Http Client (from )
+- Asynchronous Http Client Netty Utils (from )
+- AutoValue (from )
- Bean Validation API (from http://beanvalidation.org)
- Byte Buddy (without dependencies) (from )
- Byte Buddy Java agent (from )
- ClassMate (from http://github.com/cowtowncoder/java-classmate)
- Cloud Storage JSON API v1-rev58-1.21.0 (from )
- CloudWatch Metrics for AWS Java SDK (from https://aws.amazon.com/sdkforjava)
+- Cobertura code coverage (from http://cobertura.sourceforge.net)
+- Commons Digester (from http://commons.apache.org/digester/)
+- Commons Digester (from http://commons.apache.org/digester/)
+- Commons IO (from http://commons.apache.org/io/)
- Commons Lang (from http://commons.apache.org/lang/)
+- Converter: Jackson (from )
- Data Mapper for Jackson (from http://jackson.codehaus.org)
+- Doxia :: APT Module (from )
+- Doxia :: Core (from )
+- Doxia :: Decoration Model (from http://maven.apache.org/doxia/doxia-sitetools/doxia-decoration-model/)
+- Doxia :: FML Module (from )
+- Doxia :: Sink API (from )
+- Doxia :: Site Renderer (from http://maven.apache.org/doxia/doxia-sitetools/doxia-site-renderer/)
+- Doxia :: XDoc Module (from )
+- Doxia :: XHTML Module (from )
- Elastic JNA Distribution (from https://github.com/java-native-access/jna)
- Elasticsearch: 5.0.0-alpha5 (from https://github.com/elastic/elasticsearch)
- FindBugs-jsr305 (from http://findbugs.sourceforge.net/)
+- Google APIs Client Library for Java (from )
+- Google App Engine extensions to the Google HTTP Client Library for Java. (from )
+- Google Cloud Core (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-core)
+- Google Cloud Core HTTP (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-core-http)
+- Google Cloud Core gRPC (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-core-grpc)
- Google Cloud Datastore (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-datastore)
- Google Cloud Key Management Service (KMS) API v1-rev22-1.23.0 (from )
- Google Cloud Logging (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-logging)
- Google Cloud Pub/Sub (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-pubsub)
+- Google Cloud Storage (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-storage)
- Google HTTP Client Library for Java (from https://github.com/google/google-http-java-client.git)
- Google HTTP Client Library for Java (from https://github.com/google/google-http-java-client.git)
+- Google OAuth Client Library for Java (from )
- Gson (from https://github.com/google/gson)
- Guava InternalFutureFailureAccess and InternalFutures (from )
- Guava ListenableFuture only (from )
@@ -176,9 +237,12 @@ The following software have components provided under the terms of this license:
- Guava: Google Core Libraries for Java (from https://github.com/google/guava.git)
- HPPC Collections (from http://labs.carrotsearch.com)
- Hibernate Validator Engine (from )
+- Identity and Access Management (IAM) API v1-rev247-1.23.0 (from )
+- J2ObjC Annotations (from https://github.com/google/j2objc/)
- J2ObjC Annotations (from https://github.com/google/j2objc/)
- JBoss Logging 3 (from http://www.jboss.org)
- JCIP Annotations under Apache License (from http://stephenc.github.com/jcip-annotations)
+- JDOM (from http://www.jdom.org)
- JMES Path Query library (from https://aws.amazon.com/sdkforjava)
- JSON Small and Fast Parser (from http://www.minidev.net/)
- JSON Web Token support for the JVM (from https://github.com/jwtk/jjwt.git)
@@ -186,18 +250,28 @@ The following software have components provided under the terms of this license:
- JSON.simple (from http://code.google.com/p/json-simple/)
- JSONassert (from https://github.com/skyscreamer/JSONassert)
- Jackson (from http://jackson.codehaus.org)
+- Jackson (from http://jackson.codehaus.org)
+- Jackson 2 extensions to the Google HTTP Client Library for Java. (from https://github.com/google/google-http-java-client.git/google-http-client-jackson2)
- Jackson dataformat: CBOR (from http://github.com/FasterXML/jackson-dataformats-binary)
- Jackson datatype: JSR310 (from http://wiki.fasterxml.com/JacksonModuleJSR310)
+- Jackson extensions to the Google HTTP Client Library for Java. (from )
- Jackson-annotations (from http://github.com/FasterXML/jackson)
- Jackson-core (from https://github.com/FasterXML/jackson-core)
- Jackson-dataformat-Smile (from http://github.com/FasterXML/jackson-dataformat-smile)
+- Jackson-dataformat-XML (from http://wiki.fasterxml.com/JacksonExtensionXmlDataBinding)
- Jackson-dataformat-YAML (from https://github.com/FasterXML/jackson)
+- Jackson-datatype-Joda (from http://wiki.fasterxml.com/JacksonModuleJoda)
- Jackson-datatype-jdk8 (from )
- Jackson-module-JAXB-annotations (from http://wiki.fasterxml.com/JacksonJAXBAnnotations)
- Jackson-module-parameter-names (from )
- Java Libraries for Amazon Simple WorkFlow (from https://aws.amazon.com/sdkforjava)
+- Java Native Access (from https://github.com/java-native-access/jna)
+- Java Native Access Platform (from https://github.com/java-native-access/jna)
- Java Servlet API (from http://servlet-spec.java.net)
+- Java UUID Generator (from http://wiki.fasterxml.com/JugHome)
- Javassist (from http://www.javassist.org/)
+- Jetty Server (from )
+- Jetty Utilities (from )
- Joda-Time (from http://www.joda.org/joda-time/)
- Json Path (from https://github.com/jayway/JsonPath)
- Lucene Common Analyzers (from )
@@ -216,26 +290,71 @@ The following software have components provided under the terms of this license:
- Lucene Spatial Extras (from )
- Lucene Suggest (from )
- MapStruct Core (from )
+- Maven Artifact (from )
+- Maven Artifact Manager (from )
+- Maven Core (from https://git-wip-us.apache.org/repos/asf/maven.git/maven-core)
+- Maven Error Diagnostics (from )
+- Maven Model (from https://git-wip-us.apache.org/repos/asf/maven.git/maven-model)
+- Maven Monitor (from )
+- Maven Plugin API (from https://git-wip-us.apache.org/repos/asf/maven.git/maven-plugin-api)
+- Maven Plugin Descriptor Model (from )
+- Maven Plugin Parameter Documenter API (from )
+- Maven Plugin Registry Model (from )
+- Maven Profile Model (from )
+- Maven Project (from )
+- Maven Reporting API (from )
+- Maven Repository Metadata Model (from )
+- Maven Settings (from )
+- Metrics Core (from https://github.com/dropwizard/metrics)
+- Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java)
+- Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java)
+- Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java)
+- Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Logback Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
+- Mockito (from http://www.mockito.org)
- Mockito (from http://mockito.org)
- Mockito (from http://mockito.org)
-- Mockito (from http://www.mockito.org)
- Mojo's Maven plugin for Cobertura (from http://mojo.codehaus.org/cobertura-maven-plugin/)
+- Netty Reactive Streams Implementation (from )
- Netty/Buffer (from http://netty.io/)
- Netty/Codec (from )
+- Netty/Codec/DNS (from )
- Netty/Codec/HTTP (from )
+- Netty/Codec/HTTP2 (from )
+- Netty/Codec/Socks (from )
- Netty/Common (from )
- Netty/Handler (from )
+- Netty/Handler/Proxy (from )
- Netty/Resolver (from )
+- Netty/Resolver/DNS (from )
+- Netty/TomcatNative [BoringSSL - Static] (from )
- Netty/Transport (from http://netty.io/)
+- Netty/Transport/Native/Unix/Common (from )
- Nimbus JOSE+JWT (from https://bitbucket.org/connect2id/nimbus-jose-jwt)
- Nimbus LangTag (from https://bitbucket.org/connect2id/nimbus-language-tags)
+- Non-Blocking Reactive Foundation for the JVM (from https://github.com/reactor/reactor)
- OAuth 2.0 SDK with OpenID Connect extensions (from https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions)
- OAuth2 for Spring Security (from )
- Objenesis (from http://objenesis.org)
+- OkHttp (from )
+- OkHttp Logging Interceptor (from )
+- OkHttp URLConnection (from )
+- Okio (from )
+- OpenCensus (from https://github.com/census-instrumentation/opencensus-java)
- OpenCensus (from https://github.com/census-instrumentation/opencensus-java)
- OpenCensus (from https://github.com/census-instrumentation/opencensus-java)
+- OpenCensus (from https://github.com/census-instrumentation/opencensus-java)
+- OpenCensus (from https://github.com/census-instrumentation/opencensus-java)
+- Plexus :: Default Container (from )
+- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils)
+- Plexus Default Interactivity Handler (from )
+- Plexus I18N Component (from )
+- Plexus Velocity Component (from )
- PowerMock (from http://www.powermock.org)
+- Protocol Buffer extensions to the Google HTTP Client Library for Java. (from )
+- Reactive Streams Netty driver (from https://github.com/reactor/reactor-netty)
+- Retrofit (from )
+- Servlet Specification 2.5 API (from )
- SnakeYAML (from http://www.snakeyaml.org)
- Spatial4J (from http://www.locationtech.org/projects/locationtech.spatial4j)
- Spring AOP (from https://github.com/spring-projects/spring-framework)
@@ -244,25 +363,20 @@ The following software have components provided under the terms of this license:
- Spring Boot Actuator (from http://projects.spring.io/spring-boot/)
- Spring Boot Actuator AutoConfigure (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-actuator-autoconfigure)
- Spring Boot Actuator Starter (from http://projects.spring.io/spring-boot/)
-- Spring Boot Actuator Starter (from http://projects.spring.io/spring-boot/)
- Spring Boot AutoConfigure (from http://projects.spring.io/spring-boot/)
- Spring Boot Configuration Processor (from http://projects.spring.io/spring-boot/)
- Spring Boot Dependencies (from http://projects.spring.io/spring-boot/)
- Spring Boot Jersey Starter (from http://projects.spring.io/spring-boot/)
-- Spring Boot Jersey Starter (from http://projects.spring.io/spring-boot/)
- Spring Boot Json Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-json)
+- Spring Boot Log4J2 Starter (from http://projects.spring.io/spring-boot/)
- Spring Boot Logging Starter (from http://projects.spring.io/spring-boot/)
- Spring Boot Security Starter (from http://projects.spring.io/spring-boot/)
-- Spring Boot Security Starter (from http://projects.spring.io/spring-boot/)
- Spring Boot Starter (from http://projects.spring.io/spring-boot/)
- Spring Boot Test (from http://projects.spring.io/spring-boot/)
- Spring Boot Test Auto-Configure (from http://projects.spring.io/spring-boot/)
- Spring Boot Test Starter (from http://projects.spring.io/spring-boot/)
-- Spring Boot Test Starter (from http://projects.spring.io/spring-boot/)
- Spring Boot Tomcat Starter (from http://projects.spring.io/spring-boot/)
- Spring Boot Validation Starter (from http://projects.spring.io/spring-boot/)
-- Spring Boot Validation Starter (from http://projects.spring.io/spring-boot/)
-- Spring Boot Web Starter (from http://projects.spring.io/spring-boot/)
- Spring Boot Web Starter (from http://projects.spring.io/spring-boot/)
- Spring Commons Logging Bridge (from https://github.com/spring-projects/spring-framework)
- Spring Context (from https://github.com/spring-projects/spring-framework)
@@ -273,20 +387,39 @@ The following software have components provided under the terms of this license:
- Spring Plugin - Metadata Extension (from )
- Spring Security JWT Library (from http://github.com/spring-projects/spring-security-oauth)
- Spring TestContext Framework (from https://github.com/spring-projects/spring-framework)
+- Spring Transaction (from https://github.com/spring-projects/spring-framework)
- Spring Web (from https://github.com/spring-projects/spring-framework)
- Spring Web MVC (from https://github.com/spring-projects/spring-framework)
- T-Digest (from https://github.com/tdunning/t-digest)
+- Woodstox (from https://github.com/FasterXML/woodstox)
+- Xerces2-j (from https://xerces.apache.org/xerces2-j/)
- aggs-matrix-stats (from https://github.com/elastic/elasticsearch)
- cli (from https://github.com/elastic/elasticsearch)
+- com.google.api.grpc:grpc-google-cloud-pubsub-v1 (from https://github.com/googleapis/googleapis)
+- com.google.api.grpc:proto-google-cloud-logging-v2 (from https://github.com/googleapis/googleapis)
+- com.google.api.grpc:proto-google-cloud-pubsub-v1 (from https://github.com/googleapis/googleapis)
+- com.google.api.grpc:proto-google-common-protos (from https://github.com/googleapis/googleapis)
- com.google.api.grpc:proto-google-iam-v1 (from https://github.com/googleapis/googleapis)
- commons-collections (from )
- compiler (from http://github.com/spullara/mustache.java)
+- datastore-v1-proto-client (from )
- elasticsearch-core (from https://github.com/elastic/elasticsearch)
- error-prone annotations (from )
+- error-prone annotations (from )
+- io.grpc:grpc-alts (from https://github.com/grpc/grpc-java)
+- io.grpc:grpc-auth (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-context (from https://github.com/grpc/grpc-java)
+- io.grpc:grpc-context (from https://github.com/grpc/grpc-java)
+- io.grpc:grpc-core (from https://github.com/grpc/grpc-java)
+- io.grpc:grpc-grpclb (from https://github.com/grpc/grpc-java)
+- io.grpc:grpc-netty-shaded (from https://github.com/grpc/grpc-java)
+- io.grpc:grpc-protobuf (from https://github.com/grpc/grpc-java)
+- io.grpc:grpc-protobuf-lite (from https://github.com/grpc/grpc-java)
+- io.grpc:grpc-stub (from https://github.com/grpc/grpc-java)
- ion-java (from https://github.com/amznlabs/ion-java/)
- jackson-databind (from http://github.com/FasterXML/jackson)
- jackson-databind (from http://github.com/FasterXML/jackson)
+- javatuples (from http://www.javatuples.org)
- javax.inject (from http://code.google.com/p/atinject/)
- javax.ws.rs-api (from http://jax-rs-spec.java.net)
- jersey-container-servlet (from git://java.net/jersey~code/project/jersey-container-servlet)
@@ -304,28 +437,31 @@ The following software have components provided under the terms of this license:
- lang-mustache (from https://github.com/elastic/elasticsearch)
- lettuce (from http://github.com/mp911de/lettuce/wiki)
- micrometer-core (from https://github.com/micrometer-metrics/micrometer)
+- micrometer-registry-azure-monitor (from https://github.com/micrometer-metrics/micrometer)
- org.xmlunit:xmlunit-core (from http://www.xmlunit.org/)
+- oro (from )
- parent-join (from https://github.com/elastic/elasticsearch)
- powermock-api-mockito (from )
- powermock-api-mockito-common (from )
- powermock-api-support (from )
+- powermock-api-support (from )
- powermock-core (from http://www.powermock.org)
- powermock-module-junit4 (from http://www.powermock.org)
- powermock-module-junit4-common (from )
- powermock-reflect (from )
+- proto-google-cloud-datastore-v1 (from https://github.com/googleapis/api-client-staging)
+- proton-j (from )
- rank-eval (from https://github.com/elastic/elasticsearch)
- rest (from https://github.com/elastic/elasticsearch)
- rest-high-level (from https://github.com/elastic/elasticsearch)
- rxjava (from https://github.com/ReactiveX/RxJava)
+- rxjava (from https://github.com/ReactiveX/RxJava)
- secure-sm (from https://github.com/elastic/elasticsearch)
- spring-security-config (from http://spring.io/spring-security)
- spring-security-core (from http://spring.io/spring-security)
- spring-security-oauth2-client (from http://spring.io/spring-security)
-- spring-security-oauth2-client (from http://spring.io/spring-security)
- spring-security-oauth2-core (from http://spring.io/spring-security)
- spring-security-oauth2-jose (from http://spring.io/spring-security)
-- spring-security-oauth2-jose (from http://spring.io/spring-security)
-- spring-security-test (from http://spring.io/spring-security)
- spring-security-test (from http://spring.io/spring-security)
- spring-security-web (from http://spring.io/spring-security)
- springfox-core (from https://github.com/springfox/springfox)
@@ -344,37 +480,70 @@ The following software have components provided under the terms of this license:
- tomcat-embed-el (from http://tomcat.apache.org/)
- tomcat-embed-websocket (from http://tomcat.apache.org/)
- x-content (from https://github.com/elastic/elasticsearch)
+- xml-apis (from )
========================================================================
BSD-2-Clause
========================================================================
The following software have components provided under the terms of this license:
+- Apache Commons CLI (from http://commons.apache.org/proper/commons-cli/)
+- GAX (Google Api eXtensions) (from https://github.com/googleapis)
+- GAX (Google Api eXtensions) (from https://github.com/googleapis)
+- GAX (Google Api eXtensions) (from https://github.com/googleapis)
- Hamcrest Core (from http://hamcrest.org/)
- Lucene Common Analyzers (from )
+- Plexus :: Default Container (from )
+- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils)
- jersey-ext-bean-validation (from )
- jersey-spring4 (from )
+- oro (from )
========================================================================
BSD-3-Clause
========================================================================
The following software have components provided under the terms of this license:
+- API Common (from https://github.com/googleapis)
+- ASM Analysis (from )
+- ASM Commons (from )
- ASM Core (from )
+- ASM Core (from )
+- ASM Tree (from )
+- ASM Util (from )
- ASM library repackaged as OSGi bundle (from )
- Apache Commons Codec (from http://commons.apache.org/proper/commons-codec/)
- Apache Commons Codec (from http://commons.apache.org/proper/commons-codec/)
-- Byte Buddy (without dependencies) (from )
+- GAX (Google Api eXtensions) (from https://github.com/googleapis)
+- GAX (Google Api eXtensions) (from https://github.com/googleapis)
+- GAX (Google Api eXtensions) (from https://github.com/googleapis)
+- Google APIs Client Library for Java (from )
+- Google Auth Library for Java - Credentials (from )
+- Google Auth Library for Java - OAuth2 HTTP (from )
- Hamcrest library (from )
+- JDOM (from http://www.jdom.org)
+- JSch (from http://www.jcraft.com/jsch/)
- Lucene Common Analyzers (from )
- Lucene Core (from )
- Lucene Suggest (from )
+- Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java)
+- Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java)
+- Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java)
+- Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Logback Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
- Mockito (from http://www.mockito.org)
+- NanoHttpd-Core (from )
- Netty/Codec/HTTP (from )
+- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils)
+- Protocol Buffer Java API (from https://developers.google.com/protocol-buffers/)
+- Protocol Buffers [Util] (from )
- Reflections (from http://github.com/ronmamo/reflections)
- SnakeYAML (from http://www.snakeyaml.org)
- Spring Core (from https://github.com/spring-projects/spring-framework)
+- Stax2 API (from http://github.com/FasterXML/stax2-api)
+- ThreeTen backport (from https://www.threeten.org/threetenbp)
+- classworlds (from http://classworlds.codehaus.org/)
+- jaxen (from http://jaxen.codehaus.org/)
- jersey-container-servlet (from git://java.net/jersey~code/project/jersey-container-servlet)
- jersey-container-servlet-core (from git://java.net/jersey~code/project/jersey-container-servlet-core)
- jersey-core-client (from git://java.net/jersey~code/jersey-client)
@@ -383,6 +552,13 @@ The following software have components provided under the terms of this license:
- jersey-inject-hk2 (from )
- jersey-media-jaxb (from )
+========================================================================
+CC-BY-2.5
+========================================================================
+The following software have components provided under the terms of this license:
+
+- Checker Qual (from https://checkerframework.org)
+
========================================================================
CC-BY-3.0
========================================================================
@@ -395,8 +571,19 @@ CC-BY-4.0
========================================================================
The following software have components provided under the terms of this license:
+- Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java)
+- Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java)
+- Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java)
+- Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Logback Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
+========================================================================
+CC0-1.0
+========================================================================
+The following software have components provided under the terms of this license:
+
+- reactive-streams (from http://www.reactive-streams.org/)
+
========================================================================
CDDL-1.0
========================================================================
@@ -415,6 +602,7 @@ The following software have components provided under the terms of this license:
- JavaBeans(TM) Activation Framework (from http://java.sun.com/javase/technologies/desktop/javabeans/jaf/index.jsp)
- Run Level Service (from )
- ServiceLocator Default Implementation (from git://java.net/hk2~git/hk2-locator)
+- Servlet Specification 2.5 API (from )
- aopalliance-repackaged (from )
- javax.annotation-api (from http://jcp.org/en/jsr/detail?id=250)
- javax.ws.rs-api (from http://jax-rs-spec.java.net)
@@ -435,6 +623,7 @@ The following software have components provided under the terms of this license:
- Java Architecture For XML Binding (from )
- Java Servlet API (from http://servlet-spec.java.net)
+- JavaBeans Activation Framework (from )
- JavaMail API (from )
- OSGi resource locator (from )
- jersey-ext-bean-validation (from )
@@ -457,6 +646,10 @@ The following software have components provided under the terms of this license:
- JUnit (from http://junit.org)
- Logback Classic Module (from )
- Logback Core Module (from )
+- Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java)
+- Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java)
+- Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java)
+- Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Logback Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
- SnakeYAML (from http://www.snakeyaml.org)
- jts-core (from )
@@ -469,6 +662,8 @@ The following software have components provided under the terms of this license:
- ASM library repackaged as OSGi bundle (from )
- Class Model for Hk2 (from )
+- Cobertura Limited Runtime (from http://cobertura.sourceforge.net)
+- Cobertura code coverage (from http://cobertura.sourceforge.net)
- Commons Lang (from http://commons.apache.org/lang/)
- HK2 API module (from git://java.net/hk2~git/hk2-api)
- HK2 Implementation Utilities (from )
@@ -479,6 +674,7 @@ The following software have components provided under the terms of this license:
- HK2 module of HK2 itself (from )
- Java Architecture For XML Binding (from )
- Java Servlet API (from http://servlet-spec.java.net)
+- JavaBeans Activation Framework (from )
- JavaBeans Activation Framework API jar (from )
- JavaMail API (from )
- Mojo's Maven plugin for Cobertura (from http://mojo.codehaus.org/cobertura-maven-plugin/)
@@ -514,9 +710,12 @@ GPL-2.0-with-classpath-exception
========================================================================
The following software have components provided under the terms of this license:
+- Checker Qual (from https://checkerframework.org)
+- Cobertura code coverage (from http://cobertura.sourceforge.net)
- HK2 Implementation Utilities (from )
- Java Architecture For XML Binding (from )
- Java Servlet API (from http://servlet-spec.java.net)
+- JavaBeans Activation Framework (from )
- JavaBeans Activation Framework API jar (from )
- JavaMail API (from )
- OSGi resource locator (from )
@@ -548,23 +747,40 @@ The following software have components provided under the terms of this license:
- SnakeYAML (from http://www.snakeyaml.org)
- javax.ws.rs-api (from http://jax-rs-spec.java.net)
+========================================================================
+JSON
+========================================================================
+The following software have components provided under the terms of this license:
+
+- JSON in Java (from https://github.com/douglascrockford/JSON-java)
+
========================================================================
LGPL-2.1-only
========================================================================
The following software have components provided under the terms of this license:
+- Cobertura code coverage (from http://cobertura.sourceforge.net)
- Commons Lang (from http://commons.apache.org/lang/)
- Elastic JNA Distribution (from https://github.com/java-native-access/jna)
+- Java Native Access (from https://github.com/java-native-access/jna)
+- Java Native Access Platform (from https://github.com/java-native-access/jna)
- Javassist (from http://www.javassist.org/)
- Logback Classic Module (from )
- Logback Core Module (from )
+- Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java)
+- Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java)
+- Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java)
+- Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Logback Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
+- java-getopt (from )
========================================================================
LGPL-2.1-or-later
========================================================================
The following software have components provided under the terms of this license:
+- Java Native Access (from https://github.com/java-native-access/jna)
+- Java Native Access Platform (from https://github.com/java-native-access/jna)
- SnakeYAML (from http://www.snakeyaml.org)
========================================================================
@@ -573,28 +789,59 @@ LGPL-3.0-only
The following software have components provided under the terms of this license:
- Apache Log4j API (from )
+- Apache Log4j Core (from )
========================================================================
MIT
========================================================================
The following software have components provided under the terms of this license:
+- Animal Sniffer Annotations (from )
+- Azure AD Spring Security Integration Spring Boot Starter (from https://github.com/Microsoft/azure-spring-boot)
+- Azure Cosmos DB Spring Boot Starter (from https://github.com/Microsoft/azure-spring-boot)
+- Azure Java Client Authentication Library for AutoRest (from https://github.com/Azure/autorest-clientruntime-for-java)
+- Azure Java Client Runtime for AutoRest (from https://github.com/Azure/autorest-clientruntime-for-java)
+- Azure Metrics Spring Boot Starter (from https://github.com/Microsoft/azure-spring-boot)
+- Azure Spring Boot AutoConfigure (from https://github.com/Microsoft/azure-spring-boot)
+- Azure Spring Boot Starter (from https://github.com/Microsoft/azure-spring-boot)
- Bouncy Castle Provider (from http://www.bouncycastle.org/java.html)
- Checker Qual (from https://checkerframework.org)
+- Checker Qual (from https://checkerframework.org)
- Elasticsearch: 5.0.0-alpha5 (from https://github.com/elastic/elasticsearch)
+- Extensions on Apache Proton-J library (from https://github.com/Azure/qpid-proton-j-extensions)
- JOpt Simple (from http://pholser.github.io/jopt-simple)
- JUL to SLF4J bridge (from http://www.slf4j.org)
+- Java Client Runtime for AutoRest (from https://github.com/Azure/autorest-clientruntime-for-java)
- Java JWT (from http://www.jwt.io)
- Lucene Core (from )
- Lucene Sandbox (from )
+- Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java)
+- Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java)
+- Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java)
+- Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Logback Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
+- Microsoft Azure Java Core Library (from https://github.com/Azure/azure-sdk-for-java)
+- Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java)
+- Microsoft Azure SDK annotations (from https://github.com/Microsoft/java-api-annotations)
+- Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java)
+- Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java)
+- Microsoft Azure SDK for Service Bus (from https://github.com/Azure/azure-sdk-for-java)
+- Microsoft Azure client library for Blob Storage (from https://github.com/Azure/azure-sdk-for-java)
+- Microsoft Azure client library for Identity (from https://github.com/Azure/azure-sdk-for-java)
+- Microsoft Azure client library for KeyVault Secrets (from https://github.com/Azure/azure-sdk-for-java)
+- Microsoft Azure common module for Storage (from https://github.com/Azure/azure-sdk-for-java)
- Mockito (from http://mockito.org)
- Mockito (from http://www.mockito.org)
- Mockito (from http://mockito.org)
- Netty/Codec/HTTP (from )
- Netty/Common (from )
+- Plexus :: Default Container (from )
+- Plexus Default Interactivity Handler (from )
- Project Lombok (from https://projectlombok.org)
- SLF4J API Module (from http://www.slf4j.org)
+- Spring Data for Azure Cosmos DB SQL API (from https://github.com/Microsoft/spring-data-cosmosdb)
+- adal4j (from https://github.com/AzureAD/azure-activedirectory-library-for-java)
+- azure-documentdb (from https://azure.microsoft.com/en-us/services/cosmos-db/)
- msal4j (from https://github.com/AzureAD/microsoft-authentication-library-for-java)
- powermock-api-mockito (from )
- spring-security-core (from http://spring.io/spring-security)
@@ -604,6 +851,7 @@ MPL-1.1
========================================================================
The following software have components provided under the terms of this license:
+- Cobertura code coverage (from http://cobertura.sourceforge.net)
- Javassist (from http://www.javassist.org/)
========================================================================
@@ -619,18 +867,40 @@ Public-Domain
The following software have components provided under the terms of this license:
- HdrHistogram (from http://hdrhistogram.github.io/HdrHistogram/)
+- JTidy (from http://jtidy.sourceforge.net)
- LatencyUtils (from http://latencyutils.github.io/LatencyUtils/)
+- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils)
- jersey-core-common (from )
- jersey-core-server (from git://java.net/jersey~code/jersey-server)
- jts-core (from )
+- reactive-streams (from http://www.reactive-streams.org/)
+- xml-apis (from )
+
+========================================================================
+SISSL-1.2
+========================================================================
+The following software have components provided under the terms of this license:
+
+- Servlet Specification 2.5 API (from )
========================================================================
SPL-1.0
========================================================================
The following software have components provided under the terms of this license:
+- Checker Qual (from https://checkerframework.org)
- Checker Qual (from https://checkerframework.org)
+========================================================================
+W3C
+========================================================================
+The following software have components provided under the terms of this license:
+
+- JTidy (from http://jtidy.sourceforge.net)
+- Xerces2-j (from https://xerces.apache.org/xerces2-j/)
+- jaxen (from http://jaxen.codehaus.org/)
+- xml-apis (from )
+
========================================================================
WTFPL
========================================================================
@@ -644,6 +914,7 @@ public-domain
The following software have components provided under the terms of this license:
- AWS SDK for Java - Models (from https://aws.amazon.com/sdkforjava)
+- Asynchronous Http Client (from )
- Bouncy Castle PKIX, CMS, EAC, TSP, PKCS, OCSP, CMP, and CRMF APIs (from http://www.bouncycastle.org/java.html)
- Bouncy Castle Provider (from http://www.bouncycastle.org/java.html)
- Guava: Google Core Libraries for Java (from https://github.com/google/guava.git)
@@ -651,6 +922,10 @@ The following software have components provided under the terms of this license:
- HdrHistogram (from http://hdrhistogram.github.io/HdrHistogram/)
- Joda-Time (from http://www.joda.org/joda-time/)
- LatencyUtils (from http://latencyutils.github.io/LatencyUtils/)
+- Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java)
+- Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java)
+- Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java)
+- Microsoft Azure client library for Blob Storage (from https://github.com/Azure/azure-sdk-for-java)
- Project Lombok (from https://projectlombok.org)
- Spring Security JWT Library (from http://github.com/spring-projects/spring-security-oauth)
- Spring Web (from https://github.com/spring-projects/spring-framework)
@@ -662,7 +937,10 @@ The following software have components provided under the terms of this license:
- Bouncy Castle PKIX, CMS, EAC, TSP, PKCS, OCSP, CMP, and CRMF APIs (from http://www.bouncycastle.org/java.html)
- Bouncy Castle Provider (from http://www.bouncycastle.org/java.html)
+- Byte Buddy (without dependencies) (from )
+- Servlet Specification 2.5 API (from )
- System Rules (from http://stefanbirkner.github.io/system-rules/)
+- jaxen (from http://jaxen.codehaus.org/)
- jersey-container-servlet (from git://java.net/jersey~code/project/jersey-container-servlet)
- jersey-container-servlet-core (from git://java.net/jersey~code/project/jersey-container-servlet-core)
- jersey-core-client (from git://java.net/jersey~code/jersey-client)
@@ -673,5 +951,6 @@ The following software have components provided under the terms of this license:
- jersey-spring4 (from )
- jts-core (from )
- jts-io-common (from )
+- xml-apis (from )
diff --git a/devops/azure-pipeline.yml b/devops/azure-pipeline.yml
new file mode 100644
index 0000000000000000000000000000000000000000..f18e06e92989fdc4ffd5830653af8c4f3907dcab
--- /dev/null
+++ b/devops/azure-pipeline.yml
@@ -0,0 +1,63 @@
+#####################
+# README: Defines a template to be used as a starting point for defining a service pipeline
+#####################
+
+trigger:
+ batch: true
+ branches:
+ include:
+ - master
+ paths:
+ exclude:
+ - /**/*.md
+ - .gitignore
+ - images/
+
+pr:
+ autoCancel: true
+ branches:
+ include:
+ - '*'
+ paths:
+ exclude:
+ - /**/*.md
+ - .gitignore
+ - images/
+
+resources:
+ repositories:
+ - repository: osdu-infrastructure
+ type: git
+ name: osdu-r2/osdu-infrastructure
+
+variables:
+ - group: 'Azure Common Secrets'
+ - group: 'Azure - Common'
+
+ - name: serviceName
+ value: 'indexer'
+
+stages:
+ - template: devops/service-pipelines/build-stage.yml@osdu-infrastructure
+ parameters:
+ mavenGoal: 'package'
+ mavenPublishJUnitResults: true
+ serviceCoreMavenOptions: '-P indexer-core'
+ mavenOptions: '-P indexer-azure'
+ copyFileContents: |
+ pom.xml
+ provider/indexer-azure/maven/settings.xml
+ provider/indexer-azure/pom.xml
+ provider/indexer-azure/target/*-spring-boot.jar
+ copyFileContentsToFlatten: ''
+ mavenSettingsFile: './maven/settings.xml'
+ serviceBase: ${{ variables.serviceName }}
+ testingRootFolder: 'testing'
+ - template: devops/service-pipelines/deploy-stages.yml@osdu-infrastructure
+ parameters:
+ serviceName: ${{ variables.serviceName }}
+ testCoreMavenPomFile: 'testing/indexer-test-core/pom.xml'
+ testCoreMavenOptions: '--settings $(System.DefaultWorkingDirectory)/drop/deploy/testing/maven/settings.xml'
+ providers:
+ - name: Azure
+ environments: ['demo']
diff --git a/indexer-core/mvnw b/indexer-core/mvnw
old mode 100644
new mode 100755
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/ElasticClientHandler.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/ElasticClientHandler.java
index 3ec68d5c3f4e056ed6939ac73268973f78f40905..7035abf3ecb9820e4f09b1c7f3ff89966437152a 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/ElasticClientHandler.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/ElasticClientHandler.java
@@ -105,4 +105,4 @@ public class ElasticClientHandler {
builder.setDefaultHeaders(defaultHeaders);
return builder;
}
-}
+}
\ No newline at end of file
diff --git a/provider/indexer-aws/.env.template b/provider/indexer-aws/.env.template
index b55ecdc872088111f1c3bb74f33a75bfb4af3253..c162a98fb2867cc81f200289a3d446c57e918112 100644
--- a/provider/indexer-aws/.env.template
+++ b/provider/indexer-aws/.env.template
@@ -56,6 +56,7 @@ SNS_TOPIC_NAME=
SNS_STORAGE_TOPIC_NAME=
ENVIRONMENT=
AWS_REGION=
+LOG_LEVEL=
##### Integration test-specific - these are only used for integration tests, not the app ###
OTHER_RELEVANT_DATA_COUNTRIES=
diff --git a/provider/indexer-aws/CloudFormation/Automated/cache.yml b/provider/indexer-aws/CloudFormation/Automated/cache.yml
deleted file mode 100644
index 207f9815a887994d91731ced4215c00403587394..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/CloudFormation/Automated/cache.yml
+++ /dev/null
@@ -1,205 +0,0 @@
-# Copyright © Amazon Web Services
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-AWSTemplateFormatVersion: 2010-09-09
-Description: >-
- CloudFormation template for creating the resources used for the tenant info database for OSDU.
- It creates the DynamoDB table and the API Gateway endpoints.
-
-Parameters:
- Environment:
- Description: An environment name that will be prefixed to resource names.
- Type: String
- AllowedValues:
- - dev
- - uat
- - prod
- ConstraintDescription: Can only be "dev/uat/prod"
- Default: dev
-
- Region:
- Description: The AWS region to deploy the resources to.
- Type: String
- Default: us-east-1
-
- ApplicationName:
- Description: >
- The name of the application, which will be used to generate the ECS cluster name.
- It will be prefixed with the environment name.
- Type: String
- MinLength: '1'
- MaxLength: '64'
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: os-indexer
-
- CacheName:
- Description: The name of the cache cluster. Will be prefixed with the environment name.
- Type: String
- MinLength: '1'
- MaxLength: '64'
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: cache
-
- CacheEngine:
- Description: Which caching platform to use. Can be set to 'redis' or 'memcached'.
- Type: String
- AllowedValues:
- - redis
- - memcached
- ConstraintDescription: Can only be "redis" or "memcached"
- Default: redis
-
- NodeInstanceType:
- Description: The instance type for redis cache nodes.
- ConstraintDescription: Must be a valid instance type from the list of allowed values.
- Default: cache.t2.micro
- AllowedValues:
- - cache.m5.large
- - cache.m5.xlarge
- - cache.m5.2xlarge
- - cache.m5.4xlarge
- - cache.m5.12xlarge
- - cache.m5.24xlarge
- - cache.m4.large
- - cache.m4.xlarge
- - cache.m4.2xlarge
- - cache.m4.4xlarge
- - cache.m4.10xlarge
- - cache.t2.micro
- - cache.t2.small
- - cache.t2.medium
- - cache.c1.xlarge
- - cache.r5.large
- - cache.r5.xlarge
- - cache.r5.2xlarge
- - cache.r5.4xlarge
- - cache.r5.12xlarge
- - cache.r5.24xlarge
- - cache.r4.large
- - cache.r4.xlarge
- - cache.r4.2xlarge
- - cache.r4.4xlarge
- - cache.r4.8xlarge
- - cache.r4.16xlarge
- Type: String
-
- NumberOfCacheNodes:
- Description: An integer value specifying the number of node in the redis cache.
- Type: Number
- Default: 1
- MinValue: 1
- MaxValue: 128
-
-Conditions:
- IsSingleNode: !Equals [ !Ref NumberOfCacheNodes, 1 ]
-
- IsClustered: !Not [Condition: IsSingleNode]
-
- IsMemcached: !Equals [ !Ref CacheEngine, memcached ]
-
- IsRedis: !Equals [ !Ref CacheEngine, redis ]
-
-Resources:
- ElastiCacheVpcSecurityGroup:
- Type: AWS::EC2::SecurityGroup
- Properties:
- GroupName: !Sub "${Environment}-${CacheName}-sg"
- GroupDescription: "This is the security group that all of our ElastiCache cluster will be placed into."
- VpcId:
- Fn::ImportValue:
- !Sub "${Environment}-OSDU-VPC"
-
- ElastiCacheVpcSecurityGroupCodeBuildIngress:
- Type: AWS::EC2::SecurityGroupIngress
- Properties:
- GroupId: !Ref ElastiCacheVpcSecurityGroup
- IpProtocol: tcp
- FromPort: "6379"
- ToPort: "6379"
- SourceSecurityGroupId:
- Fn::ImportValue:
- !Sub "${Environment}-OSDU-CodeBuildSecurityGroup"
-
- ElastiCacheVpcSecurityGroupECSIngress:
- Type: AWS::EC2::SecurityGroupIngress
- Properties:
- GroupId: !Ref ElastiCacheVpcSecurityGroup
- IpProtocol: tcp
- FromPort: "6379"
- ToPort: "6379"
- SourceSecurityGroupId:
- Fn::ImportValue:
- !Sub "${Environment}-${ApplicationName}-EcsNetworkSecurityGroupId"
-
- ElastiCacheSubnetGroup:
- Type: 'AWS::ElastiCache::SubnetGroup'
- Properties:
- CacheSubnetGroupName: !Sub ${Environment}-${CacheName}-SubnetGroup
- Description: Redis cache VPC subnet group.
- SubnetIds:
- - Fn::ImportValue:
- !Sub "${Environment}-OSDU-PrivateSubnet-AZ1"
- - Fn::ImportValue:
- !Sub "${Environment}-OSDU-PrivateSubnet-AZ2"
-
- ElastiCacheCluster:
- Type: 'AWS::ElastiCache::CacheCluster'
- DependsOn: ElastiCacheSubnetGroup
- Properties:
- AutoMinorVersionUpgrade: 'false'
- AZMode: single-az # this parameter only affects Memcached clusters
- Engine: !Ref CacheEngine
- CacheNodeType: !Ref NodeInstanceType
- NumCacheNodes: !Ref NumberOfCacheNodes
- ClusterName: !Sub ${Environment}-${CacheName}
- CacheSubnetGroupName: !Ref ElastiCacheSubnetGroup
- VpcSecurityGroupIds:
- - Ref: ElastiCacheVpcSecurityGroup
-
-Outputs:
- # Redis (cluster mode disabled) replication groups don't have this attribute.
- # Therefore, Fn::GetAtt returns a value for this attribute only if the replication
- # group is clustered. Otherwise, Fn::GetAtt fails.
- ElastiCacheConfigurationEndpointUrl:
- Description: The configuration endpoint URL of the cache node.
- Value: !GetAtt ElastiCacheCluster.ConfigurationEndpoint.Address
- Condition: IsClustered
- Export:
- Name: !Sub ${Environment}-${CacheName}-ElastiCacheConfigurationEndpointUrl
-
- # This output is only applicable if the cache engine is set to Memcached
- MemcachedConfigurationEndpointPort:
- Description: The Memcached configuration endpoint port of the cache node.
- Value: !GetAtt ElastiCacheCluster.ConfigurationEndpoint.Port
- Condition: IsMemcached
- Export:
- Name: !Sub ${Environment}-${CacheName}-MemcachedConfigurationEndpointPort
-
- # This output is only applicable if the cache engine is set to Redis
- RedisEndpointAddress:
- Description: The Redis endpoint address of the cache.
- Value: !GetAtt ElastiCacheCluster.RedisEndpoint.Address
- Condition: IsRedis
- Export:
- Name: !Sub ${Environment}-${CacheName}-RedisEndpointAddress
-
- # This output is only applicable if the cache engine is set to Redis
- RedisEndpointPort:
- Description: The Redis endpoint port of the cache.
- Value: !GetAtt ElastiCacheCluster.RedisEndpoint.Port
- Condition: IsRedis
- Export:
- Name: !Sub ${Environment}-${CacheName}-RedisEndpointPort
diff --git a/provider/indexer-aws/CloudFormation/Automated/ecs-cluster.yml b/provider/indexer-aws/CloudFormation/Automated/ecs-cluster.yml
deleted file mode 100644
index 3a357d30c1edaedc60a366f266552ee354850c38..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/CloudFormation/Automated/ecs-cluster.yml
+++ /dev/null
@@ -1,764 +0,0 @@
-# Copyright © Amazon Web Services
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-AWSTemplateFormatVersion: 2010-09-09
-Description: >-
- CloudFormation template for creating the resources used for the ECS cluster the application will
- be deployed into. Will create the CodeDeploy application, the ECR repository, and the ECS cluster.
- This is separated from the rest of the ECS resources in order to avoid a circular dependency.
- Because there can be any number of cache stacks, but only one ECS stack per service, it makes sense to have
- the caches import ECS exports in order to permit access from ECS, rather than the other way around, since
- the number of cache clusters and their names can vary, and would require hardcoding them into the ECS template,
- whereas this way things stay generic and the ECS CloudFormation template does not need to be updated in order to
- add or remove ElastiCache clusters.
-
-Parameters:
- Environment:
- Description: An environment name that will be prefixed to resource names.
- Type: String
- AllowedValues:
- - dev
- - uat
- - prod
- ConstraintDescription: Can only be "dev/uat/prod"
- Default: dev
-
- Region:
- Description: The AWS region to deploy the resources to.
- Type: String
- Default: us-east-1
-
- ApplicationName:
- Description: >
- The name of the application, which will be used to generate the ECS cluster name.
- It will be prefixed with the environment name.
- Type: String
- MinLength: '1'
- MaxLength: '64'
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: os-indexer
-
- KeyName:
- Description: >
- Name of an existing EC2 KeyPair to enable SSH access to the ECS instances. Note that key pairs cannot
- be created through CloudFormation, but instead must be uploaded through the AWS Console.
- Type: AWS::EC2::KeyPair::KeyName
- Default: ecs_indexer_key
-
- DesiredCapacity:
- Description: The default number of instances to launch in the ECS cluster.
- Type: Number
- Default: '1'
-
- MaxSize:
- Description: Maximum number of instances that can be launched in the ECS cluster.
- Type: Number
- Default: '1'
-
- InstanceType:
- Description: EC2 instance type
- Type: String
- Default: t3.large
- AllowedValues:
- - m5.large
- - m5.xlarge
- - m5.2xlarge
- - m5.4xlarge
- - m5.12xlarge
- - m5.16xlarge
- - m5.24xlarge
- - m4.large
- - m4.xlarge
- - m4.2xlarge
- - m4.4xlarge
- - m4.10xlarge
- - m4.16xlarge
- - t3.nano
- - t3.micro
- - t3.small
- - t3.medium
- - t3.large
- - t3.xlarge
- - t3.2xlarge
- - c5.large
- - c5.xlarge
- - c5.2xlarge
- - c5.4xlarge
- - c5.12xlarge
- - c5.16xlarge
- - c5.24xlarge
- - r5.large
- - r5.xlarge
- - r5.2xlarge
- - r5.4xlarge
- - r5.12xlarge
- - r5.24xlarge
- - r4.large
- - r4.xlarge
- - r4.2xlarge
- - r4.4xlarge
- - r4.8xlarge
- - r4.16xlarge
- - i3.large
- - i3.xlarge
- - i3.2xlarge
- - i3.4xlarge
- - i3.10xlarge
- - i3.16xlarge
- - x1e.xlarge
- - x1e.2xlarge
- - x1e.4xlarge
- - x1e.8xlarge
- - x1e.16xlarge
- - x1e.32xlarge
- ConstraintDescription: Please choose a valid EC2 instance type for the ECS container instances.
-
- SchemaCacheName:
- Description: The name of the cache cluster for the schema cache. Will be prefixed with the environment name.
- Type: String
- MinLength: '1'
- MaxLength: '64'
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: indexerSchemaCache
-
- IndexCacheName:
- Description: The name of the cache cluster for the index cache. Will be prefixed with the environment name.
- Type: String
- MinLength: '1'
- MaxLength: '64'
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: indexerIndexCache
-
- ECSPort:
- Description: The port that the ECS Service will listen on.
- Type: Number
- Default: 80
- MinValue: 1
- MaxValue: 65535
-
- SNSTopicName:
- Description: >-
- The name of the Simple Notification Service topic for the OS Indexer Service. Defaults to osdu-indexer-messages.
- Will be prefixed with the environment name.
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: osdu-indexer-messages
- Type: String
- MinLength: '1'
- MaxLength: '64'
-
- ECSCPUAllocation:
- Description: The amount of CPU resources to allocate to each ECS task/container. Scale - 1024 = 1 vCPU core.
- Type: Number
- Default: 1024
- MinValue: 10
- MaxValue: 65535
-
- ECSMemoryAllocation:
- Description: The amount of memory (RAM) to allocate to each ECS task/container. Scale - 1 = 1MB of memory.
- Type: Number
- Default: 2048
- MinValue: 256
- MaxValue: 131072
-
- DomainName:
- Description: >-
- The optional custom DNS name for the ECS service's load balancer. If omitted, the site will only be accessible
- via the ECS service's Application Load Balancer DNS name. This value is used in the creation and signing of
- the service's SSL certificate. Leave blank is not using a custom domain for this deployment.
- Type: String
- Default: ''
-
- HostedZoneName:
- Description: >-
- The name of the hosted zone (ex: for indexer.osdu.slb.com, this would likely be osdu.slb.com).
- Leave blank is not using a custom domain for this deployment.
- Type: String
- Default: ''
-
- ElasticsearchDomainName:
- Description: The name of the Elasticsearch domain. Will be prefixed with the environment name.
- Type: String
- MinLength: '1'
- MaxLength: '64'
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: osdu-indexer
-
-Mappings:
- # This mapping is for the ECS-optimized edition of the November 13-14, 2019 release of the Amazon Linux 2 AMI
- # It will need to be periodically updated as new versions are released by Amazon.
- # The latest ECS-optimized AMI IDs can be found here: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs-optimized_AMI.html
- # The mapping is used to input the correct AMI ID based on the region the instance is being spun up in.
- AWSRegionToAMI:
- us-east-1:
- AMIID: ami-097e3d1cdb541f43e
- us-east-2:
- AMIID: ami-0fbd313043845c4f2
- us-west-1:
- AMIID: ami-03d7632ea0ab75eaa
- us-west-2:
- AMIID: ami-0fb71e703258ab7eb
- eu-north-1:
- AMIID: ami-0f8edbbca6bac13a6
- eu-west-1:
- AMIID: ami-0bf45a5f4ab05b949
- eu-west-2:
- AMIID: ami-0393b5f363fbd613a
- eu-west-3:
- AMIID: ami-03490ca40775a62f0
- eu-central-1:
- AMIID: ami-074dc9dd588b6ea52
- ap-northeast-1:
- AMIID: ami-0934e28fe3e390537
- ap-northeast-2:
- AMIID: ami-0fa5d85859452a178
- ap-south-1:
- AMIID: ami-0312d67ff59a3db34
- ap-southeast-1:
- AMIID: ami-01f07b3fa86406c96
- ap-southeast-2:
- AMIID: ami-07610e278b1ddf331
- ca-central-1:
- AMIID: ami-0057d82f917a17334
- sa-east-1:
- AMIID: ami-0c947c117562538ee
-
-Conditions:
- IncludeCustomDomain: !Not [!Equals [ !Ref DomainName, '' ]]
- IsPortStandardSSL:
- !Or [!Equals [ !Ref ECSPort, '443' ], !Equals [ !Ref ECSPort, '8443' ]]
- IsLoadBalancerHTTPS: !And # HTTPS for ECS requires a custom domain, but CloudFront will still have HTTPS/SSL
- - !Condition IncludeCustomDomain
- - !Condition IsPortStandardSSL
-
-Resources:
- # This sets up a Route 53 record for CloudFront if a custom domain is being used,
- # otherwise a default cloudfront.net value will be used instead
- CloudFrontDNSName:
- Type: AWS::Route53::RecordSetGroup
- Condition: IncludeCustomDomain
- Properties:
- HostedZoneName: !Join ['', [!Ref HostedZoneName, .]] # Route 53 requires a trailing period
- RecordSets:
- - Name: !Ref DomainName
- Type: A
- AliasTarget:
- # This hosted zone ID is for ALL CloudFront distributions, always, and should be hard-coded
- HostedZoneId: Z2FDTNDATAQYW2
- DNSName: !GetAtt ECSCloudFrontDistribution.DomainName
-
- # This sets up a Route 53 record for the ECS ALB origin if a custom domain is being used
- ECSDNSName:
- Type: AWS::Route53::RecordSetGroup
- Condition: IncludeCustomDomain
- Properties:
- HostedZoneName: !Join ['', [!Ref HostedZoneName, .]] # Route 53 requires a trailing period
- RecordSets:
- - Name: !Join ['.', ['origin', !Ref DomainName]] # prefix the ECS origin record with 'origin.'
- Type: A
- AliasTarget:
- HostedZoneId: !GetAtt ECSALB.CanonicalHostedZoneID # this value comes from the ALB attributes
- DNSName: !GetAtt ECSALB.DNSName
- EvaluateTargetHealth: true # Route 53 routes traffic to ECS targets based on their health checks
- DependsOn: ECSALB
-
- CodeDeployApplication:
- Type: AWS::CodeDeploy::Application
- Properties:
- ApplicationName: !Sub ${Environment}-${ApplicationName}-code-deploy
- ComputePlatform: ECS
-
- ECRRepository:
- Type: AWS::ECR::Repository
- Properties:
- RepositoryName: !Sub ${Environment}-${ApplicationName}-repository
- RepositoryPolicyText:
- Version: "2012-10-17"
- Statement:
- - Sid: AllowPushPull
- Effect: Allow
- Principal:
- AWS:
- - !Sub arn:aws:iam::${AWS::AccountId}:root
- - Fn::ImportValue:
- !Sub "${Environment}-CodeBuildRoleArn"
- - Fn::ImportValue:
- !Sub "${Environment}-CFNRoleArn"
- - Fn::ImportValue:
- !Sub "${Environment}-PipelineRoleArn"
- Service:
- - codebuild.amazonaws.com
- Action:
- - "ecr:GetDownloadUrlForLayer"
- - "ecr:BatchGetImage"
- - "ecr:BatchCheckLayerAvailability"
- - "ecr:PutImage"
- - "ecr:InitiateLayerUpload"
- - "ecr:UploadLayerPart"
- - "ecr:CompleteLayerUpload"
-
- ApplicationECSCluster:
- Type: AWS::ECS::Cluster
- Properties:
- ClusterName: !Sub ${Environment}-${ApplicationName}-cluster
- Tags:
- - Key: Environment
- Value: !Ref Environment
-
- CloudWatchLogsGroup:
- Type: AWS::Logs::LogGroup
- Properties:
- LogGroupName: !Join ['-', [ECSLogGroup, !Ref 'ApplicationName']]
- RetentionInDays: 365
-
- TaskDefinition:
- Type: AWS::ECS::TaskDefinition
- Properties:
- Family: !Join ['', [!Ref 'AWS::StackName', -, !Ref 'ApplicationName']]
- ContainerDefinitions:
- - Name: !Ref 'ApplicationName'
- Cpu: !Ref ECSCPUAllocation
- Essential: 'true'
- Image: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${Environment}-${ApplicationName}-repository:latest
- Memory: !Ref ECSMemoryAllocation
- LogConfiguration:
- LogDriver: awslogs
- Options:
- awslogs-group: !Ref 'CloudWatchLogsGroup'
- awslogs-region: !Ref 'AWS::Region'
- awslogs-stream-prefix: !Ref 'ApplicationName'
- MountPoints:
- - ContainerPath: /root/.m2
- SourceVolume: docker-volume
- PortMappings:
- - ContainerPort: !Ref ECSPort
- Environment:
- - Name: AWS_ACCESS_KEY_ID
- Value: '{{resolve:secretsmanager:dev-IndexerServiceIamCredentials:SecretString:access_key}}'
- - Name: AWS_SECRET_KEY
- Value: '{{resolve:secretsmanager:dev-IndexerServiceIamCredentials:SecretString:secret_key}}'
- - Name: ENVIRONMENT
- Value: !Ref Environment
- - Name: VSTS_FEED_USER
- Value: '{{resolve:secretsmanager:dev-VSTSFeedToken:SecretString:vsts_feed_user}}'
- - Name: VSTS_FEED_TOKEN
- Value: '{{resolve:secretsmanager:dev-VSTSFeedToken:SecretString:vsts_feed_token}}'
- - Name: CACHE_CLUSTER_SCHEMA_ENDPOINT
- Value:
- Fn::ImportValue:
- !Sub "${Environment}-${SchemaCacheName}-RedisEndpointAddress"
- - Name: CACHE_CLUSTER_SCHEMA_PORT
- Value:
- Fn::ImportValue:
- !Sub "${Environment}-${SchemaCacheName}-RedisEndpointPort"
- - Name: CACHE_CLUSTER_INDEX_ENDPOINT
- Value:
- Fn::ImportValue:
- !Sub "${Environment}-${IndexCacheName}-RedisEndpointAddress"
- - Name: CACHE_CLUSTER_INDEX_PORT
- Value:
- Fn::ImportValue:
- !Sub "${Environment}-${IndexCacheName}-RedisEndpointPort"
- - Name: APPLICATION_PORT
- Value: !Ref ECSPort
- - Name: AWS_REGION
- Value: !Ref 'AWS::Region'
- - Name: AWS_ACCOUNT_ID
- Value: !Ref 'AWS::AccountId'
- - Name: SNS_TOPIC_NAME
- Value: !Ref SNSTopicName
- - Name: ELASTIC_HOST
- Value:
- Fn::ImportValue:
- !Sub "${Environment}-${ElasticsearchDomainName}-ElasticsearchDomainEndpoint"
- - Name: ELASTIC_PORT
- Value: '443' # the Elasticsearch port is not configurable on AWS, and is always 80 for HTTP and 443 for HTTPS, so there's no value in using a CFN parameter
- - Name: JAVA_HEAP_MEMORY
- Value: !Ref ECSMemoryAllocation
- - Name: STORAGE_HOST
- Value:
- Fn::ImportValue:
- !Sub "${Environment}-os-storage-EcsCloudFrontDomainName"
- - Name: SNS_STORAGE_TOPIC_NAME
- Value:
- Fn::ImportValue:
- !Sub "${Environment}-OSDUStorageSNSTopic"
- Volumes:
- - Name: docker-volume
-
- ECSALB:
- Type: AWS::ElasticLoadBalancingV2::LoadBalancer
- Properties:
- Name: !Sub ECSALB-${ApplicationName}
- Scheme: internet-facing
- LoadBalancerAttributes:
- - Key: idle_timeout.timeout_seconds
- Value: '30'
- Subnets:
- - Fn::ImportValue:
- !Sub "${Environment}-OSDU-PublicSubnet-AZ1"
- - Fn::ImportValue:
- !Sub "${Environment}-OSDU-PublicSubnet-AZ2"
- SecurityGroups:
- - Fn::ImportValue:
- !Sub "${Environment}-${ApplicationName}-EcsNetworkSecurityGroupId"
-
- ALBListener:
- Type: AWS::ElasticLoadBalancingV2::Listener
- DependsOn: ECSServiceRole
- Properties:
- DefaultActions:
- - Type: forward
- TargetGroupArn: !Ref 'ECSTargetGroup'
- LoadBalancerArn: !Ref 'ECSALB'
- Port: !Ref ECSPort
- Protocol: !If [IsLoadBalancerHTTPS, HTTPS, HTTP]
-
- LoadBalancerALBListenerCertificate:
- Type: AWS::ElasticLoadBalancingV2::ListenerCertificate
- Condition: IncludeCustomDomain
- Properties:
- Certificates:
- - Fn::ImportValue:
- !Sub "${Environment}-${ApplicationName}-LoadBalancerSSLCertificateArn"
- ListenerArn: !Ref 'ALBListener'
-
- ECSALBPrimaryListenerRule:
- Type: AWS::ElasticLoadBalancingV2::ListenerRule
- DependsOn: ALBListener
- Properties:
- Actions:
- - Type: forward
- TargetGroupArn: !Ref 'ECSTargetGroup'
- Conditions:
- - Field: path-pattern
- Values: [/]
- ListenerArn: !Ref 'ALBListener'
- Priority: 1
-
- ECSTargetGroup:
- Type: AWS::ElasticLoadBalancingV2::TargetGroup
- DependsOn: ECSALB
- Properties:
- HealthCheckIntervalSeconds: 120
- HealthCheckPath: /api/indexer/v2/liveness_check
- HealthCheckProtocol: !If [IsLoadBalancerHTTPS, HTTPS, HTTP]
- HealthCheckTimeoutSeconds: 5
- HealthyThresholdCount: 2
- Name: !Sub ECSTargetGroup-New-${ApplicationName}
- Port: !Ref ECSPort
- Protocol: !If [IsLoadBalancerHTTPS, HTTPS, HTTP]
- UnhealthyThresholdCount: 2
- VpcId:
- Fn::ImportValue:
- !Sub "${Environment}-OSDU-VPC"
-
- ECSCloudFrontDistribution:
- Type: AWS::CloudFront::Distribution
- DependsOn: ECSALB
- Properties:
- DistributionConfig:
- Comment: 'Cloudfront Distribution pointing ALB Origin'
- Origins:
- - DomainName: !GetAtt 'ECSALB.DNSName'
- Id: !Ref 'ECSALB'
- CustomOriginConfig:
- HTTPPort: !Ref ECSPort # The ports are the same because we'll only ever be accessing the ECS cluster over one protocol, as set in OriginProtocolPolicy below
- HTTPSPort: !Ref ECSPort # The ports are the same because we'll only ever be accessing the ECS cluster over one protocol, as set in OriginProtocolPolicy below
- OriginProtocolPolicy: !If [IsLoadBalancerHTTPS, https-only, http-only] # this only affects the origin, not CloudFront / the user's request
- OriginKeepaliveTimeout: '60'
- OriginReadTimeout: '60'
- OriginSSLProtocols:
- - TLSv1
- - TLSv1.1
- - TLSv1.2
- - SSLv3
- Enabled: true
- HttpVersion: 'http2'
- Aliases:
- - Fn::If:
- - IncludeCustomDomain
- - !Ref DomainName
- - !Ref AWS::NoValue
- DefaultCacheBehavior:
- AllowedMethods:
- - GET
- - HEAD
- - OPTIONS
- - PUT
- - POST
- - PATCH
- - DELETE
- Compress: true
- TargetOriginId: !Ref 'ECSALB'
- DefaultTTL: 5
- MaxTTL: 30
- ForwardedValues:
- QueryString: true
- Cookies:
- Forward: all
- Headers:
- - Authorization
- - Data-Partition-Id
- - Content-Type
- - Kind
- - Limit
- - Cursor
- ViewerProtocolPolicy: redirect-to-https # CloudFront requests will always be HTTPS, regardless of the origin or the request
- ViewerCertificate:
- AcmCertificateArn:
- Fn::If:
- - IncludeCustomDomain
- - Fn::ImportValue:
- !Sub "${Environment}-${ApplicationName}-LoadBalancerSSLCertificateArn"
- - Ref: AWS::NoValue
- CloudFrontDefaultCertificate:
- Fn::If:
- - IncludeCustomDomain
- - Ref: AWS::NoValue
- - true
- SslSupportMethod:
- Fn::If:
- - IncludeCustomDomain
- - sni-only # sni-only is free; 'vip' is the only other option, which allows viewers without Server Name Indication (SNI) support by using dedicated IP addresses, but it costs $600/mo per SSL certificate
- - Ref: AWS::NoValue
- MinimumProtocolVersion:
- Fn::If:
- - IncludeCustomDomain
- - TLSv1
- - Ref: AWS::NoValue # this is not used when using the default CloudFront certificate (which is always TLSv1)
-
- ECSAutoScalingGroup:
- Type: AWS::AutoScaling::AutoScalingGroup
- Properties:
- VPCZoneIdentifier:
- - Fn::ImportValue:
- !Sub "${Environment}-OSDU-PublicSubnet-AZ1"
- - Fn::ImportValue:
- !Sub "${Environment}-OSDU-PublicSubnet-AZ2"
- LaunchConfigurationName: !Ref 'ContainerInstances'
- MinSize: '1'
- MaxSize: !Ref 'MaxSize'
- DesiredCapacity: !Ref 'DesiredCapacity'
- CreationPolicy:
- ResourceSignal:
- Timeout: PT15M
- UpdatePolicy:
- AutoScalingReplacingUpdate:
- WillReplace: 'true'
-
- ContainerInstances:
- Type: AWS::AutoScaling::LaunchConfiguration
- Properties:
- ImageId: !FindInMap [AWSRegionToAMI, !Ref 'AWS::Region', AMIID]
- SecurityGroups:
- - Fn::ImportValue:
- !Sub "${Environment}-${ApplicationName}-EcsNetworkSecurityGroupId"
- InstanceType: !Ref 'InstanceType'
- IamInstanceProfile: !Ref 'EC2InstanceProfile'
- KeyName: !Ref 'KeyName'
- UserData:
- Fn::Base64: !Sub |
- #!/bin/bash -xe
- echo ECS_CLUSTER=${ApplicationECSCluster} >> /etc/ecs/ecs.config
- yum install -y aws-cfn-bootstrap
- /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource ECSAutoScalingGroup --region ${AWS::Region}
-
- Service:
- Type: AWS::ECS::Service
- DependsOn: ALBListener
- Properties:
- Cluster: !Ref 'ApplicationECSCluster'
- DesiredCount: '1'
- LoadBalancers:
- - ContainerName: !Ref 'ApplicationName'
- ContainerPort: !Ref ECSPort
- TargetGroupArn: !Ref 'ECSTargetGroup'
- Role: !Ref 'ECSServiceRole'
- TaskDefinition: !Ref 'TaskDefinition'
-
- ECSServiceRole:
- Type: AWS::IAM::Role
- Properties:
- AssumeRolePolicyDocument:
- Statement:
- - Effect: Allow
- Principal:
- Service: [ecs.amazonaws.com]
- Action: ['sts:AssumeRole']
- Path: /
- Policies:
- - PolicyName: !Sub ${Environment}-${ApplicationName}-ecs-service
- PolicyDocument:
- Statement:
- - Effect: Allow
- Action: ['elasticloadbalancing:DeregisterInstancesFromLoadBalancer', 'elasticloadbalancing:DeregisterTargets',
- 'elasticloadbalancing:Describe*', 'elasticloadbalancing:RegisterInstancesWithLoadBalancer',
- 'elasticloadbalancing:RegisterTargets', 'ec2:Describe*', 'ec2:AuthorizeSecurityGroupIngress']
- Resource: '*'
-
- ServiceScalingTarget:
- Type: AWS::ApplicationAutoScaling::ScalableTarget
- DependsOn: Service
- Properties:
- MaxCapacity: 2
- MinCapacity: 1
- ResourceId: !Join ['', [service/, !Ref 'ApplicationECSCluster', /, !GetAtt [Service, Name]]]
- RoleARN: !GetAtt [AutoscalingRole, Arn]
- ScalableDimension: ecs:service:DesiredCount
- ServiceNamespace: ecs
-
- ServiceScalingPolicy:
- Type: AWS::ApplicationAutoScaling::ScalingPolicy
- Properties:
- PolicyName: !Sub ScalingPolicy-${ApplicationName}
- PolicyType: StepScaling
- ScalingTargetId: !Ref 'ServiceScalingTarget'
- StepScalingPolicyConfiguration:
- AdjustmentType: PercentChangeInCapacity
- Cooldown: 60
- MetricAggregationType: Average
- StepAdjustments:
- - MetricIntervalLowerBound: 0
- ScalingAdjustment: 200
-
- ALB500sAlarmScaleUp:
- Type: AWS::CloudWatch::Alarm
- Properties:
- EvaluationPeriods: '1'
- Statistic: Average
- Threshold: '10'
- AlarmDescription: Alarm triggering ECS to scale up if our ALB generates too many HTTP 500 errors.
- Period: '60'
- AlarmActions: [!Ref 'ServiceScalingPolicy']
- Namespace: AWS/ApplicationELB
- Dimensions:
- - Name: LoadBalancer
- Value: !GetAtt
- - ECSALB
- - LoadBalancerFullName
- ComparisonOperator: GreaterThanThreshold
- MetricName: HTTPCode_ELB_5XX_Count
-
- EC2Role:
- Type: AWS::IAM::Role
- Properties:
- AssumeRolePolicyDocument:
- Statement:
- - Effect: Allow
- Principal:
- Service: [ec2.amazonaws.com]
- Action: ['sts:AssumeRole']
- Path: /
- Policies:
- - PolicyName: !Sub ${Environment}-${ApplicationName}-ecs-service
- PolicyDocument:
- Statement:
- - Effect: Allow
- Action: ['ecs:CreateCluster', 'ecs:DeregisterContainerInstance', 'ecs:DiscoverPollEndpoint',
- 'ecs:Poll', 'ecs:RegisterContainerInstance', 'ecs:StartTelemetrySession',
- 'ecs:Submit*', 'logs:CreateLogStream', 'logs:PutLogEvents', 'ecr:*']
- Resource: '*'
-
- AutoscalingRole:
- Type: AWS::IAM::Role
- Properties:
- AssumeRolePolicyDocument:
- Statement:
- - Effect: Allow
- Principal:
- Service: [application-autoscaling.amazonaws.com]
- Action: ['sts:AssumeRole']
- Path: /
- Policies:
- - PolicyName: !Sub ${Environment}-${ApplicationName}-service-autoscaling
- PolicyDocument:
- Statement:
- - Effect: Allow
- Action: ['application-autoscaling:*', 'cloudwatch:DescribeAlarms', 'cloudwatch:PutMetricAlarm',
- 'ecs:DescribeServices', 'ecs:UpdateService']
- Resource: '*'
-
- EC2InstanceProfile:
- Type: AWS::IAM::InstanceProfile
- Properties:
- Path: /
- Roles: [!Ref 'EC2Role']
-
-Outputs:
- ApplicationECSClusterArn:
- Description: The ARN of the application's ECS cluster.
- Value: !GetAtt ApplicationECSCluster.Arn
- Export:
- Name: !Sub ${Environment}-${ApplicationName}-EcsClusterArn
-
- ApplicationECSClusterName:
- Description: The logical name of the application's ECS cluster.
- Value: !Ref ApplicationECSCluster
- Export:
- Name: !Sub ${Environment}-${ApplicationName}-EcsClusterName
-
- ECSServiceArn:
- Description: The ARN of the Indexer Service service in the ECS cluster.
- Value: !Ref 'Service'
- Export:
- Name: !Sub ${Environment}-${ApplicationName}-EcsServiceArn
-
- ECSServiceName:
- Description: The name of the Legal Service service in the ECS cluster.
- Value: !GetAtt Service.Name
- Export:
- Name: !Sub ${Environment}-${ApplicationName}-EcsServiceName
-
- ECSALBUrl:
- Description: The Indexer Service ALB DNS URL.
- Value: !Join ['', [!GetAtt [ECSALB, DNSName]]]
- Export:
- Name: !Sub ${Environment}-${ApplicationName}-EcsAlbUrl
-
- ECSALBCustomDNSName:
- Description: The custom DNS name of the ECS service's ALB origin.
- Condition: IncludeCustomDomain
- Value: !Join ['.', ['origin', !Ref DomainName]]
- Export:
- Name: !Sub ${Environment}-${ApplicationName}-EcsAlbCustomDnsName
-
- ECSCloudFrontCustomDNSName:
- Description: The custom DNS name of the ECS service's CloudFront Distribution.
- Condition: IncludeCustomDomain
- Value: !Ref DomainName
- Export:
- Name: !Sub ${Environment}-${ApplicationName}-EcsCloudFrontCustomDnsName
-
- ECSCloudFrontDomainName:
- Description: The custom DNS name of the ECS service's CloudFront Distribution.
- Value: !GetAtt ECSCloudFrontDistribution.DomainName
- Export:
- Name: !Sub ${Environment}-${ApplicationName}-EcsCloudFrontDomainName
-
- TaskDefinitionArn:
- Description: The ARN of the Indexer Service ECS task definition.
- Value: !Ref 'TaskDefinition'
- Export:
- Name: !Sub ${Environment}-${ApplicationName}-EcsTaskDefinitionArn
-
- IndexerEC2RoleArn:
- Description: The ARN of the application's EC2 role.
- Value: !GetAtt EC2Role.Arn
- Export:
- Name: !Sub ${Environment}-${ApplicationName}-EC2RoleArn
diff --git a/provider/indexer-aws/CloudFormation/Automated/ecs-network.yml b/provider/indexer-aws/CloudFormation/Automated/ecs-network.yml
deleted file mode 100644
index 0fa408776e7b9a4040531b58afcafc4550e8f239..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/CloudFormation/Automated/ecs-network.yml
+++ /dev/null
@@ -1,150 +0,0 @@
-# Copyright © Amazon Web Services
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-AWSTemplateFormatVersion: 2010-09-09
-Description: >-
- CloudFormation template for creating the network resources used for the ECS cluster the application will
- be deployed into. This is separated from the rest of the ECS resources in order to avoid a circular dependency.
- Because there can be any number of cache stacks, but only one ECS stack per service, it makes sense to have
- the caches import ECS exports in order to permit access from ECS, rather than the other way around, since
- the number of cache clusters and their names can vary, and would require hardcoding them into the ECS template,
- whereas this way things stay generic and the ECS CloudFormation template does not need to be updated in order to
- add or remove ElastiCache clusters.
-
-Parameters:
- Environment:
- Description: An environment name that will be prefixed to resource names.
- Type: String
- AllowedValues:
- - dev
- - uat
- - prod
- ConstraintDescription: Can only be "dev/uat/prod"
- Default: dev
-
- Region:
- Description: The AWS region to deploy the resources to.
- Type: String
- Default: us-east-1
-
- ApplicationName:
- Description: >
- The name of the application, which will be used to generate the ECS cluster name.
- It will be prefixed with the environment name.
- Type: String
- MinLength: '1'
- MaxLength: '64'
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: os-indexer
-
- ECSPort:
- Description: The port that the ECS Service will listen on.
- Type: Number
- Default: 443
- MinValue: 1
- MaxValue: 65535
-
- DomainName:
- Description: >-
- The optional custom DNS name for the service's load balancer. If omitted, the site will only be accessible
- via the ECS service's Application Load Balancer DNS name. This value is used in the creation and signing of
- the service's SSL certificate. Leave blank for none.
- Type: String
- Default: ''
-
- AcmCertificateArn:
- Description: >-
- The Amazon Resource Name (ARN) of an existing AWS Certificate Manager (ACM) certificate.
- If omitted, a new SSL certified will be requested/generated (only if the custom domain name
- parameter is provided, otherwise the ECS service's ALB will not use SSL/HTTPS).
- Type: String
- AllowedPattern: "^(|arn:aws:acm:.*)$"
- Default: ''
-
-Conditions:
- IncludeCustomDomain: !Not [!Equals [ !Ref DomainName, '' ]]
- UseExistingACMSSLCertificate: !And
- - !Not [!Equals [ !Ref AcmCertificateArn, '' ]]
- - !Condition IncludeCustomDomain
- ShouldRequestNewSSLCertificate: !And
- - !Not [!Condition UseExistingACMSSLCertificate]
- - !Condition IncludeCustomDomain
- ShouldExportSSLCertificate: !Or
- - !Condition IncludeCustomDomain
- - !Condition UseExistingACMSSLCertificate
-
-Resources:
- # If an existing SSL certificate is not provided, but a custom domain is, request one
- LoadBalancerSSLCertificate:
- Type: 'AWS::CertificateManager::Certificate'
- Condition: ShouldRequestNewSSLCertificate
- Properties:
- DomainName: !Ref DomainName
- SubjectAlternativeNames:
- - !Join ['.', ['origin', !Ref DomainName]] #
-
- ECSSecurityGroup:
- Type: AWS::EC2::SecurityGroup
- Properties:
- GroupName: !Sub "${Environment}-${ApplicationName}-sg"
- GroupDescription: Indexer Service ECS Security Group
- VpcId:
- Fn::ImportValue:
- !Sub "${Environment}-OSDU-VPC"
-
- # Public access to the specified ECS Listening Port
- ECSSecurityGroupECSListenerInbound:
- Type: AWS::EC2::SecurityGroupIngress
- Properties:
- GroupId: !Ref 'ECSSecurityGroup'
- IpProtocol: tcp
- FromPort: !Ref ECSPort
- ToPort: !Ref ECSPort
- CidrIp: 0.0.0.0/0
-
- # SSH access for instances in our VPC's jump box subnet group
- # TODO: Update when the jump box is created as a part of the Util CFN, for now it is public
- ECSSecurityGroupSSHInbound:
- Type: AWS::EC2::SecurityGroupIngress
- Properties:
- GroupId: !Ref 'ECSSecurityGroup'
- IpProtocol: tcp
- FromPort: '22'
- ToPort: '22'
- CidrIp: 0.0.0.0/0
-
- # Open Application Load Balancer port range to self-access
- ECSSecurityGroupALBports:
- Type: AWS::EC2::SecurityGroupIngress
- Properties:
- GroupId: !Ref 'ECSSecurityGroup'
- IpProtocol: tcp
- FromPort: '31000'
- ToPort: '61000'
- SourceSecurityGroupId: !Ref 'ECSSecurityGroup'
-
-Outputs:
- EcsNetworkSecurityGroupId:
- Description: The ID of the Indexer Service ECS EC2 security group.
- Value: !Ref 'ECSSecurityGroup'
- Export:
- Name: !Sub ${Environment}-${ApplicationName}-EcsNetworkSecurityGroupId
-
- LoadBalancerSSLCertificateArn:
- Condition: ShouldExportSSLCertificate
- Description: The ARN of the SSL certificate to be used for both ECS and CloudFront (includes both DNS names).
- Value: !If [UseExistingACMSSLCertificate, !Ref AcmCertificateArn, !Ref 'LoadBalancerSSLCertificate']
- Export:
- Name: !Sub ${Environment}-${ApplicationName}-LoadBalancerSSLCertificateArn
diff --git a/provider/indexer-aws/CloudFormation/Automated/elasticsearch.yml b/provider/indexer-aws/CloudFormation/Automated/elasticsearch.yml
deleted file mode 100644
index 7a18783ae484ffcb6b248c98e131db4d28ee63a6..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/CloudFormation/Automated/elasticsearch.yml
+++ /dev/null
@@ -1,242 +0,0 @@
-# Copyright © Amazon Web Services
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-AWSTemplateFormatVersion: 2010-09-09
-Description: >-
- CloudFormation template for creating the resources used for the tenant info database for OSDU.
- It creates the DynamoDB table and the API Gateway endpoints.
-
-Parameters:
- Environment:
- Description: An environment name that will be prefixed to resource names.
- Type: String
- AllowedValues:
- - dev
- - uat
- - prod
- ConstraintDescription: Can only be "dev/uat/prod"
- Default: dev
-
- ApplicationName:
- Description: >
- The name of the application, which will be used to generate the ECS cluster name.
- It will be prefixed with the environment name.
- Type: String
- MinLength: '1'
- MaxLength: '64'
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: os-indexer
-
- SearchApplicationName:
- Description: >
- The name of the application, which will be used to generate the ECS cluster name.
- It will be prefixed with the environment name.
- Type: String
- MinLength: '1'
- MaxLength: '64'
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: os-search
-
- Region:
- Description: The AWS region to deploy the resources to.
- Type: String
- Default: us-east-1
-
- ElasticsearchDomainName:
- Description: The name of the Elasticsearch domain. Will be prefixed with the environment name.
- Type: String
- MinLength: '1'
- MaxLength: '64'
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: osdu-indexer
-
- ElasticsearchNodeInstanceType:
- Description: The instance type for the main Elasticsearch nodes.
- ConstraintDescription: Must be a valid instance type from the list of allowed values.
- Default: t2.medium.elasticsearch
- AllowedValues:
- - t2.small.elasticsearch
- - t2.medium.elasticsearch
- - m5.large.elasticsearch
- - m5.xlarge.elasticsearch
- - m5.2xlarge.elasticsearch
- - m5.4xlarge.elasticsearch
- - m5.12xlarge.elasticsearch
- - c5.large.elasticsearch
- - c5.xlarge.elasticsearch
- - c5.2xlarge.elasticsearch
- - c5.4xlarge.elasticsearch
- - c5.9xlarge.elasticsearch
- - c5.18xlarge.elasticsearch
- - r5.large.elasticsearch
- - r5.xlarge.elasticsearch
- - r5.2xlarge.elasticsearch
- - r5.4xlarge.elasticsearch
- - r5.12xlarge.elasticsearch
- - i3.large.elasticsearch
- - i3.xlarge.elasticsearch
- - i3.2xlarge.elasticsearch
- - i3.4xlarge.elasticsearch
- - i3.8xlarge.elasticsearch
- - i3.16xlarge.elasticsearch
- Type: String
-
- DedicatedMasterInstanceType:
- Description: >
- The instance type for the dedicated master nodes. These nodes perform cluster management
- tasks, but doesn't hold data or respond to data upload requests.
- ConstraintDescription: Must be a valid instance type from the list of allowed values.
- Default: t2.medium.elasticsearch
- AllowedValues:
- - t2.small.elasticsearch
- - t2.medium.elasticsearch
- - m5.large.elasticsearch
- - m5.xlarge.elasticsearch
- - m5.2xlarge.elasticsearch
- - m5.4xlarge.elasticsearch
- - m5.12xlarge.elasticsearch
- - c5.large.elasticsearch
- - c5.xlarge.elasticsearch
- - c5.2xlarge.elasticsearch
- - c5.4xlarge.elasticsearch
- - c5.9xlarge.elasticsearch
- - c5.18xlarge.elasticsearch
- - r5.large.elasticsearch
- - r5.xlarge.elasticsearch
- - r5.2xlarge.elasticsearch
- - r5.4xlarge.elasticsearch
- - r5.12xlarge.elasticsearch
- - i3.large.elasticsearch
- - i3.xlarge.elasticsearch
- - i3.2xlarge.elasticsearch
- - i3.4xlarge.elasticsearch
- - i3.8xlarge.elasticsearch
- - i3.16xlarge.elasticsearch
- Type: String
-
- NumberOfElasticsearchNodes:
- Description: An integer value specifying the number of Elasticsearch primary nodes in the cluster.
- Type: Number
- Default: 2
- MinValue: 1
- MaxValue: 40
-
- NumberOfDedicatedMasterNodes:
- Description: An integer value specifying the number of dedicated master nodes.
- Type: Number
- Default: 2
- MinValue: 2
- MaxValue: 5
-
- ZoneAwarenessEnabled:
- Description: >
- When Zone Awareness is enabled, Elasticsearch allocates the nodes and replica
- index shards that belong to a cluster across multiple AZs in the deployment region.
- Type: String
- AllowedValues:
- - true
- - false
- Default: false
-
- ElasticsearchVersion:
- Description: >
- The version of Elasticsearch to deploy on the cluster. Defaults to 6.8. Note
- that an update requires a full replacement of the Elasticsearch cluster.
- Type: String
- AllowedValues:
- - 1.5
- - 2.3
- - 5.1
- - 5.3
- - 5.5
- - 5.6
- - 6.0
- - 6.2
- - 6.3
- - 6.4
- - 6.5
- - 6.6
- - 6.8
- - 6.8
- - 7.1
- Default: 6.8
-
- EBSVolumeSize:
- Description: >
- The size of the EBS volume (per instance; total cluster size = EBS volume size x Instance count)
- Maximum size varies by instance type, from 35GiB for t2 instances, up to 12TiB for r5.12xlarge.
- Type: Number
- Default: 10
- MinValue: 10
- MaxValue: 12000
-
-Resources:
- ElasticsearchDomain:
- Type: AWS::Elasticsearch::Domain
- Properties:
- DomainName: !Sub ${Environment}-${ElasticsearchDomainName}
- ElasticsearchVersion: !Ref ElasticsearchVersion
- ElasticsearchClusterConfig:
- DedicatedMasterEnabled: "true"
- InstanceCount: !Ref NumberOfElasticsearchNodes
- ZoneAwarenessEnabled: !Ref ZoneAwarenessEnabled
- InstanceType: !Ref ElasticsearchNodeInstanceType
- DedicatedMasterType: !Ref DedicatedMasterInstanceType
- DedicatedMasterCount: !Ref NumberOfDedicatedMasterNodes
- EBSOptions:
- EBSEnabled: true
- VolumeSize: !Ref EBSVolumeSize
- VolumeType: "gp2"
- NodeToNodeEncryptionOptions:
- Enabled: false
- SnapshotOptions:
- AutomatedSnapshotStartHour: "0"
- AccessPolicies:
- Version: 2012-10-17
- Statement:
- - Effect: "Allow"
- Principal: "*"
- Resource: '*'
- Action: "*"
- AdvancedOptions:
- rest.action.multi.allow_explicit_index: "true"
- Tags:
- -
- Key: "Environment"
- Value: !Ref Environment
- VPCOptions:
- SubnetIds:
- - Fn::ImportValue:
- !Sub "${Environment}-OSDU-PrivateSubnet-AZ1"
- SecurityGroupIds:
- - Fn::ImportValue:
- !Sub "${Environment}-${ApplicationName}-EcsNetworkSecurityGroupId"
-
-Outputs:
- # Elasticsearch domain ARN
- ElasticsearchDomainArn:
- Description: The ARN of the Elasticsearch domain.
- Value: !GetAtt ElasticsearchDomain.DomainArn
- Export:
- Name: !Sub ${Environment}-${ElasticsearchDomainName}-ElasticsearchDomainArn
-
- # Elasticsearch domain endpoint
- ElasticsearchDomainEndpoint:
- Description: The endpoint URL of the Elasticsearch domain.
- Value: !GetAtt ElasticsearchDomain.DomainEndpoint
- Export:
- Name: !Sub ${Environment}-${ElasticsearchDomainName}-ElasticsearchDomainEndpoint
diff --git a/provider/indexer-aws/CloudFormation/Automated/iam-credentials.yml b/provider/indexer-aws/CloudFormation/Automated/iam-credentials.yml
deleted file mode 100644
index 3d5c9f144d52070bfebb81eacc4f32ba4b193f31..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/CloudFormation/Automated/iam-credentials.yml
+++ /dev/null
@@ -1,114 +0,0 @@
-# Copyright © Amazon Web Services
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-AWSTemplateFormatVersion: 2010-09-09
-Description: >-
- CloudFormation template for creating the resources used for application SDK access for OSDU services.
- It creates the IAM account, access keys, and optional key rotation.
-
-Parameters:
- Environment:
- Description: An environment name that will be prefixed to resource names.
- Type: String
- AllowedValues:
- - dev
- - uat
- - prod
- ConstraintDescription: Can only be "dev/uat/prod"
- Default: dev
-
- Region:
- Description: The AWS region to deploy the resources to.
- Type: String
- Default: us-east-1
-
- IndexerServiceIamUsername:
- Description: The username of the service user for the OS Indexer Service.
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Type: String
- Default: service-user-os-indexer
- MinLength: '1'
- MaxLength: '64'
-
- IndexerServiceIamKeyRotationSerial:
- Description: This integer value can only ever be incremented, and an increase in value results in a rotation of the user's access key.
- Type: Number
- Default: 1
-
-Resources:
- IndexerServiceIamUser:
- Type: AWS::IAM::User
- Properties:
- Policies:
- - PolicyName: !Sub ${Environment}-IndexerServiceUserPolicy
- PolicyDocument:
- Version: '2012-10-17'
- Statement:
- -
- Action:
- - 's3:*'
- - 'sns:*'
- - 'sqs:*'
- - 'dynamodb:*'
- - 'logs:*'
- - 'cloudwatch:*'
- - 'es:*'
- - 'cognito-identity:*'
- - 'cognito-idp:*'
- - 'sts:AssumeRole'
- - "iam:*"
- Effect: Allow
- Resource: '*'
- UserName: !Sub ${Environment}-${IndexerServiceIamUsername}
-
- IndexerServiceIamUserAccessKey:
- Type: AWS::IAM::AccessKey
- DependsOn: IndexerServiceIamUser
- Properties:
- Serial: !Ref IndexerServiceIamKeyRotationSerial # this value can only ever be incremented, and an increase in value results in a rotation of the user's access key
- Status: Active
- UserName: !Sub ${Environment}-${IndexerServiceIamUsername}
-
- IAMCredentialsSecret:
- Type: 'AWS::SecretsManager::Secret'
- Properties:
- Name: !Sub ${Environment}-IndexerServiceIamCredentials
- Description: The IAM service account credentials for the search service.
- SecretString:
- Fn::Sub:
- - '{"access_key":"${AccessKey}","secret_key":"${SecretKey}"}'
- - {AccessKey: !Ref IndexerServiceIamUserAccessKey, SecretKey: !GetAtt IndexerServiceIamUserAccessKey.SecretAccessKey}
- Tags:
- - Key: Environment
- Value: !Ref Environment
-
-Outputs:
- IndexerServiceIamUserAccessKeyId:
- Description: The access key ID for the service user for the Schema Repository.
- Value: !Ref IndexerServiceIamUserAccessKey
- Export:
- Name: !Sub ${Environment}-IndexerServiceIamUserAccessKeyId
-
- IndexerServiceIamUserSecretAccessKey:
- Description: The secret access key for the service user for the Schema Repository.
- Value: !GetAtt IndexerServiceIamUserAccessKey.SecretAccessKey
- Export:
- Name: !Sub ${Environment}-IndexerServiceIamUserSecretAccessKey
-
- IndexerServiceIamUserArn:
- Description: The ARN of the service IAM user account.
- Value: !GetAtt IndexerServiceIamUser.Arn
- Export:
- Name: !Sub ${Environment}-IndexerServiceIamUserArn
diff --git a/provider/indexer-aws/CloudFormation/Automated/sns-topic.yml b/provider/indexer-aws/CloudFormation/Automated/sns-topic.yml
deleted file mode 100644
index f90f9188963232b89978dde65f51597fb79f98e9..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/CloudFormation/Automated/sns-topic.yml
+++ /dev/null
@@ -1,108 +0,0 @@
-# Copyright © Amazon Web Services
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-AWSTemplateFormatVersion: 2010-09-09
-Description: >-
- CloudFormation template for creating the resources used for the sending messages to topic and queues to receive the
- messages for OSDU's indexer service. It creates the SNS Topic and the corresponding SQS Queues with their associated policies.
-
-Parameters:
- Environment:
- Description: an environment name that will be prefixed to resource names.
- Type: String
- AllowedValues:
- - dev
- - uat
- - prod
- ConstraintDescription: Can only be "dev/uat/prod"
- Default: dev
-
- Region:
- Description: The AWS region to deploy the resources to.
- Type: String
- Default: us-east-1
-
- SNSTopicName:
- Description: >-
- The name of the Simple Notification Service topic for the OS Indexer Service. Defaults to osdu-indexer-messages.
- Will be prefixed with the environment name.
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: osdu-indexer-messages
- Type: String
- MinLength: '1'
- MaxLength: '64'
-
- SQSQueueName:
- Description: >-
- The name of the Simple Queue Service queue for the OS Indexer Service. Defaults to osdu-indexer-queue.
- Will be prefixed with the environment name.
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: osdu-indexer-queue
- Type: String
- MinLength: '1'
- MaxLength: '64'
-
-Resources:
- OSDUIndexerSNSTopic:
- Type: 'AWS::SNS::Topic'
- Properties:
- DisplayName: !Sub ${Environment}-${SNSTopicName}
- TopicName: !Sub ${Environment}-${SNSTopicName}
- Subscription:
- - Endpoint:
- Fn::GetAtt:
- - OSDUIndexerSQSQueue
- - Arn
- Protocol: sqs
-
- OSDUIndexerSQSQueue:
- Type: AWS::SQS::Queue
- Properties:
- QueueName: !Sub ${Environment}-${SQSQueueName}
-
- OSDUQueuePolicy:
- Type: AWS::SQS::QueuePolicy
- Properties:
- PolicyDocument:
- Version: "2012-10-17"
- Id: OSDUQueuePolicy
- Statement:
- - Sid: Allow-SendMessage-To-Queues-From-SNS-Topic
- Effect: Allow
- Principal: "*"
- Action:
- - sqs:SendMessage
- - sqs:ReceiveMessage
- Resource: "*"
- Condition:
- ArnEquals:
- aws:SourceArn:
- Ref: OSDUIndexerSNSTopic
- Queues:
- - Ref: OSDUIndexerSQSQueue
-
-Outputs:
- OSDUIndexerSNSTopicTopicName:
- Value: !Sub ${Environment}-${SNSTopicName}
- Description: Topic Name of the Indexer Service Message Bus SNS Topic
- Export:
- Name: !Sub ${Environment}-OSDUIndexerSNSTopic
-
- OSDUIndexerSQSQueueName:
- Value: !Sub ${Environment}-${SQSQueueName}
- Description: Queue Name of Subscribed Indexer Service Message Bus SQS Queue
- Export:
- Name: !Sub ${Environment}-OSDUIndexerSQSQueue
diff --git a/provider/indexer-aws/CloudFormation/JarDeploy/CodePipeline-JarDeploy.yml b/provider/indexer-aws/CloudFormation/JarDeploy/CodePipeline-JarDeploy.yml
deleted file mode 100644
index f75d29cd5944fc25edfcf213bac567194319af4e..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/CloudFormation/JarDeploy/CodePipeline-JarDeploy.yml
+++ /dev/null
@@ -1,249 +0,0 @@
-# Copyright © Amazon Web Services
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-AWSTemplateFormatVersion: 2010-09-09
-
-Description: >
- This CloudFormation script creates the deployment pipeline for OSDU's indexer service. The CodePipeline
- should automatically trigger whenever commits are made on the tracked branch. The start and end
- of the CodePipeline should trigger a SNS alert to keep track of when the deployment has started
- and when it finishes.
-
-Parameters:
- Environment:
- Description: Environment Name. Defaults to 'dev'. Can only be dev/uat/prod.
- Type: String
- AllowedValues:
- - dev
- - uat
- - prod
- Default: dev
-
- DeploymentRegion:
- Description: The AWS region to deploy the application to. The default is us-east-1.
- Type: String
- Default: us-east-1
-
- SNSNotificationEmail:
- Description: The email address to send SNS notifications about the build to.
- Type: String
- Default: barclay.walsh@parivedasolutions.com
-
- CodeCommitRepositoryName:
- Description: The name of the Code Commit Repository that the CodePipeline source is connected to.
- Type: String
- Default: os-indexer
-
- JarServiceBase:
- Description: The name of the service base path for the JAR files (e.g. 'indexer').
- Type: String
- Default: indexer
-
- CodeCommitBranchName:
- Description: The name of the Code Commit branch that the CodePipeline source is connected to.
- Type: String
- Default: dev
-
-Resources:
- ArtifactStoreBucket:
- Type: AWS::S3::Bucket
- DeletionPolicy: Delete
- Properties:
- VersioningConfiguration:
- Status: Enabled
-
- ArtifactStoreBucketPolicy:
- Type: AWS::S3::BucketPolicy
- Properties:
- Bucket: !Ref ArtifactStoreBucket
- PolicyDocument:
- Statement:
- - Action:
- - s3:*
- Effect: Allow
- Resource:
- - !Sub arn:aws:s3:::${ArtifactStoreBucket}
- - !Sub arn:aws:s3:::${ArtifactStoreBucket}/*
- Principal:
- AWS:
- - !Sub arn:aws:iam::${AWS::AccountId}:root
- - !ImportValue
- 'Fn::Sub': '${Environment}-CodeBuildRoleArn'
- - !ImportValue
- 'Fn::Sub': '${Environment}-PipelineRoleArn'
- - !ImportValue
- 'Fn::Sub': '${Environment}-CFNRoleArn'
-
- CachingBucket:
- Type: AWS::S3::Bucket
- DeletionPolicy: Delete
- Properties:
- VersioningConfiguration:
- Status: Enabled
-
- CachingBucketPolicy:
- Type: AWS::S3::BucketPolicy
- Properties:
- Bucket: !Ref CachingBucket
- PolicyDocument:
- Statement:
- - Action:
- - s3:*
- Effect: Allow
- Resource:
- - !Sub arn:aws:s3:::${CachingBucket}
- - !Sub arn:aws:s3:::${CachingBucket}/*
- Principal:
- AWS:
- - !Sub arn:aws:iam::${AWS::AccountId}:root
- - !ImportValue
- 'Fn::Sub': '${Environment}-CodeBuildRoleArn'
- - !ImportValue
- 'Fn::Sub': '${Environment}-PipelineRoleArn'
- - !ImportValue
- 'Fn::Sub': '${Environment}-CFNRoleArn'
-
- SNSCodePipelineDeploymentFailed:
- Type: AWS::SNS::Topic
- Properties:
- Subscription:
- - Endpoint: !Ref SNSNotificationEmail
- Protocol: email
- TopicName: !Sub '${Environment}-OS-Indexer-Deployment-CodePipeline-JarDeploy-Failed'
-
- EventRuleCodePipelineFailed:
- Type: AWS::Events::Rule
- Properties:
- Description: Triggered whenever the CodePipeline deployment stage has failed.
- EventPattern:
- source:
- - "aws.codepipeline"
- detail-type:
- - "CodePipeline Stage Execution State Change"
- detail:
- state:
- - "FAILED"
- pipeline:
- - !Sub '${Environment}-OSDU-OS-Indexer-CodePipeline-JarDeploy'
-
- Name: !Sub ${Environment}-CodePipelineEventRule-${CodeCommitRepositoryName}-JarDeploy
- Targets:
- -
- Arn:
- !Ref SNSCodePipelineDeploymentFailed
- Id: "Deployment-CodePipeline-JarDeploy-Failed"
- InputTransformer:
- InputPathsMap:
- pipeline : "$.detail.pipeline"
- InputTemplate: '"The Pipeline <pipeline> has failed."'
-
- Pipeline:
- Type: AWS::CodePipeline::Pipeline
- Properties:
- ArtifactStore:
- Location: !Ref ArtifactStoreBucket
- Type: S3
- Name: !Sub '${Environment}-OSDU-OS-Indexer-CodePipeline-JarDeploy'
- RoleArn: !ImportValue
- 'Fn::Sub': '${Environment}-PipelineRoleArn'
- Stages:
- - Name: Source
- Actions:
- - Name: Source
- ActionTypeId:
- Category: Source
- Owner: AWS
- Provider: CodeCommit
- Version: '1'
- Configuration:
- BranchName: !Ref CodeCommitBranchName
- RepositoryName: !Ref CodeCommitRepositoryName
- OutputArtifacts:
- - Name: Source
- RunOrder: '1'
-
- - Name: CodeBuild
- Actions:
- - Name: Jar-CodeBuild
- ActionTypeId:
- Category: Build
- Owner: AWS
- Provider: CodeBuild
- Version: '1'
- InputArtifacts:
- - Name: Source
- OutputArtifacts:
- - Name: Jar-CodeBuild
- Configuration:
- ProjectName: !Ref JarCodeBuild
- RunOrder: '2'
-
- JarCodeBuild:
- Type: AWS::CodeBuild::Project
- Properties:
- Name: !Sub ${Environment}-jar-codebuild-${CodeCommitRepositoryName}
- Description: CodeBuild commands which run after the CloudFormation deployment.
- ServiceRole: !ImportValue
- 'Fn::Sub': '${Environment}-CodeBuildRoleArn'
- Artifacts:
- Type: S3
- Location: !Ref ArtifactStoreBucket
- Name: !Sub ${Environment}-jar-codebuild
- Environment:
- Type: LINUX_CONTAINER
- ComputeType: BUILD_GENERAL1_SMALL
- Image: aws/codebuild/standard:2.0
- EnvironmentVariables:
- - Name: ENVIRONMENT
- Type: PLAINTEXT
- Value: !Ref Environment
- - Name: AWS_ACCOUNT_ID
- Type: PLAINTEXT
- Value: !Ref AWS::AccountId
- - Name: AWS_REGION
- Type: PLAINTEXT
- Value: !Ref DeploymentRegion
- - Name: APPLICATION_NAME
- Type: PLAINTEXT
- Value: !Ref CodeCommitRepositoryName
- - Name: JAR_SERVICE_BASE
- Type: PLAINTEXT
- Value: !Ref JarServiceBase
- - Name: M2_REPO_S3_BUCKET
- Type: PLAINTEXT
- Value: !Sub "${Environment}-${AWS::AccountId}-persistent-maven-m2-bucket"
- - Name: JAR_DEPLOY_S3_BUCKET
- Type: PLAINTEXT
- Value: !Sub ${Environment}-${AWS::AccountId}-osdu-jar-deploy
- PrivilegedMode: true
- Source:
- BuildSpec: ./provider/indexer-aws/buildspec-jar-deploy.yml
- Location: !Sub https://git-codecommit.${AWS::Region}.amazonaws.com/v1/repos/${CodeCommitRepositoryName}
- Type: CODECOMMIT
- Cache:
- Type: S3
- Location: !Sub ${CachingBucket}/${Environment}
- TimeoutInMinutes: 15
- VpcConfig:
- SecurityGroupIds:
- - Fn::ImportValue:
- !Sub "${Environment}-OSDU-CodeBuildSecurityGroup"
- Subnets:
- - Fn::ImportValue:
- !Sub "${Environment}-OSDU-PrivateSubnet-AZ1"
- - Fn::ImportValue:
- !Sub "${Environment}-OSDU-PrivateSubnet-AZ2"
- VpcId:
- Fn::ImportValue:
- !Sub "${Environment}-OSDU-VPC"
diff --git a/provider/indexer-aws/CloudFormation/Manual/01-CreateCodePipeline.yml b/provider/indexer-aws/CloudFormation/Manual/01-CreateCodePipeline.yml
deleted file mode 100644
index 0109633f0071acd48c8520bf240dd962068ead11..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/CloudFormation/Manual/01-CreateCodePipeline.yml
+++ /dev/null
@@ -1,358 +0,0 @@
-# Copyright © Amazon Web Services
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-AWSTemplateFormatVersion: 2010-09-09
-Description: >
- This CloudFormation script creates the deployment pipeline for OSDU's indexer
- service. The CodePipeline should automatically trigger whenever commits are
- made on the tracked branch. The start and end of the CodePipeline should
- trigger a SNS alert to keep track of when the deployment has started and when
- it finishes.
-Parameters:
- Environment:
- Description: Environment Name. Defaults to 'dev'. Can only be dev/uat/prod.
- Type: String
- AllowedValues:
- - dev
- - uat
- - prod
- Default: dev
-
- DeploymentRegion:
- Description: The AWS region to deploy the application to. The default is us-east-1.
- Type: String
- Default: us-east-1
-
- SNSNotificationEmail:
- Description: The email address to send SNS notifications about the build to.
- Type: String
- Default: barclay.walsh@parivedasolutions.com
-
- CodeCommitRepositoryName:
- Description: >-
- The name of the Code Commit Repository that the CodePipeline source is
- connected to.
- Type: String
- Default: os-indexer
-
- CodeCommitBranchName:
- Description: >-
- The name of the Code Commit branch that the CodePipeline source is
- connected to.
- Type: String
- Default: dev
-
- MasterStackName:
- Description: The name of the master stack that is being deployed by the CodePipeline.
- Type: String
- Default: os-indexer-master-stack
-
- MasterTemplateName:
- Description: >-
- The name of the master template that is called when creating the master
- stack.
- Type: String
- Default: provider/indexer-aws/CloudFormation/Master/os-indexer-master.yml
-
- SchemaCacheName:
- Description: The name of the cache cluster for the schema cache. Needs to match the value in the environment params JSON. Will be prefixed with the environment name.
- Type: String
- MinLength: '1'
- MaxLength: '64'
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: indexerSchemaCache
-
- IndexCacheName:
- Description: The name of the cache cluster for the index cache. Needs to match the value in the environment params JSON. Will be prefixed with the environment name.
- Type: String
- MinLength: '1'
- MaxLength: '64'
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: indexerIndexCache
-
-Resources:
- ArtifactStoreBucket:
- Type: 'AWS::S3::Bucket'
- DeletionPolicy: Delete
- Properties:
- VersioningConfiguration:
- Status: Enabled
- ArtifactStoreBucketPolicy:
- Type: 'AWS::S3::BucketPolicy'
- Properties:
- Bucket: !Ref ArtifactStoreBucket
- PolicyDocument:
- Statement:
- - Action:
- - 's3:*'
- Effect: Allow
- Resource:
- - !Sub 'arn:aws:s3:::${ArtifactStoreBucket}'
- - !Sub 'arn:aws:s3:::${ArtifactStoreBucket}/*'
- Principal:
- AWS:
- - !Sub 'arn:aws:iam::${AWS::AccountId}:root'
- - !ImportValue
- 'Fn::Sub': '${Environment}-CodeBuildRoleArn'
- - !ImportValue
- 'Fn::Sub': '${Environment}-PipelineRoleArn'
- - !ImportValue
- 'Fn::Sub': '${Environment}-CFNRoleArn'
- CachingBucket:
- Type: AWS::S3::Bucket
- DeletionPolicy: Delete
- Properties:
- VersioningConfiguration:
- Status: Enabled
-
- CachingBucketPolicy:
- Type: AWS::S3::BucketPolicy
- Properties:
- Bucket: !Ref CachingBucket
- PolicyDocument:
- Statement:
- - Action:
- - s3:*
- Effect: Allow
- Resource:
- - !Sub arn:aws:s3:::${CachingBucket}
- - !Sub arn:aws:s3:::${CachingBucket}/*
- Principal:
- AWS:
- - !Sub arn:aws:iam::${AWS::AccountId}:root
- - !ImportValue
- 'Fn::Sub': '${Environment}-CodeBuildRoleArn'
- - !ImportValue
- 'Fn::Sub': '${Environment}-PipelineRoleArn'
- - !ImportValue
- 'Fn::Sub': '${Environment}-CFNRoleArn'
-
- SNSCodePipelineDeploymentFailed:
- Type: 'AWS::SNS::Topic'
- Properties:
- Subscription:
- - Endpoint: !Ref SNSNotificationEmail
- Protocol: email
- TopicName: !Sub '${Environment}-OS-Indexer-Deployment-CodePipeline-Failed'
- EventRuleCodePipelineFailed:
- Type: 'AWS::Events::Rule'
- Properties:
- Description: Triggered whenever the CodePipeline deployment stage has failed.
- EventPattern:
- source:
- - aws.codepipeline
- detail-type:
- - CodePipeline Stage Execution State Change
- detail:
- state:
- - FAILED
- pipeline:
- - !Sub '${Environment}-OSDU-OS-Indexer-CodePipeline'
- Name: !Sub '${Environment}-CodePipelineEventRule-${CodeCommitRepositoryName}'
- Targets:
- - Arn: !Ref SNSCodePipelineDeploymentFailed
- Id: Deployment-CodePipeline-Failed
- InputTransformer:
- InputPathsMap:
- pipeline: $.detail.pipeline
- InputTemplate: '"The Pipeline <pipeline> has failed."'
- Pipeline:
- Type: 'AWS::CodePipeline::Pipeline'
- Properties:
- ArtifactStore:
- Location: !Ref ArtifactStoreBucket
- Type: S3
- Name: !Sub '${Environment}-OSDU-OS-Indexer-CodePipeline'
- RoleArn: !ImportValue
- 'Fn::Sub': '${Environment}-PipelineRoleArn'
- Stages:
- - Name: Source
- Actions:
- - Name: Source
- ActionTypeId:
- Category: Source
- Owner: AWS
- Provider: CodeCommit
- Version: '1'
- Configuration:
- BranchName: !Ref CodeCommitBranchName
- RepositoryName: !Ref CodeCommitRepositoryName
- OutputArtifacts:
- - Name: Source
- RunOrder: '1'
- - Name: Pre-Deployment-CodeBuild
- Actions:
- - Name: Pre-Deployment-CodeBuild
- ActionTypeId:
- Category: Build
- Owner: AWS
- Provider: CodeBuild
- Version: '1'
- InputArtifacts:
- - Name: Source
- OutputArtifacts:
- - Name: Pre-Deployment-CodeBuild
- Configuration:
- ProjectName: !Ref PreDeploymentCodeBuild
- RunOrder: '2'
- - Name: Deployment
- Actions:
- - Name: CloudFormation-Deployment-Master
- ActionTypeId:
- Category: Deploy
- Owner: AWS
- Provider: CloudFormation
- Version: '1'
- InputArtifacts:
- - Name: Source
- Configuration:
- ActionMode: CREATE_UPDATE
- Capabilities: CAPABILITY_NAMED_IAM
- RoleArn: !ImportValue
- 'Fn::Sub': '${Environment}-CFNRoleArn'
- StackName: !Sub '${Environment}-${MasterStackName}'
- TemplatePath: !Sub 'Source::${MasterTemplateName}'
- TemplateConfiguration: !Sub >-
- Source::provider/indexer-aws/CloudFormation/Params/${Environment}.template_configuration.json
- RunOrder: '3'
-
- - Name: Post-Deployment-CodeBuild
- Actions:
- - Name: Post-Deployment-CodeBuild
- ActionTypeId:
- Category: Build
- Owner: AWS
- Provider: CodeBuild
- Version: '1'
- InputArtifacts:
- - Name: Source
- OutputArtifacts:
- - Name: Post-Deployment-CodeBuild
- Configuration:
- ProjectName: !Ref PostDeploymentCodeBuild
- RunOrder: '4'
- PreDeploymentCodeBuild:
- Type: 'AWS::CodeBuild::Project'
- Properties:
- Name: !Sub '${Environment}-pre-deployment-codebuild-${CodeCommitRepositoryName}'
- Description: CodeBuild commands which run prior to the CloudFormation deployment.
- ServiceRole: !ImportValue
- 'Fn::Sub': '${Environment}-CodeBuildRoleArn'
- Artifacts:
- Type: S3
- Location: !Ref ArtifactStoreBucket
- Name: !Sub '${Environment}-pre-deployment-codebuild'
- Environment:
- Type: LINUX_CONTAINER
- ComputeType: BUILD_GENERAL1_SMALL
- Image: aws/codebuild/standard:2.0
- EnvironmentVariables:
- - Name: ENVIRONMENT
- Type: PLAINTEXT
- Value: !Ref Environment
- - Name: AWS_ACCOUNT_ID
- Type: PLAINTEXT
- Value: !Ref 'AWS::AccountId'
- - Name: AWS_REGION
- Type: PLAINTEXT
- Value: !Ref DeploymentRegion
- - Name: CFN_S3_BUCKET
- Value: !ImportValue
- 'Fn::Sub': '${Environment}-S3BucketCloudFormation'
- - Name: APPLICATION_NAME
- Type: PLAINTEXT
- Value: !Ref CodeCommitRepositoryName
- PrivilegedMode: false
- Source:
- BuildSpec: ./provider/indexer-aws/buildspec-pre-deploy.yml
- Location: !Sub >-
- https://git-codecommit.${AWS::Region}.amazonaws.com/v1/repos/${CodeCommitRepositoryName}
- Type: CODECOMMIT
- TimeoutInMinutes: 15
-
- PostDeploymentCodeBuild:
- Type: AWS::CodeBuild::Project
- Properties:
- Name: !Sub ${Environment}-post-deployment-codebuild-${CodeCommitRepositoryName}
- Description: CodeBuild commands which run after the CloudFormation deployment.
- ServiceRole: !ImportValue
- 'Fn::Sub': '${Environment}-CodeBuildRoleArn'
- Artifacts:
- Type: S3
- Location: !Ref ArtifactStoreBucket
- Name: !Sub ${Environment}-post-deployment-codebuild
- Environment:
- Type: LINUX_CONTAINER
- ComputeType: BUILD_GENERAL1_SMALL
- Image: aws/codebuild/standard:2.0
- EnvironmentVariables:
- - Name: ENVIRONMENT
- Type: PLAINTEXT
- Value: !Ref Environment
- - Name: AWS_ACCOUNT_ID
- Type: PLAINTEXT
- Value: !Ref AWS::AccountId
- - Name: AWS_REGION
- Type: PLAINTEXT
- Value: !Ref DeploymentRegion
- - Name: VSTS_FEED_USER
- Type: PLAINTEXT
- Value: '{{resolve:secretsmanager:dev-VSTSFeedToken:SecretString:vsts_feed_user}}'
- - Name: VSTS_FEED_TOKEN
- Type: PLAINTEXT
- Value: '{{resolve:secretsmanager:dev-VSTSFeedToken:SecretString:vsts_feed_token}}'
- - Name: IMAGE_TAG
- Type: PLAINTEXT
- Value: latest
- - Name: IMAGE_REPO_NAME
- Type: PLAINTEXT
- Value: !Sub ${Environment}-${CodeCommitRepositoryName}-repository
- - Name: QUEUE_IMAGE_REPO_NAME
- Type: PLAINTEXT
- Value: !Sub ${Environment}-${CodeCommitRepositoryName}-queue-repository
- - Name: SCHEMA_CACHE_NAME
- Type: PLAINTEXT
- Value: !Ref SchemaCacheName
- - Name: INDEX_CACHE_NAME
- Type: PLAINTEXT
- Value: !Ref IndexCacheName
- - Name: APPLICATION_NAME
- Type: PLAINTEXT
- Value: !Ref CodeCommitRepositoryName
- - Name: M2_REPO_S3_BUCKET
- Type: PLAINTEXT
- Value: !Sub "${Environment}-${AWS::AccountId}-persistent-maven-m2-bucket"
- PrivilegedMode: true
- Source:
- BuildSpec: ./provider/indexer-aws/buildspec-post-deploy.yml
- Location: !Sub https://git-codecommit.${AWS::Region}.amazonaws.com/v1/repos/${CodeCommitRepositoryName}
- Type: CODECOMMIT
- Cache:
- Type: S3
- Location: !Sub ${CachingBucket}/${Environment}
- TimeoutInMinutes: 15
- VpcConfig:
- SecurityGroupIds:
- - Fn::ImportValue:
- !Sub "${Environment}-OSDU-CodeBuildSecurityGroup"
- Subnets:
- - Fn::ImportValue:
- !Sub "${Environment}-OSDU-PrivateSubnet-AZ1"
- - Fn::ImportValue:
- !Sub "${Environment}-OSDU-PrivateSubnet-AZ2"
- VpcId:
- Fn::ImportValue:
- !Sub "${Environment}-OSDU-VPC"
\ No newline at end of file
diff --git a/provider/indexer-aws/CloudFormation/Master/os-indexer-master.yml b/provider/indexer-aws/CloudFormation/Master/os-indexer-master.yml
deleted file mode 100644
index c4a49e0b6ab7580627d96052056f23df5e745680..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/CloudFormation/Master/os-indexer-master.yml
+++ /dev/null
@@ -1,625 +0,0 @@
-# Copyright © Amazon Web Services
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-AWSTemplateFormatVersion: 2010-09-09
-Description: Creates all AWS resources used by OSDU's Indexer Service. Requires having previously setup the CodeCommit repository, as well as the CodePipeline (manual template).
-Parameters:
-
- VersionNumber:
- Description: Version Number for the pom to deploy the jar and Docker Image deployment in the Dockerfile
- Type: String
- Default: '0.0.1'
-
- ServiceName:
- Description: >-
- Service name for jar deployment in the Dockerfile
- Type: String
- Default: 'indexer'
-
- Environment:
- Description: The name of the environment.
- Type: String
- AllowedValues:
- - dev
- - uat
- - prod
- ConstraintDescription: Environment can only be "dev/uat/prod".
- Default: dev
-
- DeploymentRegion:
- Description: The AWS region to deploy the resources to.
- Type: String
- Default: us-east-1
-
- ApplicationName:
- Description: >
- The name of the indexer application, should be equal to the repository name.
- Type: String
- MinLength: '1'
- MaxLength: '64'
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: os-indexer
-
- SearchApplicationName:
- Description: >
- The name of the Search Service application (ex: os-search). Should be the same as the Search Service repo name.
- Type: String
- MinLength: '1'
- MaxLength: '64'
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: os-search
-
- KeyName:
- Description: >
- Name of an existing EC2 KeyPair to enable SSH access to the ECS instances. Note that key pairs cannot
- be created through CloudFormation, but instead must be uploaded through the AWS Console.
- Type: AWS::EC2::KeyPair::KeyName
- Default: ecs_indexer_key
-
- DesiredCapacity:
- Description: The default number of instances to launch in the ECS cluster.
- Type: Number
- Default: '1'
-
- MinSize:
- Description: Maximum number of instances that can be launched in the ECS cluster.
- Type: Number
- Default: '0'
-
- MaxSize:
- Description: Maximum number of instances that can be launched in the ECS cluster.
- Type: Number
- Default: '1'
-
- InstanceType:
- Description: EC2 instance type
- Type: String
- Default: t3.large
- AllowedValues:
- - m5.large
- - m5.xlarge
- - m5.2xlarge
- - m5.4xlarge
- - m5.12xlarge
- - m5.16xlarge
- - m5.24xlarge
- - m4.large
- - m4.xlarge
- - m4.2xlarge
- - m4.4xlarge
- - m4.10xlarge
- - m4.16xlarge
- - t3.nano
- - t3.micro
- - t3.small
- - t3.medium
- - t3.large
- - t3.xlarge
- - t3.2xlarge
- - c5.large
- - c5.xlarge
- - c5.2xlarge
- - c5.4xlarge
- - c5.12xlarge
- - c5.16xlarge
- - c5.24xlarge
- - r5.large
- - r5.xlarge
- - r5.2xlarge
- - r5.4xlarge
- - r5.12xlarge
- - r5.24xlarge
- - r4.large
- - r4.xlarge
- - r4.2xlarge
- - r4.4xlarge
- - r4.8xlarge
- - r4.16xlarge
- - i3.large
- - i3.xlarge
- - i3.2xlarge
- - i3.4xlarge
- - i3.10xlarge
- - i3.16xlarge
- - x1e.xlarge
- - x1e.2xlarge
- - x1e.4xlarge
- - x1e.8xlarge
- - x1e.16xlarge
- - x1e.32xlarge
- ConstraintDescription: Please choose a valid EC2 instance type for the ECS container instances.
-
- IndexerServiceIamUsername:
- Description: The username of the service user for the OS Indexer Service.
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Type: String
- Default: service-user-os-indexer
- MinLength: '1'
- MaxLength: '64'
-
- IndexerServiceIamKeyRotationSerial:
- Description: This integer value can only ever be incremented, and an increase in value results in a rotation of the user's access key.
- Type: Number
- Default: 1
-
- SNSTopicName:
- Description: >-
- The name of the Simple Notification Service topic for the OS Indexer Service. Defaults to osdu-indexer-messages.
- Will be prefixed with the environment name.
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: osdu-indexer-messages
- Type: String
- MinLength: '1'
- MaxLength: '64'
-
- SQSQueueName:
- Description: >-
- The name of the Simple Queue Service queue for the OS Indexer Service. Defaults to osdu-indexer-queue.
- Will be prefixed with the environment name.
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: osdu-indexer-queue
- Type: String
- MinLength: '1'
- MaxLength: '64'
-
- IndexCacheName:
- Description: The name of the cache cluster for the legal tag cache. Will be prefixed with the environment name.
- Type: String
- MinLength: '1'
- MaxLength: '64'
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: indexerIndexCache
-
- IndexCacheEngine:
- Description: Which caching platform to use for the legal tag cache. Can be set to 'redis' or 'memcached'.
- Type: String
- AllowedValues:
- - redis
- - memcached
- ConstraintDescription: Can only be "redis" or "memcached"
- Default: redis
-
- IndexCacheNodeInstanceType:
- Description: The instance type for redis cache nodes for the legal tag cache.
- ConstraintDescription: Must be a valid instance type from the list of allowed values.
- Default: cache.t2.micro
- AllowedValues:
- - cache.m5.large
- - cache.m5.xlarge
- - cache.m5.2xlarge
- - cache.m5.4xlarge
- - cache.m5.12xlarge
- - cache.m5.24xlarge
- - cache.m4.large
- - cache.m4.xlarge
- - cache.m4.2xlarge
- - cache.m4.4xlarge
- - cache.m4.10xlarge
- - cache.t2.micro
- - cache.t2.small
- - cache.t2.medium
- - cache.c1.xlarge
- - cache.r5.large
- - cache.r5.xlarge
- - cache.r5.2xlarge
- - cache.r5.4xlarge
- - cache.r5.12xlarge
- - cache.r5.24xlarge
- - cache.r4.large
- - cache.r4.xlarge
- - cache.r4.2xlarge
- - cache.r4.4xlarge
- - cache.r4.8xlarge
- - cache.r4.16xlarge
- Type: String
-
- IndexCacheNumberOfCacheNodes:
- Description: An integer value specifying the number of node in the redis cache for the legal tag cache.
- Type: Number
- Default: 1
- MinValue: 1
- MaxValue: 128
-
- SchemaCacheName:
- Description: The name of the cache cluster for the schema cache. Will be prefixed with the environment name.
- Type: String
- MinLength: '1'
- MaxLength: '64'
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: indexerSchemaCache
-
- SchemaCacheEngine:
- Description: Which caching platform to use for the schema cache. Can be set to 'redis' or 'memcached'.
- Type: String
- AllowedValues:
- - redis
- - memcached
- ConstraintDescription: Can only be "redis" or "memcached"
- Default: redis
-
- SchemaCacheNodeInstanceType:
- Description: The instance type for redis cache nodes for the schema cache.
- ConstraintDescription: Must be a valid instance type from the list of allowed values.
- Default: cache.t2.micro
- AllowedValues:
- - cache.m5.large
- - cache.m5.xlarge
- - cache.m5.2xlarge
- - cache.m5.4xlarge
- - cache.m5.12xlarge
- - cache.m5.24xlarge
- - cache.m4.large
- - cache.m4.xlarge
- - cache.m4.2xlarge
- - cache.m4.4xlarge
- - cache.m4.10xlarge
- - cache.t2.micro
- - cache.t2.small
- - cache.t2.medium
- - cache.c1.xlarge
- - cache.r5.large
- - cache.r5.xlarge
- - cache.r5.2xlarge
- - cache.r5.4xlarge
- - cache.r5.12xlarge
- - cache.r5.24xlarge
- - cache.r4.large
- - cache.r4.xlarge
- - cache.r4.2xlarge
- - cache.r4.4xlarge
- - cache.r4.8xlarge
- - cache.r4.16xlarge
- Type: String
-
- SchemaCacheNumberOfCacheNodes:
- Description: An integer value specifying the number of node in the redis cache for the schema cache.
- Type: Number
- Default: 1
- MinValue: 1
- MaxValue: 128
-
- ElasticsearchDomainName:
- Description: The name of the Elasticsearch domain. Will be prefixed with the environment name.
- Type: String
- MinLength: '1'
- MaxLength: '64'
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: osdu-indexer
-
- ElasticsearchNodeInstanceType:
- Description: The instance type for the main Elasticsearch nodes.
- ConstraintDescription: Must be a valid instance type from the list of allowed values.
- Default: t2.medium.elasticsearch
- AllowedValues:
- - t2.small.elasticsearch
- - t2.medium.elasticsearch
- - m5.large.elasticsearch
- - m5.xlarge.elasticsearch
- - m5.2xlarge.elasticsearch
- - m5.4xlarge.elasticsearch
- - m5.12xlarge.elasticsearch
- - c5.large.elasticsearch
- - c5.xlarge.elasticsearch
- - c5.2xlarge.elasticsearch
- - c5.4xlarge.elasticsearch
- - c5.9xlarge.elasticsearch
- - c5.18xlarge.elasticsearch
- - r5.large.elasticsearch
- - r5.xlarge.elasticsearch
- - r5.2xlarge.elasticsearch
- - r5.4xlarge.elasticsearch
- - r5.12xlarge.elasticsearch
- - i3.large.elasticsearch
- - i3.xlarge.elasticsearch
- - i3.2xlarge.elasticsearch
- - i3.4xlarge.elasticsearch
- - i3.8xlarge.elasticsearch
- - i3.16xlarge.elasticsearch
- Type: String
-
- DedicatedMasterInstanceType:
- Description: >
- The instance type for the dedicated master nodes. These nodes perform cluster management
- tasks, but doesn't hold data or respond to data upload requests.
- ConstraintDescription: Must be a valid instance type from the list of allowed values.
- Default: t2.medium.elasticsearch
- AllowedValues:
- - t2.small.elasticsearch
- - t2.medium.elasticsearch
- - m5.large.elasticsearch
- - m5.xlarge.elasticsearch
- - m5.2xlarge.elasticsearch
- - m5.4xlarge.elasticsearch
- - m5.12xlarge.elasticsearch
- - c5.large.elasticsearch
- - c5.xlarge.elasticsearch
- - c5.2xlarge.elasticsearch
- - c5.4xlarge.elasticsearch
- - c5.9xlarge.elasticsearch
- - c5.18xlarge.elasticsearch
- - r5.large.elasticsearch
- - r5.xlarge.elasticsearch
- - r5.2xlarge.elasticsearch
- - r5.4xlarge.elasticsearch
- - r5.12xlarge.elasticsearch
- - i3.large.elasticsearch
- - i3.xlarge.elasticsearch
- - i3.2xlarge.elasticsearch
- - i3.4xlarge.elasticsearch
- - i3.8xlarge.elasticsearch
- - i3.16xlarge.elasticsearch
- Type: String
-
- NumberOfElasticsearchNodes:
- Description: An integer value specifying the number of Elasticsearch primary nodes in the cluster.
- Type: Number
- Default: 1
- MinValue: 1
- MaxValue: 40
-
- NumberOfDedicatedMasterNodes:
- Description: An integer value specifying the number of dedicated master nodes.
- Type: Number
- Default: 2
- MinValue: 2
- MaxValue: 5
-
- ZoneAwarenessEnabled:
- Description: >
- When Zone Awareness is enabled, Elasticsearch allocates the nodes and replica
- index shards that belong to a cluster across multiple AZs in the deployment region.
- Type: String
- AllowedValues:
- - true
- - false
- Default: false
-
- ElasticsearchVersion:
- Description: >
- The version of Elasticsearch to deploy on the cluster. Defaults to 6.8. Note
- that an update requires a full replacement of the Elasticsearch cluster.
- Type: String
- AllowedValues:
- - 1.5
- - 2.3
- - 5.1
- - 5.3
- - 5.5
- - 5.6
- - 6.0
- - 6.2
- - 6.3
- - 6.4
- - 6.5
- - 6.6
- - 6.8
- - 6.8
- - 7.1
- Default: 6.8
-
- EBSVolumeSize:
- Description: >
- The size of the EBS volume, in GiB, (per instance; total cluster size =
- EBS volume size x Instance count). Maximum size varies by instance type, from 35GiB
- for t2 instances, up to 12TiB for r5.12xlarge.
- Type: Number
- Default: 10
- MinValue: 10
- MaxValue: 12000
-
- ECSPort:
- Description: The port that the ECS Service will listen on.
- Type: Number
- Default: 80
- MinValue: 1
- MaxValue: 65535
-
- ECSCPUAllocation:
- Description: The amount of CPU resources to allocate to each ECS task/container. Scale - 1024 = 1 vCPU core.
- Type: Number
- Default: 1024
- MinValue: 10
- MaxValue: 65535
-
- ECSMemoryAllocation:
- Description: The amount of memory (RAM) to allocate to each ECS task/container. Scale - 1 = 1MB of memory.
- Type: Number
- Default: 2048
- MinValue: 256
- MaxValue: 131072
-
- DomainName:
- Description: >-
- The optional custom DNS name for the ECS service's load balancer. If omitted, the site will only be accessible
- via the ECS service's Application Load Balancer DNS name. This value is used in the creation and signing of
- the service's SSL certificate. Leave blank is not using a custom domain for this deployment.
- Type: String
- Default: ''
-
- HostedZoneName:
- Description: >-
- The name of the hosted zone (ex: for indexer.osdu.slb.com, this would likely be osdu.slb.com).
- Leave blank is not using a custom domain for this deployment.
- Type: String
- Default: ''
-
- AcmCertificateArn:
- Description: >-
- The Amazon Resource Name (ARN) of an existing AWS Certificate Manager (ACM) certificate.
- If omitted, a new SSL certified will be requested/generated (only if the custom domain name
- parameter is provided, otherwise the ECS service's ALB will not use SSL/HTTPS).
- Type: String
- AllowedPattern: "^(|arn:aws:acm:.*)$"
- Default: ''
-
-Resources:
-
- #### Shared Resources ################################################################
-
- IAMCredentialsStack:
- Type: 'AWS::CloudFormation::Stack'
- Properties:
- TemplateURL: !Sub
- - https://s3.amazonaws.com/${CloudFormationS3Bucket}/${ApplicationName}/Automated/${CFNTemplateFilename}
- - CloudFormationS3Bucket: !ImportValue
- 'Fn::Sub': '${Environment}-S3BucketCloudFormation'
- CFNTemplateFilename: iam-credentials.yml
- Parameters:
- Environment: !Ref Environment
- Region: !Ref DeploymentRegion
- IndexerServiceIamUsername: !Ref IndexerServiceIamUsername
- IndexerServiceIamKeyRotationSerial: !Ref IndexerServiceIamKeyRotationSerial
-
- MessageBusSNSStack:
- Type: 'AWS::CloudFormation::Stack'
- Properties:
- TemplateURL: !Sub
- - https://s3.amazonaws.com/${CloudFormationS3Bucket}/${ApplicationName}/Automated/${CFNTemplateFilename}
- - CloudFormationS3Bucket: !ImportValue
- 'Fn::Sub': '${Environment}-S3BucketCloudFormation'
- CFNTemplateFilename: sns-topic.yml
- Parameters:
- Environment: !Ref Environment
- Region: !Ref DeploymentRegion
- SNSTopicName: !Ref SNSTopicName
- SQSQueueName: !Ref SQSQueueName
-
- #### ECS Resources ###################################################################
-
- ECSNetworkStack:
- Type: 'AWS::CloudFormation::Stack'
- DependsOn: IAMCredentialsStack
- Properties:
- TemplateURL: !Sub
- - https://s3.amazonaws.com/${CloudFormationS3Bucket}/${ApplicationName}/Automated/${CFNTemplateFilename}
- - CloudFormationS3Bucket: !ImportValue
- 'Fn::Sub': '${Environment}-S3BucketCloudFormation'
- CFNTemplateFilename: ecs-network.yml
- Parameters:
- Environment: !Ref Environment
- Region: !Ref DeploymentRegion
- ApplicationName: !Ref ApplicationName
- ECSPort: !Ref ECSPort
- DomainName: !Ref DomainName
- AcmCertificateArn: !Ref AcmCertificateArn
-
- ECSClusterStack:
- Type: 'AWS::CloudFormation::Stack'
- DependsOn: [SchemaCacheStack, IndexCacheStack]
- Properties:
- TemplateURL: !Sub
- - https://s3.amazonaws.com/${CloudFormationS3Bucket}/${ApplicationName}/Automated/${CFNTemplateFilename}
- - CloudFormationS3Bucket: !ImportValue
- 'Fn::Sub': '${Environment}-S3BucketCloudFormation'
- CFNTemplateFilename: ecs-cluster.yml
- Parameters:
- Environment: !Ref Environment
- Region: !Ref DeploymentRegion
- ApplicationName: !Ref ApplicationName
- KeyName: !Ref KeyName
- DesiredCapacity: !Ref DesiredCapacity
- MaxSize: !Ref MaxSize
- InstanceType: !Ref InstanceType
- SchemaCacheName: !Ref SchemaCacheName
- IndexCacheName: !Ref IndexCacheName
- ECSPort: !Ref ECSPort
- SNSTopicName: !Ref SNSTopicName
- ECSMemoryAllocation: !Ref ECSMemoryAllocation
- DomainName: !Ref DomainName
- HostedZoneName: !Ref HostedZoneName
- ElasticsearchDomainName: !Ref ElasticsearchDomainName
-
- #### Caching Resources ###############################################################
-
- IndexCacheStack:
- Type: 'AWS::CloudFormation::Stack'
- DependsOn: ECSNetworkStack
- Properties:
- TemplateURL: !Sub
- - https://s3.amazonaws.com/${CloudFormationS3Bucket}/${ApplicationName}/Automated/${CFNTemplateFilename}
- - CloudFormationS3Bucket: !ImportValue
- 'Fn::Sub': '${Environment}-S3BucketCloudFormation'
- CFNTemplateFilename: cache.yml
- Parameters:
- Environment: !Ref Environment
- Region: !Ref DeploymentRegion
- ApplicationName: !Ref ApplicationName
- CacheName: !Ref IndexCacheName
- CacheEngine: !Ref IndexCacheEngine
- NodeInstanceType: !Ref IndexCacheNodeInstanceType
- NumberOfCacheNodes: !Ref IndexCacheNumberOfCacheNodes
-
- SchemaCacheStack:
- Type: 'AWS::CloudFormation::Stack'
- DependsOn: ECSNetworkStack
- Properties:
- TemplateURL: !Sub
- - https://s3.amazonaws.com/${CloudFormationS3Bucket}/${ApplicationName}/Automated/${CFNTemplateFilename}
- - CloudFormationS3Bucket: !ImportValue
- 'Fn::Sub': '${Environment}-S3BucketCloudFormation'
- CFNTemplateFilename: cache.yml
- Parameters:
- Environment: !Ref Environment
- Region: !Ref DeploymentRegion
- ApplicationName: !Ref ApplicationName
- CacheName: !Ref SchemaCacheName
- CacheEngine: !Ref SchemaCacheEngine
- NodeInstanceType: !Ref SchemaCacheNodeInstanceType
- NumberOfCacheNodes: !Ref SchemaCacheNumberOfCacheNodes
-
- #### Elasticsearch Resources #########################################################
-
- ElasticsearchStack:
- Type: 'AWS::CloudFormation::Stack'
- DependsOn: [IAMCredentialsStack, ECSNetworkStack]
- Properties:
- TemplateURL: !Sub
- - https://s3.amazonaws.com/${CloudFormationS3Bucket}/${ApplicationName}/Automated/${CFNTemplateFilename}
- - CloudFormationS3Bucket: !ImportValue
- 'Fn::Sub': '${Environment}-S3BucketCloudFormation'
- CFNTemplateFilename: elasticsearch.yml
- Parameters:
- Environment: !Ref Environment
- Region: !Ref DeploymentRegion
- ElasticsearchDomainName: !Ref ElasticsearchDomainName
- ElasticsearchNodeInstanceType: !Ref ElasticsearchNodeInstanceType
- DedicatedMasterInstanceType: !Ref DedicatedMasterInstanceType
- NumberOfElasticsearchNodes: !Ref NumberOfElasticsearchNodes
- NumberOfDedicatedMasterNodes: !Ref NumberOfDedicatedMasterNodes
- ZoneAwarenessEnabled: !Ref ZoneAwarenessEnabled
- ElasticsearchVersion: !Ref ElasticsearchVersion
- EBSVolumeSize: !Ref EBSVolumeSize
- ApplicationName: !Ref ApplicationName
- SearchApplicationName: !Ref SearchApplicationName
-
-Outputs:
- JarVersionNumber:
- Description: The service name associated with the JAR package for the Dockerfile.
- Value: !Ref 'VersionNumber'
- Export:
- Name: !Sub ${Environment}-${ApplicationName}-JarVersionNumber
-
- JarServiceName:
- Description: The service name associated with the JAR package for the Dockerfile.
- Value: !Ref 'ServiceName'
- Export:
- Name: !Sub ${Environment}-${ApplicationName}-JarServiceName
diff --git a/provider/indexer-aws/CloudFormation/Params/dev.template_configuration.json b/provider/indexer-aws/CloudFormation/Params/dev.template_configuration.json
deleted file mode 100644
index f33fd69a62180cc5c08e9962a46aae62705cabf5..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/CloudFormation/Params/dev.template_configuration.json
+++ /dev/null
@@ -1,44 +0,0 @@
-{
- "Parameters" : {
- "Environment" : "dev",
- "DeploymentRegion" : "us-east-1",
- "ApplicationName" : "os-indexer",
- "SearchApplicationName" : "os-search",
- "KeyName": "indexer-ecs-keypair",
- "DesiredCapacity": "2",
- "MinSize": "0",
- "MaxSize": "3",
- "InstanceType": "t3.large",
- "IndexerServiceIamUsername": "service-user-os-indexer",
- "IndexerServiceIamKeyRotationSerial": "1",
- "SNSTopicName": "osdu-indexer-messages",
- "SQSQueueName": "osdu-indexer-queue",
- "IndexCacheName": "indexerIndexCache",
- "IndexCacheEngine": "redis",
- "IndexCacheNodeInstanceType": "cache.t2.micro",
- "IndexCacheNumberOfCacheNodes": "1",
- "SchemaCacheName": "indexerSchemaCache",
- "SchemaCacheEngine": "redis",
- "SchemaCacheNodeInstanceType": "cache.t2.micro",
- "SchemaCacheNumberOfCacheNodes": "1",
- "ElasticsearchDomainName": "osdu-indexer",
- "ElasticsearchNodeInstanceType": "t2.medium.elasticsearch",
- "DedicatedMasterInstanceType": "t2.medium.elasticsearch",
- "NumberOfElasticsearchNodes": "4",
- "NumberOfDedicatedMasterNodes": "3",
- "ZoneAwarenessEnabled": "false",
- "ElasticsearchVersion": "6.8",
- "EBSVolumeSize": "10",
- "ECSPort": "443",
- "ECSCPUAllocation": "1024",
- "ECSMemoryAllocation": "3072",
- "DomainName": "",
- "HostedZoneName": "",
- "AcmCertificateArn": "",
- "ServiceName": "indexer",
- "VersionNumber": "1.0.5-SNAPSHOT"
- },
- "Tags" : {
- "Environment" : "dev"
- }
-}
diff --git a/provider/indexer-aws/CloudFormation/Params/prod.template_configuration.json b/provider/indexer-aws/CloudFormation/Params/prod.template_configuration.json
deleted file mode 100644
index cf2353331086afe91f04f7abcaa4c9736a4a5d84..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/CloudFormation/Params/prod.template_configuration.json
+++ /dev/null
@@ -1,44 +0,0 @@
-{
- "Parameters" : {
- "Environment" : "prod",
- "DeploymentRegion" : "us-east-1",
- "ApplicationName" : "os-indexer",
- "SearchApplicationName" : "os-search",
- "KeyName": "indexer-ecs-keypair",
- "DesiredCapacity": "2",
- "MinSize": "0",
- "MaxSize": "3",
- "InstanceType": "t3.large",
- "IndexerServiceIamUsername": "service-user-os-indexer",
- "IndexerServiceIamKeyRotationSerial": "1",
- "SNSTopicName": "osdu-indexer-messages",
- "SQSQueueName": "osdu-indexer-queue",
- "IndexCacheName": "indexerIndexCache",
- "IndexCacheEngine": "redis",
- "IndexCacheNodeInstanceType": "cache.t2.micro",
- "IndexCacheNumberOfCacheNodes": "1",
- "SchemaCacheName": "indexerSchemaCache",
- "SchemaCacheEngine": "redis",
- "SchemaCacheNodeInstanceType": "cache.t2.micro",
- "SchemaCacheNumberOfCacheNodes": "1",
- "ElasticsearchDomainName": "osdu-indexer",
- "ElasticsearchNodeInstanceType": "t2.medium.elasticsearch",
- "DedicatedMasterInstanceType": "t2.medium.elasticsearch",
- "NumberOfElasticsearchNodes": "2",
- "NumberOfDedicatedMasterNodes": "2",
- "ZoneAwarenessEnabled": "false",
- "ElasticsearchVersion": "6.8",
- "EBSVolumeSize": "10",
- "ECSPort": "443",
- "ECSCPUAllocation": "1024",
- "ECSMemoryAllocation": "3072",
- "DomainName": "",
- "HostedZoneName": "",
- "AcmCertificateArn": "",
- "ServiceName": "indexer",
- "VersionNumber": "1.0.5-SNAPSHOT"
- },
- "Tags" : {
- "Environment" : "prod"
- }
-}
diff --git a/provider/indexer-aws/CloudFormation/Params/uat.template_configuration.json b/provider/indexer-aws/CloudFormation/Params/uat.template_configuration.json
deleted file mode 100644
index 10d7331ed0253013feff3eaf96e468aca3b80394..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/CloudFormation/Params/uat.template_configuration.json
+++ /dev/null
@@ -1,44 +0,0 @@
-{
- "Parameters" : {
- "Environment" : "uat",
- "DeploymentRegion" : "us-east-1",
- "ApplicationName" : "os-indexer",
- "SearchApplicationName" : "os-search",
- "KeyName": "indexer-ecs-keypair",
- "DesiredCapacity": "2",
- "MinSize": "0",
- "MaxSize": "3",
- "InstanceType": "t3.large",
- "IndexerServiceIamUsername": "service-user-os-indexer",
- "IndexerServiceIamKeyRotationSerial": "1",
- "SNSTopicName": "osdu-indexer-messages",
- "SQSQueueName": "osdu-indexer-queue",
- "IndexCacheName": "indexerIndexCache",
- "IndexCacheEngine": "redis",
- "IndexCacheNodeInstanceType": "cache.t2.micro",
- "IndexCacheNumberOfCacheNodes": "1",
- "SchemaCacheName": "indexerSchemaCache",
- "SchemaCacheEngine": "redis",
- "SchemaCacheNodeInstanceType": "cache.t2.micro",
- "SchemaCacheNumberOfCacheNodes": "1",
- "ElasticsearchDomainName": "osdu-indexer",
- "ElasticsearchNodeInstanceType": "t2.medium.elasticsearch",
- "DedicatedMasterInstanceType": "t2.medium.elasticsearch",
- "NumberOfElasticsearchNodes": "2",
- "NumberOfDedicatedMasterNodes": "2",
- "ZoneAwarenessEnabled": "false",
- "ElasticsearchVersion": "6.8",
- "EBSVolumeSize": "10",
- "ECSPort": "443",
- "ECSCPUAllocation": "1024",
- "ECSMemoryAllocation": "3072",
- "DomainName": "",
- "HostedZoneName": "",
- "AcmCertificateArn": "",
- "ServiceName": "indexer",
- "VersionNumber": "1.0.5-SNAPSHOT"
- },
- "Tags" : {
- "Environment" : "uat"
- }
-}
diff --git a/provider/indexer-aws/Dockerfile b/provider/indexer-aws/build-aws/Dockerfile
similarity index 74%
rename from provider/indexer-aws/Dockerfile
rename to provider/indexer-aws/build-aws/Dockerfile
index 07b7281a7d160013df35a5a150fdcaa8f5bf2c53..4af0e6ae8f3b38370edf5354972609066316b86b 100644
--- a/provider/indexer-aws/Dockerfile
+++ b/provider/indexer-aws/build-aws/Dockerfile
@@ -12,15 +12,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+# https://docs.spring.io/spring-boot/docs/current/reference/html/deployment.html
FROM amazoncorretto:8
-ARG versionNumber
-ARG service
-ENV serviceName=${service}-aws
-ENV awsJar=${serviceName}-${versionNumber}-spring-boot.jar
-
+ARG JAR_FILE=provider/indexer-aws/target/*spring-boot.jar
WORKDIR /
-COPY provider/${serviceName}/target/${awsJar} ${awsJar}
+COPY ${JAR_FILE} app.jar
EXPOSE 8080
-
-CMD ["sh","-c", " java -jar ${awsJar}"]
+ENTRYPOINT java $JAVA_OPTS -jar /app.jar
\ No newline at end of file
diff --git a/provider/indexer-aws/buildspec-jar-deploy.yml b/provider/indexer-aws/buildspec-jar-deploy.yml
deleted file mode 100644
index 546f75409fd9e37722dd85a7298633e2f3b0932b..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/buildspec-jar-deploy.yml
+++ /dev/null
@@ -1,59 +0,0 @@
-# Copyright © Amazon Web Services
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: 0.2
-
-phases:
- install:
- runtime-versions:
- java: openjdk8
- commands:
- - echo Entered the install phase...
- - apt-get update -y
- - apt-get install -y maven
- - java -version
- - mvn clean # .m2 is not created until the first Maven command
- - cp ./provider/indexer-aws/maven/settings.xml /root/.m2/settings.xml # copy the AWS-specific settings.xml to the CodeBuild instance's .m2 folder
- - cat /root/.m2/settings.xml
- - java -version
- - export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
- - echo $JAVA_HOME
- - mvn -version
- - echo "Look below for M2 bucket name:"
- - echo $M2_REPO_S3_BUCKET
- - aws s3 sync s3://$M2_REPO_S3_BUCKET /root/.m2 # copy previous state of the shared libraries' .m2 folder from S3 to local
- - nohup /usr/local/bin/dockerd --host=unix:///var/run/docker.sock --host=tcp://127.0.0.1:2375 --storage-driver=overlay2& # start the Docker Daemon
- - timeout 15 sh -c "until docker info; do echo .; sleep 1; done" # wait for Docker to be ready before proceeding to the build steps
- build:
- commands:
- - echo os-indexer Java build started on `date`...
- - java -version
- - export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
- - mvn -version
- - echo All environment variables
- - printenv
- - mvn clean test -P indexer-core,indexer-aws
- - echo ...os-indexer Java build completed on `date`.
- - echo os-indexer beginning packaging to jar...
- - mvn clean install -P indexer-core,indexer-aws -Ddeployment.environment=$ENVIRONMENT
- - echo Uploading os-indexer JAR to S3...
- - aws s3 cp provider/$JAR_SERVICE_BASE-aws/target s3://$JAR_DEPLOY_S3_BUCKET/$JAR_SERVICE_BASE-aws --recursive --exclude "*" --include "*.jar" # build and push the JAR(s) to S3
-
-cache:
- paths:
- - '/root/.m2/**/*'
-
-artifacts:
- files:
- - '**/*'
diff --git a/provider/indexer-aws/buildspec-post-deploy.yml b/provider/indexer-aws/buildspec-post-deploy.yml
deleted file mode 100644
index 5a576fa4c687e91a904ee66939f01d99df4b0981..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/buildspec-post-deploy.yml
+++ /dev/null
@@ -1,87 +0,0 @@
-# Copyright © Amazon Web Services
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: 0.2
-
-phases:
- install:
- runtime-versions:
- java: openjdk8
- commands:
- - echo Entered the install phase...
- - apt-get update -y
- - apt-get install -y maven
- - java -version
- - mvn clean # .m2 is not created until the first Maven command
- - cp ./provider/indexer-aws/maven/settings.xml /root/.m2/settings.xml # copy the AWS-specific settings.xml to the CodeBuild instance's .m2 folder
- - cat /root/.m2/settings.xml
- - java -version
- - export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
- - echo $JAVA_HOME
- - mvn -version
- - echo "Look below for M2 bucket name:"
- - echo $M2_REPO_S3_BUCKET
- - aws s3 sync s3://$M2_REPO_S3_BUCKET /root/.m2 # copy previous state of the shared libraries' .m2 folder from S3 to local
- - nohup /usr/local/bin/dockerd --host=unix:///var/run/docker.sock --host=tcp://127.0.0.1:2375 --storage-driver=overlay2& # start the Docker Daemon
- - timeout 15 sh -c "until docker info; do echo .; sleep 1; done" # wait for Docker to be ready before proceeding to the build steps
- pre_build:
- commands:
- - echo Logging in to Amazon ECR...
- - $(aws ecr get-login --no-include-email --region $AWS_REGION)
- - echo $AWS_ACCOUNT_ID
- - REPOSITORY_URI=$AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/$IMAGE_REPO_NAME # build and store the ECR repo URI
- - IMAGE_TAG=build-$(echo $CODEBUILD_BUILD_ID | awk -F":" '{print $2}') # generate a version tag from the commit hash for the Docker image
- - COMMIT_HASH=$(echo $CODEBUILD_RESOLVED_SOURCE_VERSION | cut -c 1-7) # get the commit hash
- build:
- commands:
- - echo Indexer-core Java build started on `date`...
- - echo os-indexer Java build started on `date`...
- - java -version
- - export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
- - mvn -version
- - echo Setting environment variables from CloudFormation Exports... # use the AWS CLI commands to query for the CloudFormation export values created in the previous step and set the required environment variables
- - echo Environment - $ENVIRONMENT
- - echo SchemaCacheName - $SCHEMA_CACHE_NAME
- - echo IndexCacheName - $INDEX_CACHE_NAME
- - echo AWSRegion - $AWS_REGION
- - export VERSIONNUMBER=$(aws cloudformation list-exports --query "Exports[?Name=='$ENVIRONMENT-$APPLICATION_NAME-JarVersionNumber'].[Value]" --output text --region $AWS_REGION)
- - echo VERSIONNUMBER - $VERSIONNUMBER
- - export SERVICE=$(aws cloudformation list-exports --query "Exports[?Name=='$ENVIRONMENT-$APPLICATION_NAME-JarServiceName'].[Value]" --output text --region $AWS_REGION)
- - echo SERVICE - $SERVICE
- - echo ...finished setting environment variables!
- - echo All environment variables
- - printenv
- - mvn clean test -P indexer-core,indexer-aws
- - echo ...os-indexer Java build completed on `date`.
- - echo os-indexer beginning packaging to jar...
- - mvn clean install -P indexer-core,indexer-aws -Ddeployment.environment=$ENVIRONMENT -Dversion.number=$VERSIONNUMBER
- - echo os-indexer Docker image build started on `date`...
- - docker build -f provider/indexer-aws/Dockerfile -t $REPOSITORY_URI:latest --build-arg versionNumber=$VERSIONNUMBER --build-arg service=$SERVICE .
- - docker tag $REPOSITORY_URI:latest $REPOSITORY_URI:$IMAGE_TAG
- - echo ...os-indexer Docker image build completed on `date`.
- - echo Pushing the Docker image to ECR...
- - docker push $REPOSITORY_URI:latest
- - docker push $REPOSITORY_URI:$IMAGE_TAG
- - echo Docker image pushed to ECR successfully!
- - ECS_CLUSTER_NAME=$(aws cloudformation list-exports --query "Exports[?Name=='$ENVIRONMENT-$APPLICATION_NAME-EcsClusterName'].[Value]" --output text --region $AWS_REGION)
- - ECS_SERVICE_NAME=$(aws cloudformation list-exports --query "Exports[?Name=='$ENVIRONMENT-$APPLICATION_NAME-EcsServiceName'].[Value]" --output text --region $AWS_REGION)
- - aws ecs update-service --cluster $ECS_CLUSTER_NAME --service $ECS_SERVICE_NAME --force-new-deployment # force a new deployment with the updated image
-
-cache:
- paths:
- - '/root/.m2/**/*'
-
-artifacts:
- files:
- - '**/*'
diff --git a/provider/indexer-aws/buildspec-pre-deploy.yml b/provider/indexer-aws/buildspec-pre-deploy.yml
deleted file mode 100644
index 14697d9c882a1905a8205f06328611e2d3c478ef..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/buildspec-pre-deploy.yml
+++ /dev/null
@@ -1,61 +0,0 @@
-# Copyright © Amazon Web Services
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: 0.2
-
-phases:
- install:
- runtime-versions:
- java: openjdk8
- docker: 18
- commands:
- - echo Entered the install phase...
- - apt-get update -y
- - apt-get install -y maven
- - java -version
- - echo $JAVA_HOME
- - export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
- - echo $JAVA_HOME
- - mvn clean # .m2 is not created until the first Maven command
- - cp ./indexer-core/maven/settings.xml /root/.m2/settings.xml # replace the default settings.xml with our custom one
- - export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
- - mvn -version
- build:
- commands:
- - echo Starting 'Copying CloudFormation scripts to S3://$CFN_S3_BUCKET/$APPLICATION_NAME'
- - pwd
- - ls
- - aws s3 cp ./provider/indexer-aws/CloudFormation "s3://$CFN_S3_BUCKET/$APPLICATION_NAME" --exclude "*" --include "*.yml" --recursive --debug
- - echo Ending 'Ending CloudFormation scripts to S3://$CFN_S3_BUCKET/$APPLICATION_NAME'
-# - echo os-indexer build started on `date`...
- - pwd
- - ls -R -la
- - java -version
- - echo $JAVA_HOME
- - export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
- - mvn -version
-# - mvn test
- post_build:
- commands:
-# - echo ...os-indexer build completed on `date`
-# - echo os-indexer beginning packaging to jar...
-# - mvn package
-
-cache:
- paths:
- - '/root/.m2/**/*'
-
-artifacts:
- files:
- - '**/*'
diff --git a/provider/indexer-aws/maven/settings.xml b/provider/indexer-aws/maven/settings.xml
index adba9a7602b0887236967efc41b49ccad08a7ca1..0ca9fe8103a2386f2ff0479c788306e1312faa3d 100644
--- a/provider/indexer-aws/maven/settings.xml
+++ b/provider/indexer-aws/maven/settings.xml
@@ -11,6 +11,6 @@
<!-- The generated token was last updated on 01/13/2020 and expires on or before 01/12/2021 -->
<password>${azure.devops.token}</password>
</server>
-
</servers>
+
</settings>
diff --git a/provider/indexer-aws/pom.xml b/provider/indexer-aws/pom.xml
index aa763a675d577afe9e3380606336703d0a09a48c..6130309d7b97ce3d8abd764bd66927219b1e9fe9 100644
--- a/provider/indexer-aws/pom.xml
+++ b/provider/indexer-aws/pom.xml
@@ -50,7 +50,7 @@
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
<artifactId>os-core-lib-aws</artifactId>
- <version>0.0.10</version>
+ <version>0.2.0</version>
</dependency>
<!-- AWS managed packages -->
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/IndexCacheImpl.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/IndexCacheImpl.java
index b8953ba2d6cc3aaee5e99565bbc2417413f5d153..c28acd5c3f9d213c5658737d8d393475c85001a9 100644
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/IndexCacheImpl.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/IndexCacheImpl.java
@@ -24,8 +24,8 @@ public class IndexCacheImpl implements IIndexCache<String, Boolean>, AutoCloseab
private RedisCache<String, Boolean> cache;
- public IndexCacheImpl(@Value("${aws.elasticache.cluster.index.endpoint}") final String REDIS_SEARCH_HOST,
- @Value("${aws.elasticache.cluster.index.port}") final String REDIS_SEARCH_PORT,
+ public IndexCacheImpl(@Value("${aws.elasticache.cluster.endpoint}") final String REDIS_SEARCH_HOST,
+ @Value("${aws.elasticache.cluster.port}") final String REDIS_SEARCH_PORT,
@Value("${aws.elasticache.cluster.index.expiration}") final String INDEX_CACHE_EXPIRATION) {
cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT),
Integer.parseInt(INDEX_CACHE_EXPIRATION) * 60, String.class, Boolean.class);
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/SchemaCacheImpl.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/SchemaCacheImpl.java
index 112b3b2f8b1a591f785edb675b7a84cfe2cf5198..e0a713737ccb356313ff9d5edf43ebeac6fda331 100644
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/SchemaCacheImpl.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/SchemaCacheImpl.java
@@ -24,8 +24,8 @@ public class SchemaCacheImpl implements ISchemaCache<String, String>, AutoClosea
private RedisCache<String, String> cache;
- public SchemaCacheImpl(@Value("${aws.elasticache.cluster.schema.endpoint}") final String REDIS_SEARCH_HOST,
- @Value("${aws.elasticache.cluster.schema.port}") final String REDIS_SEARCH_PORT,
+ public SchemaCacheImpl(@Value("${aws.elasticache.cluster.endpoint}") final String REDIS_SEARCH_HOST,
+ @Value("${aws.elasticache.cluster.port}") final String REDIS_SEARCH_PORT,
@Value("${aws.elasticache.cluster.schema.expiration}") final String SCHEMA_CACHE_EXPIRATION) {
cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT),
Integer.parseInt(SCHEMA_CACHE_EXPIRATION) * 60, String.class, String.class);
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/di/EntitlementsFactoryImpl.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/di/EntitlementsFactoryImpl.java
deleted file mode 100644
index 85517a87d6873f420b325e2201612e17181f3e1e..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/di/EntitlementsFactoryImpl.java
+++ /dev/null
@@ -1,36 +0,0 @@
-// Copyright © Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.aws.di;
-
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.entitlements.IEntitlementsFactory;
-import org.opengroup.osdu.core.common.entitlements.IEntitlementsService;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.context.annotation.Primary;
-import org.springframework.stereotype.Component;
-
-@Component
-@Primary
-public class EntitlementsFactoryImpl implements IEntitlementsFactory {
- @Value("${aws.lambda.get-groups-function-name}")
- private String getGroupsFunctionName;
-
- @Override
- public IEntitlementsService create(DpsHeaders headers) {
- EntitlementsServiceImpl service = new EntitlementsServiceImpl(headers);
- service.setEntitlementsServiceHelper(getGroupsFunctionName);
- return service;
- }
-}
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/di/EntitlementsServiceImpl.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/di/EntitlementsServiceImpl.java
deleted file mode 100644
index 3dd657c6699407a4557f4d149fcf2a241d39e762..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/di/EntitlementsServiceImpl.java
+++ /dev/null
@@ -1,138 +0,0 @@
-// Copyright © Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.aws.di;
-
-import com.amazonaws.regions.Regions;
-import com.amazonaws.services.lambda.invoke.LambdaFunctionException;
-import com.amazonaws.services.lambda.invoke.LambdaSerializationException;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.opengroup.osdu.core.common.model.entitlements.*;
-import org.opengroup.osdu.core.common.model.entitlements.MemberInfo;
-import org.opengroup.osdu.core.common.model.entitlements.Members;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.entitlements.IEntitlementsService;
-import org.opengroup.osdu.core.aws.entitlements.*;
-import org.opengroup.osdu.core.common.http.HttpResponse;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.springframework.context.annotation.Lazy;
-import org.springframework.http.HttpStatus;
-import sun.reflect.generics.reflectiveObjects.NotImplementedException;
-
-import javax.inject.Inject;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-public class EntitlementsServiceImpl implements IEntitlementsService {
- private DpsHeaders dpsHeaders;
- private EntitlementsServiceHelper entitlementsServiceHelper;
-
- private final static String ACCESS_DENIED = "Access denied";
- private final static String ACCESS_DENIED_MSG = "The user is not authorized to perform this action";
-
- @Inject
- @Lazy
- private JaxRsDpsLog jaxRsDpsLog;
-
- public EntitlementsServiceImpl(DpsHeaders headers){
- this.dpsHeaders = headers;
- }
-
- public void setEntitlementsServiceHelper(String getGroupsFunctionName){
- entitlementsServiceHelper = new EntitlementsServiceHelper(Regions.US_EAST_1, getGroupsFunctionName);
- }
-
- @Override
- public MemberInfo addMember(GroupEmail groupEmail, MemberInfo memberInfo) throws EntitlementsException {
- throw new NotImplementedException();
- }
-
- @Override
- public Members getMembers(GroupEmail groupEmail, GetMembers getMembers) throws EntitlementsException {
- throw new NotImplementedException();
- }
-
- @Override
- public Groups getGroups() throws EntitlementsException {
- Groups groups;
- GroupsRequest request = entitlementsServiceHelper.constructRequest(this.dpsHeaders.getHeaders());
-
- try{
- GroupsResult groupsResult = entitlementsServiceHelper.getGroups(request);
- groups = getGroupsFromResult(groupsResult);
- } catch (JsonProcessingException e) {
- throw new EntitlementsException(e.getMessage(), new HttpResponse());
- } catch (LambdaFunctionException e){
- throw new EntitlementsException(e.getMessage(), new HttpResponse());
- } catch (LambdaSerializationException e){
- throw new EntitlementsException(e.getMessage(), new HttpResponse());
- } catch (IOException e){
- throw new EntitlementsException(e.getMessage(), new HttpResponse());
- }
-
- return groups;
- }
-
- @Override
- public GroupInfo createGroup(CreateGroup createGroup) throws EntitlementsException {
- throw new NotImplementedException();
- }
-
- @Override
- public void deleteMember(String s, String s1) throws EntitlementsException {
- throw new NotImplementedException();
- }
-
- @Override
- public Groups authorizeAny(String... strings) throws EntitlementsException {
- throw new NotImplementedException();
- }
-
- @Override
- public void authenticate() throws EntitlementsException {
- throw new NotImplementedException();
- }
-
- private Groups getGroupsFromResult(GroupsResult result) throws EntitlementsException, IOException {
- ObjectMapper mapper = new ObjectMapper();
- Groups groups = new Groups();
- if(result.statusCode == HttpStatus.OK.value()) {
- TypeReference<List<GroupInfoRaw>> mapType = new TypeReference<List<GroupInfoRaw>>() {};
- List<GroupInfoRaw> groupInfosRaw = mapper.readValue(result.body, mapType);
- List<GroupInfo> groupInfos = new ArrayList<>();
- for(GroupInfoRaw groupInfoRaw : groupInfosRaw){
- GroupInfo groupInfo = new GroupInfo();
- groupInfo.setDescription(groupInfoRaw.groupDescription);
- groupInfo.setEmail(groupInfoRaw.groupEmail);
- groupInfo.setName(groupInfoRaw.groupName);
- groupInfos.add(groupInfo);
- }
- groups.setDesId(result.headers.get(RequestKeys.USER_HEADER_KEY));
- groups.setMemberEmail(result.headers.get(RequestKeys.USER_HEADER_KEY));
- groups.setGroups(groupInfos);
- } else {
- if(result.statusCode == HttpStatus.UNAUTHORIZED.value()){
- throw new AppException(HttpStatus.FORBIDDEN.value(), ACCESS_DENIED, ACCESS_DENIED_MSG);
- } else {
- throw new EntitlementsException(String.format("Getting groups for user returned %s status code",
- result.statusCode), new HttpResponse());
- }
- }
- return groups;
- }
-}
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/persistence/ElasticRepositoryImpl.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/persistence/ElasticRepositoryImpl.java
index 70e64ca21d70497fc21121729350d5b16f2a6a44..b65add33a07eb10ce43a89218344f01ab3dae4b0 100644
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/persistence/ElasticRepositoryImpl.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/persistence/ElasticRepositoryImpl.java
@@ -14,17 +14,19 @@
package org.opengroup.osdu.indexer.aws.persistence;
+import org.opengroup.osdu.core.aws.ssm.ParameterStorePropertySource;
+import org.opengroup.osdu.core.aws.ssm.SSMConfig;
import org.opengroup.osdu.core.common.model.search.ClusterSettings;
import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
+import javax.annotation.PostConstruct;
+
@Component
public class ElasticRepositoryImpl implements IElasticRepository {
- // TODO: Will need to be implemented later
-
@Value("${aws.es.host}")
String host;
@@ -33,6 +35,27 @@ public class ElasticRepositoryImpl implements IElasticRepository {
String userNameAndPassword = "testing";
+ @Value("${aws.elasticsearch.port}")
+ String portParameter;
+
+ @Value("${aws.elasticsearch.host}")
+ String hostParameter;
+
+ @Value("${aws.ssm}")
+ String ssmEnabledString;
+
+ private ParameterStorePropertySource ssm;
+
+ @PostConstruct
+ private void postConstruct() {
+ if( Boolean.parseBoolean(ssmEnabledString)) {
+ SSMConfig ssmConfig = new SSMConfig();
+ ssm = ssmConfig.amazonSSM();
+ host = ssm.getProperty(hostParameter).toString();
+ port = Integer.parseInt(ssm.getProperty(portParameter).toString());
+ }
+ }
+
@Override
public ClusterSettings getElasticClusterSettings(TenantInfo tenantInfo) {
return new ClusterSettings(host, port, userNameAndPassword);
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/publish/PublisherImpl.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/publish/PublisherImpl.java
index c2efd5c7e235efec8719edabd21ccf64d7bd7613..d5a5ae5195eb483fa7bc11e7eb06839c293f1367 100644
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/publish/PublisherImpl.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/publish/PublisherImpl.java
@@ -17,6 +17,8 @@ package org.opengroup.osdu.indexer.aws.publish;
import com.amazonaws.services.sns.model.MessageAttributeValue;
import com.amazonaws.services.sns.model.PublishRequest;
import com.amazonaws.services.sns.AmazonSNS;
+import org.opengroup.osdu.core.aws.ssm.ParameterStorePropertySource;
+import org.opengroup.osdu.core.aws.ssm.SSMConfig;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.aws.sns.AmazonSNSConfig;
import org.opengroup.osdu.core.aws.sns.PublishRequestBuilder;
@@ -34,16 +36,24 @@ public class PublisherImpl implements IPublisher {
AmazonSNS snsClient;
- @Value("${aws.sns.arn}")
+ private ParameterStorePropertySource ssm;
+
private String amazonSNSTopic;
- @Value("${aws.sns.region}")
+ @Value("${aws.region}")
private String amazonSNSRegion;
+ @Value("${aws.indexer.sns.topic.arn}")
+ private String parameter;
+
+
@Inject
public void init(){
AmazonSNSConfig snsConfig = new AmazonSNSConfig(amazonSNSRegion);
snsClient = snsConfig.AmazonSNS();
+ SSMConfig ssmConfig = new SSMConfig();
+ ssm = ssmConfig.amazonSSM();
+ amazonSNSTopic = ssm.getProperty(parameter).toString();
}
public void publishStatusChangedTagsToTopic(DpsHeaders headers, JobStatus indexerBatchStatus) throws Exception
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java
index bffecf18d7765fe7c52acaddc7dab246c64e13ec..5930c6a3cebbd46d0228b1dbf066162046e7d4fa 100644
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java
@@ -21,6 +21,8 @@ import com.amazonaws.services.sqs.AmazonSQS;
import com.amazonaws.services.sqs.model.SendMessageRequest;
import com.google.gson.Gson;
import org.opengroup.osdu.core.aws.sns.AmazonSNSConfig;
+import org.opengroup.osdu.core.aws.ssm.ParameterStorePropertySource;
+import org.opengroup.osdu.core.aws.ssm.SSMConfig;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.aws.sqs.AmazonSQSConfig;
import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
@@ -39,21 +41,29 @@ public class IndexerQueueTaskBuilderAws extends IndexerQueueTaskBuilder {
private AmazonSNS snsClient;
- @Value("${aws.region}")
- private String region;
+ private ParameterStorePropertySource ssm;
- @Value("${aws.sns.storage.arn}")
private String amazonSNSTopic;
private String retryString = "retry";
private Gson gson;
+ @Value("${aws.region}")
+ private String region;
+
+ @Value("${aws.storage.sns.topic.arn}")
+ String parameter;
+
+
@Inject
public void init() {
AmazonSNSConfig config = new AmazonSNSConfig(region);
snsClient = config.AmazonSNS();
gson =new Gson();
+ SSMConfig ssmConfig = new SSMConfig();
+ ssm = ssmConfig.amazonSSM();
+ amazonSNSTopic = ssm.getProperty(parameter).toString();
}
@Override
diff --git a/provider/indexer-aws/src/main/resources/application.properties b/provider/indexer-aws/src/main/resources/application.properties
index 54085fcb9b7f930dcecb71d0dffd89c118aa2a20..78fdb52f7df76dec63d2cdaffa1c5bed067c1d4b 100644
--- a/provider/indexer-aws/src/main/resources/application.properties
+++ b/provider/indexer-aws/src/main/resources/application.properties
@@ -1,38 +1,17 @@
LOG_PREFIX=indexer
server.servlet.contextPath=/api/indexer/v2/
-logging.level.org.springframework.web=DEBUG
+logging.level.org.springframework.web=${LOG_LEVEL}
server.port=${APPLICATION_PORT}
-JAVA_HEAP_OPTS=-Xms${JAVA_HEAP_MEMORY}M -Xmx${JAVA_HEAP_MEMORY}M
-JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45
aws.threads=50
DEFAULT_DATA_COUNTRY=US
CRON_INDEX_CLEANUP_THRESHOLD_DAYS=3
CRON_EMPTY_INDEX_CLEANUP_THRESHOLD_DAYS=7
-## AWS DynamoDB configuration
-aws.dynamodb.key=kind
-aws.dynamodb.table.prefix=${ENVIRONMENT}-
-aws.dynamodb.region=${AWS_REGION}
-aws.dynamodb.endpoint=dynamodb.${AWS_REGION}.amazonaws.com
-
-## AWS S3 configuration
-aws.s3.region=${AWS_REGION}
-aws.s3.endpoint=s3.${AWS_REGION}.amazonaws.com
-aws.s3.records.bucket-name=${ENVIRONMENT}-${S3_DATA_BUCKET}
-aws.s3.max-record-threads=2000
-aws.s3.enable-https=true
-
-## AWS SNS configuration
-aws.sns.region=${AWS_REGION}
-aws.sns.arn=arn:aws:sns:${AWS_REGION}:${AWS_ACCOUNT_ID}:${ENVIRONMENT}-${SNS_TOPIC_NAME}
-aws.sns.storage.arn=arn:aws:sns:${AWS_REGION}:${AWS_ACCOUNT_ID}:${SNS_STORAGE_TOPIC_NAME}
-
-## AWS SQS Configuration
-aws.sqs.queue=${ENVIRONMENT}-osdu-indexer-queue
-
# AWS ES configuration
+ELASTIC_HOST=""
+ELASTIC_PORT=0
aws.es.host=${ELASTIC_HOST}
aws.es.port=${ELASTIC_PORT}
aws.es.userNameAndPassword=notused
@@ -41,26 +20,34 @@ aws.es.serviceName=es
GAE_SERVICE=indexer
-# TODO This needs to be changed so it snot hard
-STORAGE_SCHEMA_HOST=https://${STORAGE_HOST}/api/storage/v2/schemas
-STORAGE_QUERY_RECORD_HOST=https://${STORAGE_HOST}/api/storage/v2/query/records
-STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=https://${STORAGE_HOST}/api/storage/v2/query/records:batch
+STORAGE_SCHEMA_HOST=${STORAGE_HOST}/api/storage/v2/schemas
+STORAGE_QUERY_RECORD_HOST=${STORAGE_HOST}/api/storage/v2/query/records
+STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=${STORAGE_HOST}/api/storage/v2/query/records:batch
STORAGE_RECORDS_BATCH_SIZE=20
-INDEXER_QUEUE_HOST=http://sqs.${AWS_REGION}.amazonaws.com/${AWS_ACCOUNT_ID}/${ENVIRONMENT}-osdu-indexer-queue
-
+INDEXER_QUEUE_HOST=""
## AWS ElastiCache configuration
-aws.elasticache.cluster.index.endpoint=${CACHE_CLUSTER_INDEX_ENDPOINT}
-aws.elasticache.cluster.index.port=${CACHE_CLUSTER_INDEX_PORT}
-aws.elasticache.cluster.schema.endpoint=${CACHE_CLUSTER_SCHEMA_ENDPOINT}
-aws.elasticache.cluster.schema.port=${CACHE_CLUSTER_SCHEMA_PORT}
+aws.elasticache.cluster.endpoint=${CACHE_CLUSTER_ENDPOINT}
+aws.elasticache.cluster.port=${CACHE_CLUSTER_PORT}
## Cache Settings
aws.elasticache.cluster.index.expiration=60
aws.elasticache.cluster.schema.expiration=60
-
# Maximum size of cache value
MAX_CACHE_VALUE_SIZE=1000
## AWS Lambda configuration
aws.lambda.get-groups-function-name=${ENVIRONMENT}-os-entitlements-GroupsFunction
+
+
+## Default DynamoDB Settings
+aws.dynamodb.table.prefix=${ENVIRONMENT}-
+aws.dynamodb.endpoint=dynamodb.${AWS_REGION}.amazonaws.com
+
+aws.ssm=${SSM_ENABLED}
+aws.ssm.prefix=/osdu/${ENVIRONMENT}
+
+aws.elasticsearch.host=${aws.ssm.prefix}/elastic-search/end-point
+aws.elasticsearch.port=${aws.ssm.prefix}/elastic-search/end-point-port
+aws.indexer.sns.topic.arn=${aws.ssm.prefix}/indexer/indexer-sns-topic-arn
+aws.storage.sns.topic.arn=${aws.ssm.prefix}/storage/storage-sns-topic-arn
diff --git a/provider/indexer-azure/README.md b/provider/indexer-azure/README.md
index 258014ddb6f2707175b3055aefefb7b98b7c8692..f3eb0638648711b8ede93e134bb12d89ff6e1dcf 100644
--- a/provider/indexer-azure/README.md
+++ b/provider/indexer-azure/README.md
@@ -88,7 +88,7 @@ Java version: 1.8.0_212, vendor: AdoptOpenJDK, runtime: /usr/lib/jvm/jdk8u212-b0
...
```
-You will need to configure access to the remote maven repository that holds the OSDU dependencies. This file should live within `~/.m2/settings.xml`:
+You may need to configure access to the remote maven repository that holds the OSDU dependencies. This file should live within `~/.m2/settings.xml`:
```bash
$ cat ~/.m2/settings.xml
<?xml version="1.0" encoding="UTF-8"?>
@@ -123,13 +123,16 @@ $ (cd provider/indexer-azure/ && mvn clean package)
# Note: this assumes that the environment variables for running the service as outlined
# above are already exported in your environment.
$ java -jar $(find provider/indexer-azure/target/ -name *-spring-boot.jar)
+
+# Alternately you can run using the Mavan Task
+$ mvn spring-boot:run
```
### Test the application
After the service has started it should be accessible via a web browser by visiting [http://localhost:8080/swagger-ui.html](http://localhost:8080/swagger-ui.html). If the request does not fail, you can then run the integration tests.
-> **Note**: the integration tests for `os-indexer-azure` work by validating that records submitted to `os-storage-azure` can eventually be queried by `os-search-azure`. This only works if the messages emitted by `os-storage-azure` can be consumed by `os-indexer-queue-azure`, which will submit the indexing request to `os-indexer-azure`.
+> **Note**: the integration tests for `os-indexer-azure` work by validating that records submitted to `os-storage-azure` can eventually be queried by `os-search-azure`. This only works if the messages emitted by `os-storage-azure` can be consumed by `os-indexer-queue-azure`, which will submit the indexing request to `os-indexer-azure`.
>
> In order to make sure that the integration tests are running against your local environment, you will need to make sure that the there is an instance of `os-indexer-queue-azure` that is configured to call your deployment of `os-indexer-azure`, and that this instance of `os-indexer-queue-azure` is the only consumer of the Service Bus topic.
>
@@ -156,8 +159,103 @@ Jet Brains - the authors of Intellij IDEA, have written an [excellent guide](htt
## Deploying service to Azure
-Service deployments into Azure are standardized to make the process the same for all services. The steps to deploy into
-Azure can be [found here](https://dev.azure.com/slb-des-ext-collaboration/open-data-ecosystem/_git/infrastructure-templates?path=%2Fdocs%2Fosdu%2FSERVICE_DEPLOYMENTS.md&_a=preview)
+Service deployments into Azure are standardized to make the process the same for all services if using ADO and are closely related to the infrastructure deployed. The steps to deploy into Azure can be [found here](https://github.com/azure/osdu-infrastructure)
+
+The default ADO pipeline is /devops/azure-pipeline.yml
+
+
+### Manual Deployment Steps
+
+__Environment Settings__
+
+The following environment variables are necessary to properly deploy a service to an Azure OSDU Environment.
+
+```bash
+# Group Level Variables
+export AZURE_TENANT_ID=""
+export AZURE_SUBSCRIPTION_ID=""
+export AZURE_SUBSCRIPTION_NAME=""
+export AZURE_PRINCIPAL_ID=""
+export AZURE_PRINCIPAL_SECRET=""
+export AZURE_APP_ID=""
+export AZURE_BASENAME_21=""
+export AZURE_BASENAME=""
+export AZURE_BASE=""
+export AZURE_ELASTIC_HOST=""
+export AZURE_ELASTIC_PASSWORD=""
+
+# Pipeline Level Variable
+export AZURE_SERVICE="indexer"
+export AZURE_BUILD_SUBDIR="provider/indexer-azure"
+export AZURE_TEST_SUBDIR="testing/indexer-test-azure"
+
+# Required for Azure Deployment
+export AZURE_CLIENT_ID="${AZURE_PRINCIPAL_ID}"
+export AZURE_CLIENT_SECRET="${AZURE_PRINCIPAL_SECRET}"
+export AZURE_RESOURCE_GROUP="${AZURE_BASENAME}-osdu-r2-app-rg"
+export AZURE_APPSERVICE_PLAN="${AZURE_BASENAME}-osdu-r2-sp"
+export AZURE_APPSERVICE_NAME="${AZURE_BASENAME_21}-au-${AZURE_SERVICE}"
+
+# Required for Testing
+export AZURE_AD_TENANT_ID="$AZURE_TENANT_ID"
+export INTEGRATION_TESTER="$AZURE_PRINCIPAL_ID"
+export AZURE_TESTER_SERVICEPRINCIPAL_SECRET="$AZURE_PRINCIPAL_SECRET"
+export AZURE_AD_APP_RESOURCE_ID="$AZURE_APP_ID"
+export aad_client_id="$AZURE_APP_ID"
+export STORAGE_HOST="https://{AZURE_BASENAME_21}-au-storage.azurewebsites.net/"
+export ELASTIC_HOST="$AZURE_ELASTIC_HOST"
+export ELASTIC_PORT="9243"
+export ELASTIC_USER_NAME="elastic"
+export ELASTIC_PASSWORD="$AZURE_ELASTIC_PASSWORD"
+export DEFAULT_DATA_PARTITION_ID_TENANT1="opendes"
+export DEFAULT_DATA_PARTITION_ID_TENANT2="common"
+export ENVIRONMENT="CLOUD"
+export ENTITLEMENTS_DOMAIN="contoso.com"
+export LEGAL_TAG="opendes-public-usa-dataset-7643990"
+export OTHER_RELEVANT_DATA_COUNTRIES="US"
+```
+
+
+__Azure Service Deployment__
+
+
+1. Deploy the service using the Maven Plugin _(azure_deploy)_
+
+```bash
+cd $AZURE_BUILD_SUBDIR
+mvn azure-webapp:deploy \
+ -DAZURE_TENANT_ID=$AZURE_TENANT_ID \
+ -Dazure.appservice.subscription=$AZURE_SUBSCRIPTION_ID \
+ -DAZURE_CLIENT_ID=$AZURE_CLIENT_ID \
+ -DAZURE_CLIENT_SECRET=$AZURE_CLIENT_SECRET \
+ -Dazure.appservice.resourcegroup=$AZURE_RESOURCE_GROUP \
+ -Dazure.appservice.plan=$AZURE_APPSERVICE_PLAN \
+ -Dazure.appservice.appname=$AZURE_APPSERVICE_NAME
+```
+
+2. Configure the Web App to start the SpringBoot Application _(azure_config)_
+
+```bash
+az login --service-principal -u $AZURE_CLIENT_ID -p $AZURE_CLIENT_SECRET --tenant $AZURE_TENANT_ID
+
+# Set the JAR FILE as required
+TARGET=$(find ./target/ -name '*-spring-boot.jar')
+JAR_FILE=${TARGET##*/}
+
+JAVA_COMMAND="java -jar /home/site/wwwroot/${JAR_FILE}"
+JSON_TEMPLATE='{"appCommandLine":"%s"}'
+JSON_FILE="config.json"
+echo $(printf "$JSON_TEMPLATE" "$JAVA_COMMAND") > $JSON_FILE
+
+az webapp config set --resource-group $AZURE_RESOURCE_GROUP --name $AZURE_APPSERVICE_NAME --generic-configurations @$JSON_FILE
+```
+
+3. Execute the Integration Tests against the Service Deployment _(azure_test)_
+
+```bash
+mvn clean test -f $AZURE_TEST_SUBDIR/pom.xml
+```
+
## License
@@ -165,7 +263,7 @@ Copyright © Microsoft Corporation
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
+You may obtain a copy of the License at
[http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0)
@@ -173,4 +271,4 @@ Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
-limitations under the License.
\ No newline at end of file
+limitations under the License.
diff --git a/provider/indexer-azure/pom.xml b/provider/indexer-azure/pom.xml
index 0456e424ebd1100b09b2c95dde49b89d381705ef..3ad7260a2e4f24e716f197532388004322c1ea53 100644
--- a/provider/indexer-azure/pom.xml
+++ b/provider/indexer-azure/pom.xml
@@ -253,7 +253,7 @@
<useSystemClassLoader>false</useSystemClassLoader>
<threadCount>1</threadCount>
</configuration>
- </plugin>
+ </plugin>
</plugins>
</build>
diff --git a/provider/indexer-gcp/pom.xml b/provider/indexer-gcp/pom.xml
index 3de30a3b6cbaa8e17862e56b5494cdbdd78057dc..795053370729a4d60b19488f45f28c4b24116f82 100644
--- a/provider/indexer-gcp/pom.xml
+++ b/provider/indexer-gcp/pom.xml
@@ -203,7 +203,7 @@
<useSystemClassLoader>false</useSystemClassLoader>
<threadCount>1</threadCount>
</configuration>
- </plugin>
+ </plugin>
</plugins>
</build>
diff --git a/provider/indexer-ibm/pom.xml b/provider/indexer-ibm/pom.xml
index efece35ccb23f2a6c7c26f577968c4d3b015f4d5..f2dab825e3d7ffdbc052b197083bc7046f8a17fe 100644
--- a/provider/indexer-ibm/pom.xml
+++ b/provider/indexer-ibm/pom.xml
@@ -47,19 +47,19 @@
<dependencies>
- <!-- OSDU core service dependencies -->
+ <!-- OSDU core service dependencies -->
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-core</artifactId>
<version>1.0.4-SNAPSHOT</version>
</dependency>
-
+
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-lib-ibm</artifactId>
<version>0.0.14-SNAPSHOT</version>
</dependency>
-
+
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-security</artifactId>
diff --git a/testing/indexer-test-aws/pom.xml b/testing/indexer-test-aws/pom.xml
index e10273d88673511a3b9acfa018672b33e4cff602..d1fa34d89e7d8b903aa0f135ec070080185c2ff8 100644
--- a/testing/indexer-test-aws/pom.xml
+++ b/testing/indexer-test-aws/pom.xml
@@ -62,7 +62,7 @@
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
<artifactId>os-core-lib-aws</artifactId>
- <version>0.0.10</version>
+ <version>0.2.0</version>
</dependency>
<!-- Testing -->
@@ -149,13 +149,4 @@
</dependency>
</dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <version>2.5</version>
- </plugin>
- </plugins>
- </build>
-</project>
+</project>
\ No newline at end of file
diff --git a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
index 83c36a2456d3ecbe1284db5c1ded925d49d731aa..7b1c280c6b9bd96a111c8cfcdfffcefc3c60f09d 100644
--- a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
+++ b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -16,6 +16,7 @@ package org.opengroup.osdu.step_definitions.index.record;
import lombok.extern.java.Log;
import org.opengroup.osdu.common.RecordSteps;
+import org.opengroup.osdu.core.common.model.legal.Legal;
import org.opengroup.osdu.util.AWSHTTPClient;
import cucumber.api.Scenario;
@@ -25,22 +26,53 @@ import cucumber.api.java.en.Given;
import cucumber.api.java.en.Then;
import cucumber.api.java.en.When;
import org.opengroup.osdu.util.ElasticUtilsAws;
+import org.opengroup.osdu.util.LegalTagUtilsAws;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.opengroup.osdu.util.Config.getLegalTag;
+import static org.opengroup.osdu.util.Config.getOtherRelevantDataCountries;
@Log
public class Steps extends RecordSteps {
+ protected LegalTagUtilsAws legalTagUtils;
+ private String legalTagName;
+
public Steps() {
super(new AWSHTTPClient(), new ElasticUtilsAws());
}
@Before
- public void before(Scenario scenario) {
+ public void before(Scenario scenario) throws Exception {
this.scenario = scenario;
this.httpClient = new AWSHTTPClient();
+ legalTagUtils = new LegalTagUtilsAws(this.httpClient);
+ this.legalTagName = this.legalTagUtils.createRandomName();
+ this.legalTagUtils.create(this.legalTagName);
+ }
+
+ @Override
+ public void tearDown() {
+ super.tearDown();
+ this.legalTagUtils.delete(this.legalTagName);
+ }
+
+ @Override
+ protected Legal generateLegalTag() {
+ Legal legal = new Legal();
+ Set<String> legalTags = new HashSet<>();
+ legalTags.add(this.legalTagName);
+ legal.setLegaltags(legalTags);
+ Set<String> otherRelevantCountries = new HashSet<>();
+ otherRelevantCountries.add(getOtherRelevantDataCountries());
+ legal.setOtherRelevantDataCountries(otherRelevantCountries);
+ return legal;
}
@Given("^the schema is created with the following kind$")
- public void the_schema_is_created_with_the_following_kind(DataTable dataTable) {
+ public void the_schema_is_created_with_the_following_kind(DataTable dataTable){
super.the_schema_is_created_with_the_following_kind(dataTable);
}
diff --git a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/ElasticUtilsAws.java b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/ElasticUtilsAws.java
index 77a0440064ca1c7c41194dd3ca45a93613ac6b5f..28d3646d655232dc147719945b02282748e7d498 100644
--- a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/ElasticUtilsAws.java
+++ b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/ElasticUtilsAws.java
@@ -28,6 +28,7 @@ public class ElasticUtilsAws extends ElasticUtils {
@Override
public RestClientBuilder createClientBuilder(String host, String usernameAndPassword, int port) {
+ port = Integer.parseInt(System.getProperty("ELASTIC_PORT", System.getenv("ELASTIC_PORT")));
RestClientBuilder builder = RestClient.builder(new HttpHost(host, port, "https"));
builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(REST_CLIENT_CONNECT_TIMEOUT)
.setSocketTimeout(REST_CLIENT_SOCKET_TIMEOUT));
diff --git a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/LegalTagUtilsAws.java b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/LegalTagUtilsAws.java
new file mode 100644
index 0000000000000000000000000000000000000000..71cbc0900edf92ab9c0d76cbaa4f87c969cd3663
--- /dev/null
+++ b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/LegalTagUtilsAws.java
@@ -0,0 +1,85 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.util;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.http.HttpStatus;
+
+import com.google.gson.JsonArray;
+import com.google.gson.JsonObject;
+import com.sun.jersey.api.client.ClientResponse;
+
+import javax.ws.rs.HttpMethod;
+
+public class LegalTagUtilsAws {
+
+ private HTTPClient httpClient;
+ public LegalTagUtilsAws(HTTPClient httpClient) {
+ this.httpClient = httpClient;
+ }
+ public String createRandomName() {
+ return Config.getDataPartitionIdTenant1() + "-" + System.currentTimeMillis();
+ }
+
+ public ClientResponse create(String legalTagName) throws Exception {
+ return this.create("US", legalTagName, "2099-01-25", "Public Domain Data");
+ }
+
+ protected ClientResponse create(String countryOfOrigin, String name, String expDate, String dataType)
+ throws Exception {
+ String body = getBody(countryOfOrigin, name, expDate, dataType);
+ ClientResponse response = this.httpClient.send(HttpMethod.POST, String.format("%s%s",getLegalUrl(), "legaltags"), body, httpClient.getCommonHeader(), httpClient.getAccessToken());
+
+ assertEquals(HttpStatus.SC_CREATED, response.getStatus());
+ Thread.sleep(100);
+ return response;
+ }
+
+ public ClientResponse delete(String legalTagName) {
+ return this.httpClient.send(HttpMethod.DELETE ,getLegalUrl(), "legaltags/" + legalTagName,httpClient.getCommonHeader(), httpClient.getAccessToken());
+ }
+
+ protected static String getLegalUrl() {
+ String legalUrl = System.getProperty("LEGAL_URL", System.getenv("LEGAL_URL"));
+ if (legalUrl == null || legalUrl.contains("-null")) {
+ legalUrl = "https://os-legal-dot-opendes.appspot.com/api/legal/v1/";
+ }
+ return legalUrl;
+ }
+
+ protected static String getBody(String countryOfOrigin, String name, String expDate, String dataType) {
+
+ JsonArray coo = new JsonArray();
+ coo.add(countryOfOrigin);
+
+ JsonObject properties = new JsonObject();
+ properties.add("countryOfOrigin", coo);
+ properties.addProperty("contractId", "A1234");
+ properties.addProperty("expirationDate", expDate);
+ properties.addProperty("dataType", dataType);
+ properties.addProperty("originator", "MyCompany");
+ properties.addProperty("securityClassification", "Public");
+ properties.addProperty("exportClassification", "EAR99");
+ properties.addProperty("personalData", "No Personal Data");
+
+ JsonObject tag = new JsonObject();
+ tag.addProperty("name", name);
+ tag.addProperty("description", "test for " + name);
+ tag.add("properties", properties);
+
+ return tag.toString();
+ }
+}
diff --git a/testing/indexer-test-aws/src/test/resources/logback-test.xml b/testing/indexer-test-aws/src/test/resources/logback-test.xml
new file mode 100644
index 0000000000000000000000000000000000000000..dafd8ae394dc34b62ffa81307a76900db4b66af8
--- /dev/null
+++ b/testing/indexer-test-aws/src/test/resources/logback-test.xml
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<configuration>
+ <include resource="org/springframework/boot/logging/logback/base.xml" />
+ <root level="INFO" />
+ <logger name="org.springframework" level="INFO"/>
+</configuration>
\ No newline at end of file
diff --git a/testing/indexer-test-azure/pom.xml b/testing/indexer-test-azure/pom.xml
index 4f05b04ad897eafa6c282c1ec7ef549de00450a2..0dc9d6c176adb45c71059fc4df35c7a60fe04f30 100644
--- a/testing/indexer-test-azure/pom.xml
+++ b/testing/indexer-test-azure/pom.xml
@@ -179,4 +179,4 @@
</dependency>
</dependencies>
-</project>
+</project>
\ No newline at end of file
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
index 9665864251a238064890236f28ff4a9f99442e69..18a138d0b3624b8a2c9bca77d1c0be0416603806 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
@@ -295,4 +295,4 @@ public class ElasticUtils {
}
-}
+}
\ No newline at end of file
diff --git a/testing/indexer-test-gcp/pom.xml b/testing/indexer-test-gcp/pom.xml
index c4d9a3538cbb26c8cefac8c7a84dd5f8aca79dde..63544ea5c419cb00db31cd998e7f3e1c44523796 100644
--- a/testing/indexer-test-gcp/pom.xml
+++ b/testing/indexer-test-gcp/pom.xml
@@ -145,4 +145,4 @@
<url>https://community.opengroup.org/api/v4/projects/25/packages/maven</url>
</snapshotRepository>
</distributionManagement>
-</project>
+</project>
\ No newline at end of file