diff --git a/.gitignore b/.gitignore index 80fb6f6fb61ee2b16095613fdecf9f835fc68d05..9e269f1b78050908088da343128dea99d75bacdc 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,4 @@ provider/indexer-gcp/bin/* # Environment configuration *.env +.envrc diff --git a/NOTICE b/NOTICE index f525dead152a65cf15c0ea5685eb2ceae718b2a1..e5e6ed91f54ee9456702ada24d4cb07c7a40fccf 100644 --- a/NOTICE +++ b/NOTICE @@ -166,17 +166,16 @@ The following software have components provided under the terms of this license: - Apache Commons BeanUtils (from http://commons.apache.org/proper/commons-beanutils/) - Apache Commons CLI (from http://commons.apache.org/proper/commons-cli/) - Apache Commons Codec (from http://commons.apache.org/proper/commons-codec/) -- Apache Commons Codec (from http://commons.apache.org/proper/commons-codec/) - Apache Commons Collections (from http://commons.apache.org/proper/commons-collections/) - Apache Commons Lang (from http://commons.apache.org/proper/commons-lang/) - Apache Commons Logging (from http://commons.apache.org/proper/commons-logging/) -- Apache Commons Logging (from http://commons.apache.org/proper/commons-logging/) - Apache Commons Text (from http://commons.apache.org/proper/commons-text/) - Apache Commons Validator (from http://commons.apache.org/proper/commons-validator/) - Apache Commons Validator (from http://commons.apache.org/proper/commons-validator/) - Apache Geronimo JMS Spec 2.0 (from http://geronimo.apache.org/maven/${siteId}/${version}) - Apache HttpAsyncClient (from http://hc.apache.org/httpcomponents-asyncclient) - Apache HttpClient (from http://hc.apache.org/httpcomponents-client) +- Apache HttpClient Cache (from http://hc.apache.org/httpcomponents-client) - Apache HttpCore (from http://hc.apache.org/httpcomponents-core-ga) - Apache HttpCore NIO (from http://hc.apache.org/httpcomponents-core-ga) - Apache Log4j API (from ) @@ -222,6 +221,7 @@ The following software have components provided under the terms of this license: - Doxia :: XHTML Module (from ) - Elastic JNA Distribution (from https://github.com/java-native-access/jna) - Elasticsearch: 5.0.0-alpha5 (from https://github.com/elastic/elasticsearch) +- Elasticsearch: 5.0.0-alpha5 (from https://github.com/elastic/elasticsearch) - FindBugs-jsr305 (from http://findbugs.sourceforge.net/) - Google APIs Client Library for Java (from ) - Google App Engine extensions to the Google HTTP Client Library for Java. (from ) @@ -271,6 +271,7 @@ The following software have components provided under the terms of this license: - Jackson-datatype-jdk8 (from ) - Jackson-module-JAXB-annotations (from http://wiki.fasterxml.com/JacksonJAXBAnnotations) - Jackson-module-parameter-names (from ) +- Jakarta Bean Validation API (from https://beanvalidation.org) - Java Libraries for Amazon Simple WorkFlow (from https://aws.amazon.com/sdkforjava) - Java Native Access (from https://github.com/java-native-access/jna) - Java Native Access Platform (from https://github.com/java-native-access/jna) @@ -283,20 +284,35 @@ The following software have components provided under the terms of this license: - Joda-Time (from http://www.joda.org/joda-time/) - Json Path (from https://github.com/jayway/JsonPath) - Lucene Common Analyzers (from ) +- Lucene Common Analyzers (from ) +- Lucene Core (from ) - Lucene Core (from ) - Lucene Grouping (from ) +- Lucene Grouping (from ) +- Lucene Highlighter (from ) - Lucene Highlighter (from ) - Lucene Join (from ) +- Lucene Join (from ) +- Lucene Memory (from ) - Lucene Memory (from ) - Lucene Memory (from ) +- Lucene Memory (from ) +- Lucene Miscellaneous (from ) - Lucene Miscellaneous (from ) - Lucene Queries (from ) +- Lucene Queries (from ) +- Lucene QueryParsers (from ) - Lucene QueryParsers (from ) - Lucene Sandbox (from ) +- Lucene Sandbox (from ) +- Lucene Spatial (from ) - Lucene Spatial (from ) - Lucene Spatial 3D (from ) +- Lucene Spatial 3D (from ) +- Lucene Spatial Extras (from ) - Lucene Spatial Extras (from ) - Lucene Suggest (from ) +- Lucene Suggest (from ) - MapStruct Core (from ) - Maven Artifact (from ) - Maven Artifact Manager (from ) @@ -320,8 +336,8 @@ The following software have components provided under the terms of this license: - Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java) - Mockito (from http://mockito.org) -- Mockito (from http://www.mockito.org) - Mockito (from http://mockito.org) +- Mockito (from http://www.mockito.org) - Mockito (from http://mockito.org) - Mojo's Maven plugin for Cobertura (from http://mojo.codehaus.org/cobertura-maven-plugin/) - Netty Reactive Streams Implementation (from ) @@ -340,6 +356,7 @@ The following software have components provided under the terms of this license: - Netty/Transport (from http://netty.io/) - Netty/Transport/Native/Unix/Common (from ) - Nimbus JOSE+JWT (from https://bitbucket.org/connect2id/nimbus-jose-jwt) +- Nimbus JOSE+JWT (from https://bitbucket.org/connect2id/nimbus-jose-jwt) - Nimbus LangTag (from https://bitbucket.org/connect2id/nimbus-language-tags) - Non-Blocking Reactive Foundation for the JVM (from https://github.com/reactor/reactor) - OAuth 2.0 SDK with OpenID Connect extensions (from https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions) @@ -403,6 +420,7 @@ The following software have components provided under the terms of this license: - Spring Plugin - Core (from ) - Spring Plugin - Metadata Extension (from ) - Spring Security JWT Library (from http://github.com/spring-projects/spring-security-oauth) +- Spring Security JWT Library (from http://github.com/spring-projects/spring-security-oauth) - Spring TestContext Framework (from https://github.com/spring-projects/spring-framework) - Spring Transaction (from https://github.com/spring-projects/spring-framework) - Spring Web (from https://github.com/spring-projects/spring-framework) @@ -413,6 +431,8 @@ The following software have components provided under the terms of this license: - Woodstox (from https://github.com/FasterXML/woodstox) - Xerces2-j (from https://xerces.apache.org/xerces2-j/) - aggs-matrix-stats (from https://github.com/elastic/elasticsearch) +- aggs-matrix-stats (from https://github.com/elastic/elasticsearch) +- cli (from https://github.com/elastic/elasticsearch) - cli (from https://github.com/elastic/elasticsearch) - com.google.api.grpc:grpc-google-cloud-pubsub-v1 (from https://github.com/googleapis/googleapis) - com.google.api.grpc:proto-google-cloud-logging-v2 (from https://github.com/googleapis/googleapis) @@ -423,6 +443,7 @@ The following software have components provided under the terms of this license: - compiler (from http://github.com/spullara/mustache.java) - datastore-v1-proto-client (from ) - elasticsearch-core (from https://github.com/elastic/elasticsearch) +- elasticsearch-core (from https://github.com/elastic/elasticsearch) - error-prone annotations (from ) - error-prone annotations (from ) - io.grpc:grpc-alts (from https://github.com/grpc/grpc-java) @@ -437,10 +458,8 @@ The following software have components provided under the terms of this license: - io.grpc:grpc-stub (from https://github.com/grpc/grpc-java) - ion-java (from https://github.com/amznlabs/ion-java/) - jackson-databind (from http://github.com/FasterXML/jackson) -- jackson-databind (from http://github.com/FasterXML/jackson) - java-cloudant (from https://cloudant.com) - java-cloudant (from https://cloudant.com) -- javatuples (from http://www.javatuples.org) - javax.inject (from http://code.google.com/p/atinject/) - javax.ws.rs-api (from http://jax-rs-spec.java.net) - jersey-container-servlet (from git://java.net/jersey~code/project/jersey-container-servlet) @@ -456,6 +475,7 @@ The following software have components provided under the terms of this license: - jersey-spring4 (from ) - jose4j (from https://bitbucket.org/b_c/jose4j/) - lang-mustache (from https://github.com/elastic/elasticsearch) +- lang-mustache (from https://github.com/elastic/elasticsearch) - lettuce (from http://github.com/mp911de/lettuce/wiki) - micrometer-core (from https://github.com/micrometer-metrics/micrometer) - micrometer-registry-azure-monitor (from https://github.com/micrometer-metrics/micrometer) @@ -463,6 +483,7 @@ The following software have components provided under the terms of this license: - org.xmlunit:xmlunit-core (from http://www.xmlunit.org/) - oro (from ) - parent-join (from https://github.com/elastic/elasticsearch) +- parent-join (from https://github.com/elastic/elasticsearch) - powermock-api-support (from ) - powermock-core (from http://www.powermock.org) - powermock-module-junit4 (from http://www.powermock.org) @@ -472,11 +493,15 @@ The following software have components provided under the terms of this license: - proton-j (from ) - proton-j (from ) - rank-eval (from https://github.com/elastic/elasticsearch) +- rank-eval (from https://github.com/elastic/elasticsearch) +- rest (from https://github.com/elastic/elasticsearch) - rest (from https://github.com/elastic/elasticsearch) - rest-high-level (from https://github.com/elastic/elasticsearch) +- rest-high-level (from https://github.com/elastic/elasticsearch) - rxjava (from https://github.com/ReactiveX/RxJava) - rxjava (from https://github.com/ReactiveX/RxJava) - secure-sm (from https://github.com/elastic/elasticsearch) +- secure-sm (from https://github.com/elastic/elasticsearch) - spring-security-config (from http://spring.io/spring-security) - spring-security-core (from http://spring.io/spring-security) - spring-security-oauth2-client (from http://spring.io/spring-security) @@ -497,10 +522,10 @@ The following software have components provided under the terms of this license: - swagger-models (from ) - tomcat-annotations-api (from http://tomcat.apache.org/) - tomcat-embed-core (from http://tomcat.apache.org/) -- tomcat-embed-core (from http://tomcat.apache.org/) - tomcat-embed-el (from http://tomcat.apache.org/) - tomcat-embed-websocket (from http://tomcat.apache.org/) - x-content (from https://github.com/elastic/elasticsearch) +- x-content (from https://github.com/elastic/elasticsearch) - xml-apis (from ) ======================================================================== @@ -514,6 +539,7 @@ The following software have components provided under the terms of this license: - GAX (Google Api eXtensions) (from https://github.com/googleapis) - Hamcrest Core (from http://hamcrest.org/) - Lucene Common Analyzers (from ) +- Lucene Common Analyzers (from ) - Plexus :: Default Container (from ) - Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils) - StAX (from http://stax.codehaus.org/) @@ -536,7 +562,6 @@ The following software have components provided under the terms of this license: - ASM Util (from ) - ASM library repackaged as OSGi bundle (from ) - Apache Commons Codec (from http://commons.apache.org/proper/commons-codec/) -- Apache Commons Codec (from http://commons.apache.org/proper/commons-codec/) - GAX (Google Api eXtensions) (from https://github.com/googleapis) - GAX (Google Api eXtensions) (from https://github.com/googleapis) - GAX (Google Api eXtensions) (from https://github.com/googleapis) @@ -546,9 +571,13 @@ The following software have components provided under the terms of this license: - Hamcrest library (from ) - JDOM (from http://www.jdom.org) - JSch (from http://www.jcraft.com/jsch/) +- JavaBeans Activation Framework API jar (from ) - Lucene Common Analyzers (from ) +- Lucene Common Analyzers (from ) +- Lucene Core (from ) - Lucene Core (from ) - Lucene Suggest (from ) +- Lucene Suggest (from ) - Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java) @@ -565,6 +594,7 @@ The following software have components provided under the terms of this license: - Stax2 API (from http://github.com/FasterXML/stax2-api) - ThreeTen backport (from https://www.threeten.org/threetenbp) - classworlds (from http://classworlds.codehaus.org/) +- jakarta.xml.bind-api (from ) - jaxen (from http://jaxen.codehaus.org/) - jersey-container-servlet (from git://java.net/jersey~code/project/jersey-container-servlet) - jersey-container-servlet-core (from git://java.net/jersey~code/project/jersey-container-servlet-core) @@ -651,7 +681,6 @@ The following software have components provided under the terms of this license: - jersey-ext-bean-validation (from ) - jersey-spring4 (from ) - tomcat-embed-core (from http://tomcat.apache.org/) -- tomcat-embed-core (from http://tomcat.apache.org/) ======================================================================== CPL-1.0 @@ -717,7 +746,6 @@ The following software have components provided under the terms of this license: - jersey-media-json-jackson (from git://java.net/jersey~code/project/jersey-media-json-jackson) - jersey-spring4 (from ) - tomcat-embed-core (from http://tomcat.apache.org/) -- tomcat-embed-core (from http://tomcat.apache.org/) ======================================================================== GPL-2.0-or-later @@ -756,7 +784,6 @@ The following software have components provided under the terms of this license: - jersey-media-json-jackson (from git://java.net/jersey~code/project/jersey-media-json-jackson) - jersey-spring4 (from ) - tomcat-embed-core (from http://tomcat.apache.org/) -- tomcat-embed-core (from http://tomcat.apache.org/) ======================================================================== GPL-3.0-only @@ -766,16 +793,8 @@ The following software have components provided under the terms of this license: - OSGi resource locator (from ) - Project Lombok (from https://projectlombok.org) - Project Lombok (from https://projectlombok.org) -- SnakeYAML (from http://www.snakeyaml.org) - javax.ws.rs-api (from http://jax-rs-spec.java.net) -======================================================================== -JSON -======================================================================== -The following software have components provided under the terms of this license: - -- JSON in Java (from https://github.com/douglascrockford/JSON-java) - ======================================================================== LGPL-2.1-only ======================================================================== @@ -824,13 +843,13 @@ The following software have components provided under the terms of this license: - AWS Java SDK for AWS Lambda (from https://aws.amazon.com/sdkforjava) - Animal Sniffer Annotations (from ) - Azure AD Spring Security Integration Spring Boot Starter (from https://github.com/Microsoft/azure-spring-boot) -- Azure Cosmos DB Spring Boot Starter (from https://github.com/Microsoft/azure-spring-boot) - Azure Java Client Authentication Library for AutoRest (from https://github.com/Azure/autorest-clientruntime-for-java) - Azure Java Client Runtime for AutoRest (from https://github.com/Azure/autorest-clientruntime-for-java) - Azure Metrics Spring Boot Starter (from https://github.com/Microsoft/azure-spring-boot) - Azure Spring Boot AutoConfigure (from https://github.com/Microsoft/azure-spring-boot) - Azure Spring Boot Starter (from https://github.com/Microsoft/azure-spring-boot) - Bouncy Castle Provider (from http://www.bouncycastle.org/java.html) +- Bouncy Castle Provider (from http://www.bouncycastle.org/java.html) - Checker Qual (from https://checkerframework.org) - Checker Qual (from https://checkerframework.org) - Elasticsearch: 5.0.0-alpha5 (from https://github.com/elastic/elasticsearch) @@ -840,6 +859,8 @@ The following software have components provided under the terms of this license: - Java Client Runtime for AutoRest (from https://github.com/Azure/autorest-clientruntime-for-java) - Java JWT (from http://www.jwt.io) - Lucene Core (from ) +- Lucene Core (from ) +- Lucene Sandbox (from ) - Lucene Sandbox (from ) - Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java) @@ -849,7 +870,6 @@ The following software have components provided under the terms of this license: - Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure SDK annotations (from https://github.com/Microsoft/java-api-annotations) - Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java) -- Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure SDK for Service Bus (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure client library for Blob Storage (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure client library for Identity (from https://github.com/Azure/azure-sdk-for-java) @@ -857,8 +877,8 @@ The following software have components provided under the terms of this license: - Microsoft Azure common module for Storage (from https://github.com/Azure/azure-sdk-for-java) - Mockito (from http://mockito.org) - Mockito (from http://mockito.org) -- Mockito (from http://www.mockito.org) - Mockito (from http://mockito.org) +- Mockito (from http://www.mockito.org) - Netty/Codec/HTTP (from ) - Netty/Common (from ) - Plexus :: Default Container (from ) @@ -866,9 +886,7 @@ The following software have components provided under the terms of this license: - Project Lombok (from https://projectlombok.org) - Project Lombok (from https://projectlombok.org) - SLF4J API Module (from http://www.slf4j.org) -- Spring Data for Azure Cosmos DB SQL API (from https://github.com/Microsoft/spring-data-cosmosdb) - adal4j (from https://github.com/AzureAD/azure-activedirectory-library-for-java) -- azure-documentdb (from https://azure.microsoft.com/en-us/services/cosmos-db/) - micrometer-core (from https://github.com/micrometer-metrics/micrometer) - msal4j (from https://github.com/AzureAD/microsoft-authentication-library-for-java) - spring-security-core (from http://spring.io/spring-security) @@ -890,6 +908,14 @@ The following software have components provided under the terms of this license: - Javassist (from http://www.javassist.org/) - Javassist (from http://www.javassist.org/) +======================================================================== +PHP-3.01 +======================================================================== +The following software have components provided under the terms of this license: + +- JavaBeans Activation Framework API jar (from ) +- jakarta.xml.bind-api (from ) + ======================================================================== Public-Domain ======================================================================== @@ -944,6 +970,8 @@ The following software have components provided under the terms of this license: - AWS SDK for Java - Models (from https://aws.amazon.com/sdkforjava) - Asynchronous Http Client (from ) - Bouncy Castle PKIX, CMS, EAC, TSP, PKCS, OCSP, CMP, and CRMF APIs (from http://www.bouncycastle.org/java.html) +- Bouncy Castle PKIX, CMS, EAC, TSP, PKCS, OCSP, CMP, and CRMF APIs (from http://www.bouncycastle.org/java.html) +- Bouncy Castle Provider (from http://www.bouncycastle.org/java.html) - Bouncy Castle Provider (from http://www.bouncycastle.org/java.html) - Guava: Google Core Libraries for Java (from https://github.com/google/guava.git) - Guava: Google Core Libraries for Java (from https://github.com/google/guava.git) @@ -952,11 +980,11 @@ The following software have components provided under the terms of this license: - LatencyUtils (from http://latencyutils.github.io/LatencyUtils/) - Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java) -- Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure client library for Blob Storage (from https://github.com/Azure/azure-sdk-for-java) - Project Lombok (from https://projectlombok.org) - Project Lombok (from https://projectlombok.org) - Spring Security JWT Library (from http://github.com/spring-projects/spring-security-oauth) +- Spring Security JWT Library (from http://github.com/spring-projects/spring-security-oauth) - Spring Web (from https://github.com/spring-projects/spring-framework) - reactive-streams (from http://www.reactive-streams.org/) @@ -965,11 +993,15 @@ unknown ======================================================================== The following software have components provided under the terms of this license: +- Bouncy Castle PKIX, CMS, EAC, TSP, PKCS, OCSP, CMP, and CRMF APIs (from http://www.bouncycastle.org/java.html) - Bouncy Castle PKIX, CMS, EAC, TSP, PKCS, OCSP, CMP, and CRMF APIs (from http://www.bouncycastle.org/java.html) - Bouncy Castle Provider (from http://www.bouncycastle.org/java.html) +- Bouncy Castle Provider (from http://www.bouncycastle.org/java.html) - Byte Buddy (without dependencies) (from ) +- JavaBeans Activation Framework API jar (from ) - Servlet Specification 2.5 API (from ) - System Rules (from http://stefanbirkner.github.io/system-rules/) +- jakarta.xml.bind-api (from ) - jaxen (from http://jaxen.codehaus.org/) - jersey-container-servlet (from git://java.net/jersey~code/project/jersey-container-servlet) - jersey-container-servlet-core (from git://java.net/jersey~code/project/jersey-container-servlet-core) diff --git a/devops/azure/chart/helm-config.yaml b/devops/azure/chart/helm-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf8438efec51d9e3aef1b1edd2c4ff7f4e17ca0b --- /dev/null +++ b/devops/azure/chart/helm-config.yaml @@ -0,0 +1,14 @@ +# This file contains the essential configs for the osdu on azure helm chart +global: + + # Service(s) Replica Count + replicaCount: 2 + +################################################################################ +# Specify the Gitlab branch being used for image creation +# ie: community.opengroup.org:5555/osdu/platform/system/storage/{{ .Values.global.branch }}/storage:latest +# +image: + repository: #{container-registry}#.azurecr.io + branch: #{ENVIRONMENT_NAME}# + tag: #{Build.SourceVersion}# diff --git a/devops/azure/chart/templates/deployment.yaml b/devops/azure/chart/templates/deployment.yaml index a614af30a2f95e9beb5305d2920f98446b91bbfc..f625bb35fa08968199f48c6f4e0488a0fec45450 100644 --- a/devops/azure/chart/templates/deployment.yaml +++ b/devops/azure/chart/templates/deployment.yaml @@ -63,60 +63,35 @@ spec: configMapKeyRef: name: osdu-svc-properties key: ENV_KEYVAULT - - name: AZURE_TENANT_ID - valueFrom: - configMapKeyRef: - name: osdu-svc-properties - key: ENV_TENANT_ID - name: AZURE_CLIENT_ID valueFrom: secretKeyRef: - name: clientid - key: clientid + name: active-directory + key: principal-clientid - name: AZURE_CLIENT_SECRET valueFrom: secretKeyRef: - name: clientpassword - key: clientpassword + name: active-directory + key: principal-clientpassword + - name: AZURE_TENANT_ID + valueFrom: + secretKeyRef: + name: active-directory + key: tenantid - name: aad_client_id valueFrom: secretKeyRef: - name: appid - key: appid + name: active-directory + key: application-appid - name: appinsights_key valueFrom: secretKeyRef: - name: appinsights + name: central-logging key: appinsights - - name: cosmosdb_account - valueFrom: - configMapKeyRef: - name: osdu-svc-properties - key: ENV_COSMOSDB_HOST - - name: cosmosdb_key - valueFrom: - secretKeyRef: - name: cosmos - key: cosmos - name: cosmosdb_database value: osdu-db - - name: servicebus_namespace_name - valueFrom: - configMapKeyRef: - name: osdu-svc-properties - key: ENV_SERVICEBUS_NAMESPACE - - name: servicebus_connection_string - valueFrom: - secretKeyRef: - name: servicebus - key: servicebus - name: servicebus_topic_name value: recordstopic - - name: storage_account - valueFrom: - configMapKeyRef: - name: osdu-svc-properties - key: ENV_STORAGE_ACCOUNT - name: entitlements_service_endpoint value: http://entitlements-azure/entitlements/v1 - name: entitlements_service_api_key @@ -129,5 +104,9 @@ spec: value: http://storage/api/storage/v2/query/records:batch - name: STORAGE_QUERY_RECORD_HOST value: http://storage/api/storage/v2/records + - name: partition_service_endpoint + value: http://partition/api/partition/v1 - name: azure_istioauth_enabled value: "true" + - name: azure_activedirectory_AppIdUri + value: "api://$(aad_client_id)" diff --git a/devops/azure/development-pipeline.yml b/devops/azure/development-pipeline.yml new file mode 100644 index 0000000000000000000000000000000000000000..3b09fa1b05e9ecfc936c525e58cbbc76e872c023 --- /dev/null +++ b/devops/azure/development-pipeline.yml @@ -0,0 +1,82 @@ +# Copyright © Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +trigger: + batch: true + branches: + include: + - master + paths: + exclude: + - /**/*.md + - .gitignore + - /docs + - /indexer/storage-aws + - /indexer/storage-byoc + - /indexer/storage-gcp + - /indexer/storage-ibm + +resources: + repositories: + - repository: FluxRepo + type: git + name: k8-gitops-manifests + - repository: TemplateRepo + type: git + name: infra-azure-provisioning + +variables: + - group: 'Azure - OSDU' + - group: 'Azure - OSDU Secrets' + + - name: serviceName + value: "indexer-service" + - name: chartPath + value: "devops/azure/chart" + - name: valuesFile + value: "devops/azure/chart/helm-config.yaml" + - name: 'MANIFEST_REPO' + value: $[ resources.repositories['FluxRepo'].name ] + - name: SKIP_TESTS + value: 'false' + +stages: + - template: /devops/build-stage.yml@TemplateRepo + parameters: + mavenGoal: 'package' + mavenPublishJUnitResults: true + serviceCoreMavenOptions: '-P indexer-core' + mavenOptions: '-P indexer-azure' + copyFileContents: | + pom.xml + provider/indexer-azure/maven/settings.xml + provider/indexer-azure/pom.xml + provider/indexer-azure/target/*-spring-boot.jar + copyFileContentsToFlatten: '' + mavenSettingsFile: './maven/settings.xml' + serviceBase: ${{ variables.serviceName }} + testingRootFolder: 'testing' + chartPath: ${{ variables.chartPath }} + - template: /devops/deploy-stages.yml@TemplateRepo + parameters: + serviceName: ${{ variables.serviceName }} + chartPath: ${{ variables.chartPath }} + valuesFile: ${{ variables.valuesFile }} + testCoreMavenPomFile: 'testing/indexer-test-core/pom.xml' + testCoreMavenOptions: '--settings $(System.DefaultWorkingDirectory)/drop/deploy/testing/maven/settings.xml' + skipDeploy: ${{ variables.SKIP_DEPLOY }} + skipTest: ${{ variables.SKIP_TESTS }} + providers: + - name: Azure + environments: ['dev'] diff --git a/devops/azure/pipeline.yml b/devops/azure/pipeline.yml new file mode 100644 index 0000000000000000000000000000000000000000..dbfc9fae6f271fbd28235be8a7696752796f25ad --- /dev/null +++ b/devops/azure/pipeline.yml @@ -0,0 +1,82 @@ +# Copyright © Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +trigger: + batch: true + branches: + include: + - master + paths: + exclude: + - /**/*.md + - .gitignore + - /docs + - /indexer/storage-aws + - /indexer/storage-byoc + - /indexer/storage-gcp + - /indexer/storage-ibm + +resources: + repositories: + - repository: FluxRepo + type: git + name: k8-gitops-manifests + - repository: TemplateRepo + type: git + name: infra-azure-provisioning + +variables: + - group: 'Azure - OSDU' + - group: 'Azure - OSDU Secrets' + + - name: serviceName + value: "indexer-service" + - name: chartPath + value: "devops/azure/chart" + - name: valuesFile + value: "devops/azure/chart/helm-config.yaml" + - name: 'MANIFEST_REPO' + value: $[ resources.repositories['FluxRepo'].name ] + - name: SKIP_TESTS + value: 'false' + +stages: + - template: /devops/build-stage.yml@TemplateRepo + parameters: + mavenGoal: 'package' + mavenPublishJUnitResults: true + serviceCoreMavenOptions: '-P indexer-core' + mavenOptions: '-P indexer-azure' + copyFileContents: | + pom.xml + provider/indexer-azure/maven/settings.xml + provider/indexer-azure/pom.xml + provider/indexer-azure/target/*-spring-boot.jar + copyFileContentsToFlatten: '' + mavenSettingsFile: './maven/settings.xml' + serviceBase: ${{ variables.serviceName }} + testingRootFolder: 'testing' + chartPath: ${{ variables.chartPath }} + - template: /devops/deploy-stages.yml@TemplateRepo + parameters: + serviceName: ${{ variables.serviceName }} + chartPath: ${{ variables.chartPath }} + valuesFile: ${{ variables.valuesFile }} + testCoreMavenPomFile: 'testing/indexer-test-core/pom.xml' + testCoreMavenOptions: '--settings $(System.DefaultWorkingDirectory)/drop/deploy/testing/maven/settings.xml' + skipDeploy: ${{ variables.SKIP_DEPLOY }} + skipTest: ${{ variables.SKIP_TESTS }} + providers: + - name: Azure + environments: ['demo'] diff --git a/provider/indexer-azure/README.md b/provider/indexer-azure/README.md index 285c7bc4683511773d149deb9612aa6303863144..6772abca8ff69d7dc5f90e8456805f5ee709a734 100644 --- a/provider/indexer-azure/README.md +++ b/provider/indexer-azure/README.md @@ -44,18 +44,17 @@ az keyvault secret show --vault-name $KEY_VAULT_NAME --name $KEY_VAULT_SECRET_NA | `STORAGE_SCHEMA_HOST` | `${storage_service_url}/schemas` | Endpoint of schema API | no | - | | `STORAGE_QUERY_RECORD_HOST` | `${storage_service_url}/query/records` | Endpoint of records API | no | - | | `STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST` | `${storage_service_url}/query/records:batch` | Endpoint of records batch API | no | - | -| `servicebus_namespace_name` | ex `foo-sb-namespace` | . | . | . | | `KEYVAULT_URI` | ex `https://foo-kv.vault.azure.net/` | . | . | . | | `appinsights_key` | `********` | App Insights key | yes | output of infrastructure deployments | | `aad_client_id` | `********` | AAD client application ID | yes | output of infrastructure deployment | -| `cosmosdb_account` | ex `devintosdur2cosmosacct` | Cosmos account name | no | output of infrastructure deployment | -| `cosmosdb_key` | `********` | Key for CosmosDB | yes | output of infrastructure deployments | | `cosmosdb_database` | ex `dev-osdu-r2-db` | Cosmos database for documents | no | output of infrastructure deployment | | `servicebus_topic_name` | `recordstopic` | Service Bus topic name | no | output of infrastructure deployments | | `entitlements_service_endpoint` | ex `https://entitlements.azurewebsites.net` | Entitlements service endpoint | no | Service Bus topic name | | `AZURE_CLIENT_ID` | `********` | Identity to run the service locally. This enables access to Azure resources. You only need this if running locally | yes | keyvault secret: `$KEYVAULT_URI/secrets/app-dev-sp-username` | | `AZURE_TENANT_ID` | `********` | AD tenant to authenticate users from | yes | keyvault secret: `$KEYVAULT_URI/secrets/app-dev-sp-tenant-id` | | `AZURE_CLIENT_SECRET` | `********` | Secret for `$AZURE_CLIENT_ID` | yes | keyvault secret: `$KEYVAULT_URI/secrets/app-dev-sp-password` | +| `partition_service_endpoint` | ex `https://foo-partition.azurewebsites.net` | Partition Service API endpoint | no | output of infrastructure deployment | +| `azure.activedirectory.app-resource-id` | `********` | AAD client application ID | yes | output of infrastructure deployment | | `azure_istioauth_enabled` | `true` | Flag to Disable AAD auth | no | -- | **Required to run integration tests** @@ -90,25 +89,6 @@ Java version: 1.8.0_212, vendor: AdoptOpenJDK, runtime: /usr/lib/jvm/jdk8u212-b0 ... ``` -You may need to configure access to the remote maven repository that holds the OSDU dependencies. This file should live within `~/.m2/settings.xml`: -```bash -$ cat ~/.m2/settings.xml -<?xml version="1.0" encoding="UTF-8"?> -<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd"> - <servers> - <server> - <id>os-core</id> - <username>mvn-pat</username> - <!-- Treat this auth token like a password. Do not share it with anyone, including Microsoft support. --> - <!-- The generated token expires on or before 11/14/2019 --> - <password>$PERSONAL_ACCESS_TOKEN_GOES_HERE</password> - </server> - </servers> -</settings> -``` - ### Build and run the application After configuring your environment as specified above, you can follow these steps to build and run the application. These steps should be invoked from the *repository root.* diff --git a/provider/indexer-azure/pom.xml b/provider/indexer-azure/pom.xml index 343a293f710199d880e902d304534f1c2c7f8a64..7f2f938216de5bd65c0970ab648001237ca866a2 100644 --- a/provider/indexer-azure/pom.xml +++ b/provider/indexer-azure/pom.xml @@ -139,20 +139,15 @@ <artifactId>spring-boot-configuration-processor</artifactId> <optional>true</optional> </dependency> - <dependency> - <groupId>com.microsoft.azure</groupId> - <artifactId>azure-cosmosdb-spring-boot-starter</artifactId> - <version>${azure.version}</version> - </dependency> <dependency> <groupId>org.opengroup.osdu</groupId> <artifactId>os-core-common</artifactId> + <version>0.3.12</version> </dependency> - <dependency> <groupId>org.opengroup.osdu</groupId> <artifactId>core-lib-azure</artifactId> - <version>0.0.28</version> + <version>0.0.33</version> </dependency> <dependency> diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/config/AzureBootstrapConfig.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/config/AzureBootstrapConfig.java index 918869920c0fb6b47b56132441724a5539483ca3..09f69ecde0f5d292026c9220ae1c7b89adc184fe 100644 --- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/config/AzureBootstrapConfig.java +++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/config/AzureBootstrapConfig.java @@ -25,18 +25,12 @@ import javax.inject.Named; @Component public class AzureBootstrapConfig { - @Value("${aad.oboApi}") - private String aadOboAPI; - @Value("${azure.keyvault.url}") private String keyVaultURL; @Value("${azure.servicebus.topic-name}") private String serviceBusTopicName; - @Value("${azure.servicebus.namespace-name}") - private String serviceBusNamespaceName; - @Value("${ELASTIC_CACHE_EXPIRATION}") private Integer elasticCacheExpiration; @@ -45,16 +39,10 @@ public class AzureBootstrapConfig { @Bean @Named("KEY_VAULT_URL") - public String getKeyVaultURL(){ + public String getKeyVaultURL() { return keyVaultURL; } - @Bean - @Named("SERVICE_BUS_NAMESPACE") - public String serviceBusNamespaceName() { - return serviceBusNamespaceName; - } - @Bean @Named("SERVICE_BUS_TOPIC") public String serviceBusTopicName() { @@ -64,22 +52,13 @@ public class AzureBootstrapConfig { @Bean @Named("ELASTIC_CACHE_EXPIRATION") public Integer getElasticCacheExpiration() { - return elasticCacheExpiration; + return elasticCacheExpiration; } @Bean @Named("MAX_CACHE_VALUE_SIZE") public Integer getMaxCacheValueSize() { - return maxCacheValueSize; - } - - private String authority; - private String secretKey; - - @Bean - @Named("AAD_OBO_API") - public String aadClientID() { - return aadOboAPI; + return maxCacheValueSize; } @Bean diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/di/TenantFactoryImpl.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/di/TenantFactoryImpl.java deleted file mode 100644 index df488c799062c5590d025ee7913cf15ef9334551..0000000000000000000000000000000000000000 --- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/di/TenantFactoryImpl.java +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright © Microsoft Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.opengroup.osdu.indexer.azure.di; - -import org.opengroup.osdu.core.common.cache.ICache; -import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory; -import org.opengroup.osdu.core.common.model.tenant.TenantInfo; -import org.springframework.stereotype.Component; -import org.springframework.web.context.annotation.RequestScope; - -import javax.inject.Inject; -import java.util.*; - -@Component -@RequestScope -public class TenantFactoryImpl implements ITenantFactory { - @Inject - private CosmosDBTenantInfo db; - - private Map<String, TenantInfo> tenants; - - public boolean exists(String tenantName) - { - if (this.tenants == null) - initTenants(); - return this.tenants.containsKey(tenantName); - } - - public TenantInfo getTenantInfo(String tenantName) { - if (this.tenants == null) - initTenants(); - return this.tenants.get(tenantName); - } - - public Collection<TenantInfo> listTenantInfo() { - if (this.tenants == null) - initTenants(); - return this.tenants.values(); - } - - public <V> ICache<String, V> createCache(String tenantName, String host, int port, int expireTimeSeconds, Class<V> classOfV) - { - return null; - } - - public void flushCache() {} - - private void initTenants() { - this.tenants = new HashMap<>(); - - db.findAll().forEach(doc -> { - TenantInfo ti = new TenantInfo(); - String tenantName = doc.getId(); - ti.setName(tenantName); - //set serviceprincipalAppId in Azure side instead of ServiceAccount in Gcp - ti.setServiceAccount(doc.getServiceprincipalAppId()); - this.tenants.put(tenantName, ti) ; - }); - } -} diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/di/TenantInfoDoc.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/di/TenantInfoDoc.java deleted file mode 100644 index 58d3ee84ee3e030cef2c971ae3ad565cfaff3f52..0000000000000000000000000000000000000000 --- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/di/TenantInfoDoc.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright © Microsoft Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.opengroup.osdu.indexer.azure.di; - -import com.microsoft.azure.spring.data.cosmosdb.core.mapping.Document; -import com.microsoft.azure.spring.data.cosmosdb.core.mapping.PartitionKey; -import com.microsoft.azure.spring.data.cosmosdb.repository.DocumentDbRepository; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; -import org.springframework.data.annotation.Id; - -@Data -@AllArgsConstructor -@NoArgsConstructor -@Document(collection = "TenantInfo") //collection name -public class TenantInfoDoc { - @PartitionKey - @Id - private String id; - private String serviceprincipalAppId; -} - -interface CosmosDBTenantInfo extends DocumentDbRepository<TenantInfoDoc, String> {} diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/publish/PublisherImpl.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/publish/PublisherImpl.java index 0aa05f37c5851f02b1078bcd203828b6599f57c3..7c5d2f854b68c25ea71a6e56a6f91e062491f2e6 100644 --- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/publish/PublisherImpl.java +++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/publish/PublisherImpl.java @@ -21,7 +21,7 @@ import com.google.gson.GsonBuilder; import com.google.gson.JsonElement; import com.microsoft.azure.servicebus.Message; -import com.microsoft.azure.servicebus.TopicClient; +import org.opengroup.osdu.azure.servicebus.ITopicClientFactory; import org.elasticsearch.common.Strings; import org.opengroup.osdu.core.common.model.http.DpsHeaders; @@ -34,6 +34,7 @@ import org.springframework.stereotype.Component; import org.springframework.web.context.annotation.RequestScope; import javax.inject.Inject; +import javax.inject.Named; import java.lang.reflect.Type; import java.nio.charset.StandardCharsets; import java.util.HashMap; @@ -46,11 +47,16 @@ public class PublisherImpl implements IPublisher { @Inject - private TopicClient topicClient; + private ITopicClientFactory topicClientFactory; @Inject private JaxRsDpsLog logger; + @Inject + @Named("SERVICE_BUS_TOPIC") + private String serviceBusTopic; + + @Override public void publishStatusChangedTagsToTopic(DpsHeaders headers, JobStatus indexerBatchStatus) throws Exception { @@ -68,7 +74,7 @@ public class PublisherImpl implements IPublisher { try { logger.info("Indexer publishes message " + headers.getCorrelationId()); - topicClient.send(message); + topicClientFactory.getClient(headers.getPartitionId(), serviceBusTopic).send(message); } catch (Exception e) { diff --git a/provider/indexer-azure/src/main/resources/application.properties b/provider/indexer-azure/src/main/resources/application.properties index 242afa5445dec22e2d10e79c3d48e1e9fbe70637..f6cb597dbcc58293ffd9897faadace106a77c4f1 100644 --- a/provider/indexer-azure/src/main/resources/application.properties +++ b/provider/indexer-azure/src/main/resources/application.properties @@ -23,7 +23,6 @@ AUTHORIZE_API=${entitlements_service_endpoint} AUTHORIZE_API_KEY=${entitlements_service_api_key} LEGALTAG_API=${legal_service_endpoint} - SCHEMA_CACHE_EXPIRATION=60 INDEX_CACHE_EXPIRATION=60 ELASTIC_CACHE_EXPIRATION=1440 @@ -43,6 +42,10 @@ STORAGE_RECORDS_BATCH_SIZE=20 INDEXER_QUEUE_HOST=http://127.0.0.1:9000 +# Partition Service configuration +PARTITION_API=${partition_service_endpoint} +azure.activedirectory.app-resource-id=${aad_client_id} + #AzureADconfiguration, commented below settings to disable AAD AuthN , #Uncomment it In the Istio AUTHN disabled Scenario #azure.activedirectory.session-stateless=true @@ -51,12 +54,9 @@ INDEXER_QUEUE_HOST=http://127.0.0.1:9000 # Istio Auth Enabled azure.istio.auth.enabled=${azure_istioauth_enabled} -azure.cosmosdb.uri=${cosmosdb_account} -azure.cosmosdb.key=${cosmosdb_key} azure.cosmosdb.database=${cosmosdb_database} #AzureServiceBusconfiguration -azure.servicebus.namespace-name=${servicebus_namespace_name} azure.servicebus.topic-name=${servicebus_topic_name} #Indexer-Queue-header @@ -69,8 +69,10 @@ azure.application-insights.instrumentation-key=${appinsights_key} spring.application.name=indexer-azure DEPLOYMENT_ENVIRONMENT=CLOUD -aad.oboApi=${aad_client_id} logging.transaction.enabled=true logging.slf4jlogger.enabled=true -logging.mdccontext.enabled=true \ No newline at end of file +logging.mdccontext.enabled=true + +# core-lib-azure configuration +tenantFactoryImpl.required=true diff --git a/provider/indexer-gcp/src/main/appengine/app.yaml b/provider/indexer-gcp/src/main/appengine/app.yaml index 363b4097e3a611b59391832910b92ca518800872..d33c333d6b94b1273dba8a1bee2163fb477ea319 100644 --- a/provider/indexer-gcp/src/main/appengine/app.yaml +++ b/provider/indexer-gcp/src/main/appengine/app.yaml @@ -39,4 +39,4 @@ env_variables: LEGAL_HOSTNAME: "LEGAL_HOSTNAME_VAR" REGION: "REGION_VAR" SPRING_PROFILES_ACTIVE: 'ENVIRONMENT' - SECURITY_HTTPS_CERTIFICATE_TRUST: 'SECURITY_HTTPS_CERTIFICATE_TRUST_VAR' + SECURITY_HTTPS_CERTIFICATE_TRUST: 'SECURITY_HTTPS_CERTIFICATE_TRUST_VAR' \ No newline at end of file diff --git a/provider/indexer-ibm/pom.xml b/provider/indexer-ibm/pom.xml index 67fab6b3ef896db60dd9a288b8d8057338eabd10..ae6917c0921fd6311ee88561b8e7048980e46504 100644 --- a/provider/indexer-ibm/pom.xml +++ b/provider/indexer-ibm/pom.xml @@ -55,7 +55,7 @@ <dependency> <groupId>org.opengroup.osdu.indexer</groupId> <artifactId>indexer-core</artifactId> - <version>1.0.5-SNAPSHOT</version> + <version>1.0.6-SNAPSHOT</version> </dependency> <dependency> diff --git a/testing/indexer-test-azure/pom.xml b/testing/indexer-test-azure/pom.xml index 26ee0981f960dfb7d5c787b3e205bed7da151abd..47465e7b58a47e6bada176baede2229227e3b6e5 100644 --- a/testing/indexer-test-azure/pom.xml +++ b/testing/indexer-test-azure/pom.xml @@ -63,7 +63,7 @@ <dependency> <groupId>org.opengroup.osdu</groupId> <artifactId>core-lib-azure</artifactId> - <version>0.0.9</version> + <version>0.0.33</version> </dependency> <dependency> @@ -181,7 +181,7 @@ <dependency> <groupId>org.opengroup.osdu</groupId> <artifactId>core-lib-azure</artifactId> - <version>0.0.28</version> + <version>0.0.33</version> <scope>test</scope> <exclusions> <exclusion>