Merge branch 'master' of https://community.opengroup.org/osdu/platform/system/schema-service

NOTICE

@@ -435,11 +435,13 @@ The following software have components provided under the terms of this license:
 - javatuples (from http://www.javatuples.org)
 - javax.inject (from http://code.google.com/p/atinject/)
 - jose4j (from https://bitbucket.org/b_c/jose4j/)
+- kotlin-stdlib (from )
 - lang-mustache (from https://github.com/elastic/elasticsearch)
 - lettuce (from http://github.com/mp911de/lettuce/wiki)
 - micrometer-core (from https://github.com/micrometer-metrics/micrometer)
 - micrometer-registry-azure-monitor (from https://github.com/micrometer-metrics/micrometer)
 - org.apiguardian:apiguardian-api (from https://github.com/apiguardian-team/apiguardian)
+- org.jetbrains.kotlin:kotlin-stdlib-common (from https://kotlinlang.org/)
 - org.opentest4j:opentest4j (from https://github.com/ota4j-team/opentest4j)
 - org.xmlunit:xmlunit-core (from http://www.xmlunit.org/)
 - parent-join (from https://github.com/elastic/elasticsearch)

devops/azure/chart/helm-config.yml

@@ -26,3 +26,5 @@ image:
   repository: #{container-registry}#.azurecr.io
   branch: #{ENVIRONMENT_NAME}#
   tag: #{Build.SourceVersion}#
+
+default_tenant: #{DEFAULT_TENANT}#

devops/azure/chart/task/bootstrap.yml

+parameters:
+  providers: []
+
+stages:
+
+- ${{ each provider in parameters.providers }}:
+  - ${{ each environment in provider.environments }}: 
+    - stage: 'Bootstrap_Schemas_${{ provider.name }}_${{ environment }}'
+      displayName: 'Bootstrap_Schemas_${{ provider.name }}_${{ environment }}'
+      jobs:   
+        - job: 'Bootstrap_shared_schemas_job_${{ environment }}'
+          displayName: 'Bootstrap_shared_schemas_job_${{ environment }}'
+          variables:
+          - group: '${{ provider.name }} Target Env Secrets - ${{ environment }}'
+          - group: '${{ provider.name }} Target Env - ${{ environment }}'
+          - group: '${{ provider.name }} Service Release - schema-service'
+          steps:
+            - task: UsePythonVersion@0
+              inputs:
+                versionSpec: '3.x'
+                addToPath: true
+                architecture: 'x64'
+            
+            - task: Bash@3
+              displayName: 'Deploy Shared Schemas'
+              inputs:
+                targetType: 'inline'
+                script: |
+                  pip install -r deployments/scripts/azure/requirements.txt
+
+                  export AZURE_AD_TENANT_ID=$(AZURE_TENANT_ID)
+                  export AZURE_AD_APP_RESOURCE_ID=$(AZURE_AD_APP_RESOURCE_ID)
+                  export INTEGRATION_TESTER=$(INTEGRATION_TESTER)
+                  export TESTER_SERVICEPRINCIPAL_SECRET=$(AZURE_TESTER_SERVICEPRINCIPAL_SECRET)
+                  
+                  export DATA_PARTITION=$(DEFAULT_TENANT)
+                  export DEPLOY_SCHEMAS_URL=$(DEPLOY_SCHEMAS_URL)
+                  export APP_KEY=""
+                  
+                  AZURE_SP_TOKEN=$(python deployments/scripts/azure/Token.py)
+                  export BEARER_TOKEN=$AZURE_SP_TOKEN
+
+                  echo $DATA_PARTITION
+                  echo $DEPLOY_SCHEMAS_URL
+                  
+                  python deployments/scripts/DeploySharedSchemas.py -u $DEPLOY_SCHEMAS_URL

devops/azure/chart/task/deploy.yml

+parameters:
+  environments: []
+  serviceName: 'schema-service'
+  skipTests: 'false'
+  skipDeploy: 'false'
+  valuesFile: ''
+  chartPath: ''
+
+stages:
+  - template: /devops/build-stage.yml@TemplateRepo
+    parameters:
+      mavenGoal: 'package'
+      mavenPublishJUnitResults: true
+      serviceCoreMavenOptions: '--projects schema-core -Dmaven.test.skip=true'
+      mavenOptions: '--projects provider/schema-azure -Dmaven.test.skip=true'
+      copyFileContents: |
+        pom.xml
+        provider/schema-azure/maven/settings.xml
+        provider/schema-azure/pom.xml
+        provider/schema-azure/target/*-spring-boot.jar
+      copyFileContentsToFlatten: ''
+      mavenSettingsFile: './maven/settings.xml'
+      serviceBase: ${{ parameters.serviceName }}
+      testingRootFolder: 'testing'
+      chartPath: ${{ parameters.chartPath }}
+
+  # Deploy for each environment
+  - ${{ each environment in parameters.environments }}:  
+
+      - template: /devops/deploy-stages.yml@TemplateRepo
+        parameters:
+          serviceName: ${{ parameters.serviceName }}
+          chartPath: ${{ parameters.chartPath }}
+          valuesFile: ${{ parameters.valuesFile }}
+          skipDeploy: ${{ parameters.skipDeploy }}
+          skipTest: ${{ parameters.skipTests }}
+          providers:
+            -  name: Azure
+               environments:
+               - ${{ environment.name }}
+                     
+      - template: bootstrap.yml
+        parameters:
+          providers:
+          - name: Azure
+            environments:
+            - ${{ environment.name }}
\ No newline at end of file

devops/azure/chart/templates/deployment.yaml

+
 #  Copyright © Microsoft Corporation
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
@@ -98,4 +99,12 @@ spec:
             - name: entitlements_service_endpoint
               value: http://entitlements-azure/entitlements/v1
             - name: entitlements_service_api_key
-              value: "OBSOLETE"
\ No newline at end of file
+              value: "OBSOLETE"
+            - name: partition_service_endpoint
+              value: "http://partition/api/partition/v1"
+            - name: shared_partition
+              value: {{ .Values.default_tenant}}
+            - name: azure_istioauth_enabled
+              value: "true"
+
+

devops/azure/chart/values.yaml

@@ -11,7 +11,7 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
-    
+
 global:
   replicaCount: 1
 
@@ -19,4 +19,5 @@ image:
   repository: community.opengroup.org:5555/osdu/platform/system/schema-service
   branch: master
   tag: latest
-    
+
+default_tenant: opendes

devops/azure/pipeline.yml

@@ -11,12 +11,12 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
-
 trigger:
   batch: true
   branches:
     include:
       - master
+
   paths:
     exclude:
       - /**/*.md
@@ -51,6 +51,8 @@ variables:
     value: $(Pipeline.Workspace)/.m2/repository
   - name: SKIP_TESTS
     value: 'false'
+  - name: SKIP_DEPLOY
+    value: 'false'
 
 stages:
   - template: /devops/build-stage.yml@TemplateRepo

provider/schema-aws/src/main/java/org/opengroup/osdu/schema/provider/aws/security/WebSecurity.java

+package org.opengroup.osdu.schema.provider.aws.security;
+
+import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity;
+import org.springframework.security.config.annotation.web.builders.HttpSecurity;
+import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
+import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter;
+
+@EnableWebSecurity
+@EnableGlobalMethodSecurity(prePostEnabled = true)
+public class WebSecurity extends WebSecurityConfigurerAdapter {
+    @Override
+    protected void configure(HttpSecurity http) throws Exception {
+        http.httpBasic().disable().csrf().disable(); // disable default authN. AuthN handled by endpoints proxy
+    }
+}

provider/schema-azure/README.md

@@ -38,18 +38,21 @@ az keyvault secret show --vault-name $KEY_VAULT_NAME --name $KEY_VAULT_SECRET_NA
 | `LOG_PREFIX` | `schema` | Logging prefix | no | - |
 | `AUTHORIZE_API` | ex `https://foo-entitlements.azurewebsites.net` | Entitlements API endpoint | no | output of infrastructure deployment |
 | `AUTHORIZE_API_KEY` | `********` | The API key clients will need to use when calling the entitlements | yes | -- |
+| `partition_service_endpoint` |  ex `https://foo-partition.azurewebsites.net` | Partition Service API endpoint | no | output of infrastructure deployment |
+| `azure.activedirectory.app-resource-id` | `********` | AAD client application ID  | yes | output of infrastructure deployment |
 | `azure.application-insights.instrumentation-key` | `********` | API Key for App Insights | yes | output of infrastructure deployment |
 | `azure.activedirectory.client-id` | `********` | AAD client application ID | yes | output of infrastructure deployment |
 | `azure.activedirectory.AppIdUri` | `api://${azure.activedirectory.client-id}` | URI for AAD Application | no | -- |
 | `azure.activedirectory.session-stateless` | `true` | Flag run in stateless mode (needed by AAD dependency) | no | -- |
-| `cosmosdb_account` | ex `devintosdur2cosmosacct` | Cosmos account name | no | output of infrastructure deployment |
-| `cosmosdb_database` | ex `dev-osdu-r2-db` | Cosmos database for storage documents | no | output of infrastructure deployment |
 | `azure.storage.account-name` | ex `foo-storage-account` | Storage account for storing documents | no | output of infrastructure deployment |
 | `azure.storage.enable-https` | `true` | Used by spring boot starter library | no | - |
 | `KEYVAULT_URI` | ex `https://foo-keyvault.vault.azure.net/` | URI of KeyVault that holds application secrets | no | output of infrastructure deployment |
 | `AZURE_CLIENT_ID` | `********` | Identity to run the service locally. This enables access to Azure resources. You only need this if running locally | yes | keyvault secret: `$KEYVAULT_URI/secrets/app-dev-sp-username` |
 | `AZURE_TENANT_ID` | `********` | AD tenant to authenticate users from | yes | keyvault secret: `$KEYVAULT_URI/secrets/app-dev-sp-tenant-id` |
 | `AZURE_CLIENT_SECRET` | `********` | Secret for `$AZURE_CLIENT_ID` | yes | keyvault secret: `$KEYVAULT_URI/secrets/app-dev-sp-password` |
+| `partition_service_endpoint` | ex `https//foo-partition.azurewebsites.net/api/partition/v1` | Partition API endpoint | no | output of infrastructure deployment |
+| `azure_istioauth_enabled` | `true` | Flag to Disable AAD auth | no | -- |
+| `shared_partition` | `opendes` | Default Partition for Public Shared Schemas | no | -- |
 | `server.port` | ex `8085` | port for schema service | no | -- |
 
 

provider/schema-azure/pom.xml

@@ -15,140 +15,146 @@
   ~ limitations under the License.
   -->
 <project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <parent>
-        <artifactId>os-schema</artifactId>
-        <groupId>org.opengroup.osdu</groupId>
-        <version>0.0.1</version>
-        <relativePath>../../pom.xml</relativePath>
-    </parent>
-    <modelVersion>4.0.0</modelVersion>
-
-    <artifactId>os-schema-azure</artifactId>
-    <version>0.0.1-SNAPSHOT</version>
-    <description>Azure related implementation staff.</description>
-    <packaging>jar</packaging>
-
-    <properties>
-        <osdu.corelibazure.version>0.0.40</osdu.corelibazure.version>
-        <osdu.oscorecommon.version>0.3.12</osdu.oscorecommon.version>
-        <osdu.os-schema-core.version>0.0.1</osdu.os-schema-core.version>
-        <mockito.version>1.10.19</mockito.version>
-        <cucumber.version>5.4.0</cucumber.version>
-    </properties>
-
-    <dependencyManagement>
-        <dependencies>
-            <!-- Inherit managed dependencies from core-lib-azure -->
-            <dependency>
-                <groupId>org.opengroup.osdu</groupId>
-                <artifactId>core-lib-azure</artifactId>
-                <version>${osdu.corelibazure.version}</version>
-                <type>pom</type>
-                <scope>import</scope>
-            </dependency>
-        </dependencies>
-    </dependencyManagement>
-
-    <dependencies>
-        <dependency>
-            <groupId>com.microsoft.azure</groupId>
-            <artifactId>azure-active-directory-spring-boot-starter</artifactId>
-            <exclusions>
-                <exclusion>
-                    <groupId>org.springframework.boot</groupId>
-                    <artifactId>spring-boot-starter-logging</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
-
-        <dependency>
-            <groupId>org.opengroup.osdu</groupId>
-            <artifactId>core-lib-azure</artifactId>
-            <version>${osdu.corelibazure.version}</version>
-        </dependency>
-
-        <dependency>
-            <groupId>org.opengroup.osdu</groupId>
-            <artifactId>os-core-common</artifactId>
-            <version>${osdu.oscorecommon.version}</version>
-        </dependency>
-
-        <dependency>
-            <groupId>org.opengroup.osdu</groupId>
-            <artifactId>os-schema-core</artifactId>
-            <version>${osdu.os-schema-core.version}</version>
-        </dependency>
-
-        <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-starter-test</artifactId>
-            <exclusions>
-                <exclusion>
-                    <groupId>org.springframework.boot</groupId>
-                    <artifactId>spring-boot-starter-logging</artifactId>
-                </exclusion>
-            </exclusions>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.mockito</groupId>
-            <artifactId>mockito-all</artifactId>
-            <version>${mockito.version}</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>io.cucumber</groupId>
-            <artifactId>cucumber-java8</artifactId>
-            <version>${cucumber.version}</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>io.cucumber</groupId>
-            <artifactId>cucumber-junit</artifactId>
-            <version>${cucumber.version}</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>io.cucumber</groupId>
-            <artifactId>cucumber-guice</artifactId>
-            <version>${cucumber.version}</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework</groupId>
-            <artifactId>spring-beans</artifactId>
-        </dependency>
-
-    </dependencies>
-
-    <build>
-        <plugins>
-            <plugin>
-                <groupId>org.springframework.boot</groupId>
-                <artifactId>spring-boot-maven-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <goals>
-                            <goal>repackage</goal>
-                        </goals>
-                        <configuration>
-                            <classifier>spring-boot</classifier>
-                            <mainClass>org.opengroup.osdu.schema.azure.SchemaApplication</mainClass>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<parent>
+		<artifactId>os-schema</artifactId>
+		<groupId>org.opengroup.osdu</groupId>
+		<version>0.0.1</version>
+		<relativePath>../../pom.xml</relativePath>
+	</parent>
+	<modelVersion>4.0.0</modelVersion>
+
+	<artifactId>os-schema-azure</artifactId>
+	<version>0.0.1-SNAPSHOT</version>
+	<description>Azure related implementation staff.</description>
+	<packaging>jar</packaging>
+
+	<properties>
+		<osdu.corelibazure.version>0.0.40</osdu.corelibazure.version>
+		<osdu.oscorecommon.version>0.3.18</osdu.oscorecommon.version>
+		<osdu.os-schema-core.version>0.0.1</osdu.os-schema-core.version>
+		<mockito.version>1.10.19</mockito.version>
+		<cucumber.version>5.4.0</cucumber.version>
+	</properties>
+
+	<dependencyManagement>
+		<dependencies>
+			<!-- Inherit managed dependencies from core-lib-azure -->
+			<dependency>
+				<groupId>org.opengroup.osdu</groupId>
+				<artifactId>core-lib-azure</artifactId>
+				<version>${osdu.corelibazure.version}</version>
+				<type>pom</type>
+				<scope>import</scope>
+			</dependency>
+		</dependencies>
+	</dependencyManagement>
+
+	<dependencies>
+		<dependency>
+			<groupId>com.microsoft.azure</groupId>
+			<artifactId>azure-active-directory-spring-boot-starter</artifactId>
+			<exclusions>
+				<exclusion>
+					<groupId>org.springframework.boot</groupId>
+					<artifactId>spring-boot-starter-logging</artifactId>
+				</exclusion>
+			</exclusions>
+		</dependency>
+
+		<dependency>
+			<groupId>org.opengroup.osdu</groupId>
+			<artifactId>core-lib-azure</artifactId>
+			<version>${osdu.corelibazure.version}</version>
+		</dependency>
+
+		<dependency>
+			<groupId>org.opengroup.osdu</groupId>
+			<artifactId>os-core-common</artifactId>
+			<version>${osdu.oscorecommon.version}</version>
+		</dependency>
+
+		<dependency>
+			<groupId>org.opengroup.osdu</groupId>
+			<artifactId>os-schema-core</artifactId>
+			<version>${osdu.os-schema-core.version}</version>
+		</dependency>
+
+		<dependency>
+			<groupId>org.springframework.boot</groupId>
+			<artifactId>spring-boot-starter-test</artifactId>
+			<exclusions>
+				<exclusion>
+					<groupId>org.springframework.boot</groupId>
+					<artifactId>spring-boot-starter-logging</artifactId>
+				</exclusion>
+			</exclusions>
+			<scope>test</scope>
+		</dependency>
+
+		<dependency>
+			<groupId>org.mockito</groupId>
+			<artifactId>mockito-all</artifactId>
+			<version>${mockito.version}</version>
+			<scope>test</scope>
+		</dependency>
+
+		<dependency>
+			<groupId>io.cucumber</groupId>
+			<artifactId>cucumber-java8</artifactId>
+			<version>${cucumber.version}</version>
+			<scope>test</scope>
+		</dependency>
+
+		<dependency>
+			<groupId>io.cucumber</groupId>
+			<artifactId>cucumber-junit</artifactId>
+			<version>${cucumber.version}</version>
+			<scope>test</scope>
+		</dependency>
+
+		<dependency>
+			<groupId>io.cucumber</groupId>
+			<artifactId>cucumber-guice</artifactId>
+			<version>${cucumber.version}</version>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework.boot</groupId>
+			<artifactId>spring-boot</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework</groupId>
+			<artifactId>spring-beans</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>com.squareup.okio</groupId>
+			<artifactId>okio</artifactId>
+			<version>2.2.2</version>
+		</dependency>
+
+
+	</dependencies>
+
+	<build>
+		<plugins>
+			<plugin>
+				<groupId>org.springframework.boot</groupId>
+				<artifactId>spring-boot-maven-plugin</artifactId>
+				<executions>
+					<execution>
+						<goals>
+							<goal>repackage</goal>
+						</goals>
+						<configuration>
+							<classifier>spring-boot</classifier>
+							<mainClass>org.opengroup.osdu.schema.azure.SchemaApplication</mainClass>
+						</configuration>
+					</execution>
+				</executions>
+			</plugin>
+		</plugins>
+	</build>
 
 </project>
\ No newline at end of file

provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/SchemaApplication.java

@@ -22,7 +22,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.context.annotation.ComponentScan;
 
 @SpringBootApplication(exclude = {
-        org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration.class })
+        org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration.class})
 @ComponentScan({ "org.opengroup" })
 public class SchemaApplication {
     public static void main(String[] args)

provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/definitions/AuthorityDoc.java

@@ -24,6 +24,5 @@ import org.opengroup.osdu.schema.model.Authority;
 @NoArgsConstructor
 public class AuthorityDoc {
     private String id;
-    private String dataPartitionId;
     private Authority authority;
 }

provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/definitions/EntityTypeDoc.java

@@ -24,6 +24,5 @@ import org.opengroup.osdu.schema.model.EntityType;
 @NoArgsConstructor
 public class EntityTypeDoc {
     private String id;
-    private String dataPartitionId;
     private EntityType entityType;
 }

provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/definitions/SchemaInfoDoc.java

@@ -23,6 +23,6 @@ import lombok.NoArgsConstructor;
 @NoArgsConstructor
 public class SchemaInfoDoc {
     private String id;
-    private String dataPartitionId;
+    private String partitionId;
     private FlattenedSchemaInfo flattenedSchemaInfo;
 }

provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/definitions/SourceDoc.java

@@ -24,6 +24,5 @@ import org.opengroup.osdu.schema.model.Source;
 @NoArgsConstructor
 public class SourceDoc {
     private String id;
-    private String dataPartitionId;
     private Source source;
 }

provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/di/AzureBootstrapConfig.java

@@ -14,70 +14,28 @@
 
 package org.opengroup.osdu.schema.azure.di;
 
-import com.azure.cosmos.CosmosClient;
-import com.azure.cosmos.CosmosClientBuilder;
-import com.azure.identity.DefaultAzureCredential;
-import com.azure.storage.blob.BlobServiceClient;
-import com.azure.storage.blob.BlobServiceClientBuilder;
-import org.springframework.beans.factory.annotation.Autowired;
+import javax.inject.Named;
+
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Bean;
 import org.springframework.stereotype.Component;
+
 import com.azure.security.keyvault.secrets.SecretClient;
 import com.azure.security.keyvault.secrets.models.KeyVaultSecret;
 
-import javax.inject.Named;
-
 @Component
 public class AzureBootstrapConfig {
 
-    @Value("${azure.storage.account-name}")
-    private String storageAccount;
-
-    @Value("${azure.storage.container-name}")
-    private String storageContainer;
 
     @Value("${azure.keyvault.url}")
     private String keyVaultURL;
 
-    @Value("${azure.cosmosdb.database}")
-    private String cosmosDBName;
-
-    @Bean
-    @Named("STORAGE_ACCOUNT_NAME")
-    public String storageAccount() {
-        return storageAccount;
-    }
-
-    @Bean
-    @Named("STORAGE_CONTAINER_NAME")
-    public String containerName() {
-        return storageContainer;
-    }
-
-    @Bean
-    @Named("COSMOS_DB_NAME")
-    public String cosmosDBName() {
-        return cosmosDBName;
-    }
-
     @Bean
     @Named("KEY_VAULT_URL")
     public String keyVaultURL() {
         return keyVaultURL;
     }
 
-    @Bean
-    @Named("COSMOS_ENDPOINT")
-    public String cosmosEndpoint(SecretClient kv) {
-        return getKeyVaultSecret(kv, "opendes-cosmos-endpoint");