Commit 520a191b authored by Thomas Gehrmann [SLB]'s avatar Thomas Gehrmann [SLB]
Browse files

Merge branch 'master' into 32-schema-service-r3-schema-snapshot

# Conflicts:
#	.gitignore
parents 678b09f9 eaab30cb
......@@ -41,4 +41,8 @@ hs_err_pid*
# Python virtualenv
/venv/
dist/
\ No newline at end of file
dist/
# Environment configuration
*.env
.envrc
......@@ -246,6 +246,7 @@ The following software have components provided under the terms of this license:
- Google Cloud Core HTTP (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-core-http)
- Google Cloud Core gRPC (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-core-grpc)
- Google Cloud Datastore (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-datastore)
- Google Cloud Key Management Service (KMS) API v1-rev22-1.23.0 (from )
- Google Cloud Logging (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-logging)
- Google Cloud Pub/Sub (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-pubsub)
- Google Cloud Storage (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-storage)
......@@ -435,11 +436,13 @@ The following software have components provided under the terms of this license:
- javatuples (from http://www.javatuples.org)
- javax.inject (from http://code.google.com/p/atinject/)
- jose4j (from https://bitbucket.org/b_c/jose4j/)
- kotlin-stdlib (from )
- lang-mustache (from https://github.com/elastic/elasticsearch)
- lettuce (from http://github.com/mp911de/lettuce/wiki)
- micrometer-core (from https://github.com/micrometer-metrics/micrometer)
- micrometer-registry-azure-monitor (from https://github.com/micrometer-metrics/micrometer)
- org.apiguardian:apiguardian-api (from https://github.com/apiguardian-team/apiguardian)
- org.jetbrains.kotlin:kotlin-stdlib-common (from https://kotlinlang.org/)
- org.opentest4j:opentest4j (from https://github.com/ota4j-team/opentest4j)
- org.xmlunit:xmlunit-core (from http://www.xmlunit.org/)
- parent-join (from https://github.com/elastic/elasticsearch)
......
......@@ -16,7 +16,7 @@
global:
# Service(s) Replica Count
replicaCount: 1
replicaCount: 2
################################################################################
# Specify the Gitlab branch being used for image creation
......@@ -26,3 +26,5 @@ image:
repository: #{container-registry}#.azurecr.io
branch: #{ENVIRONMENT_NAME}#
tag: #{Build.SourceVersion}#
default_tenant: #{DEFAULT_TENANT}#
parameters:
providers: []
stages:
- ${{ each provider in parameters.providers }}:
- ${{ each environment in provider.environments }}:
- stage: 'Bootstrap_Schemas_${{ provider.name }}_${{ environment }}'
displayName: 'Bootstrap_Schemas_${{ provider.name }}_${{ environment }}'
jobs:
- job: 'Bootstrap_shared_schemas_job_${{ environment }}'
displayName: 'Bootstrap_shared_schemas_job_${{ environment }}'
variables:
- group: '${{ provider.name }} Target Env Secrets - ${{ environment }}'
- group: '${{ provider.name }} Target Env - ${{ environment }}'
- group: '${{ provider.name }} Service Release - schema-service'
steps:
- task: UsePythonVersion@0
inputs:
versionSpec: '3.x'
addToPath: true
architecture: 'x64'
- task: Bash@3
displayName: 'Deploy Shared Schemas'
inputs:
targetType: 'inline'
script: |
pip install -r deployments/scripts/azure/requirements.txt
export AZURE_AD_TENANT_ID=$(AZURE_TENANT_ID)
export AZURE_AD_APP_RESOURCE_ID=$(AZURE_AD_APP_RESOURCE_ID)
export INTEGRATION_TESTER=$(INTEGRATION_TESTER)
export TESTER_SERVICEPRINCIPAL_SECRET=$(AZURE_TESTER_SERVICEPRINCIPAL_SECRET)
export DATA_PARTITION=$(DEFAULT_TENANT)
export DEPLOY_SCHEMAS_URL=$(DEPLOY_SCHEMAS_URL)
export APP_KEY=""
AZURE_SP_TOKEN=$(python deployments/scripts/azure/Token.py)
export BEARER_TOKEN=$AZURE_SP_TOKEN
echo $DATA_PARTITION
echo $DEPLOY_SCHEMAS_URL
python deployments/scripts/DeploySharedSchemas.py -u $DEPLOY_SCHEMAS_URL
parameters:
environments: []
serviceName: 'schema-service'
skipTests: 'false'
skipDeploy: 'false'
valuesFile: ''
chartPath: ''
stages:
- template: /devops/build-stage.yml@TemplateRepo
parameters:
mavenGoal: 'package'
mavenPublishJUnitResults: true
serviceCoreMavenOptions: '--projects schema-core -Dmaven.test.skip=true'
mavenOptions: '--projects provider/schema-azure -Dmaven.test.skip=true'
copyFileContents: |
pom.xml
provider/schema-azure/maven/settings.xml
provider/schema-azure/pom.xml
provider/schema-azure/target/*-spring-boot.jar
copyFileContentsToFlatten: ''
mavenSettingsFile: './maven/settings.xml'
serviceBase: ${{ parameters.serviceName }}
testingRootFolder: 'testing'
chartPath: ${{ parameters.chartPath }}
# Deploy for each environment
- ${{ each environment in parameters.environments }}:
- template: /devops/deploy-stages.yml@TemplateRepo
parameters:
serviceName: ${{ parameters.serviceName }}
chartPath: ${{ parameters.chartPath }}
valuesFile: ${{ parameters.valuesFile }}
skipDeploy: ${{ parameters.skipDeploy }}
skipTest: ${{ parameters.skipTests }}
providers:
- name: Azure
environments:
- ${{ environment.name }}
- template: bootstrap.yml
parameters:
providers:
- name: Azure
environments:
- ${{ environment.name }}
\ No newline at end of file
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
......@@ -98,4 +99,12 @@ spec:
- name: entitlements_service_endpoint
value: http://entitlements-azure/entitlements/v1
- name: entitlements_service_api_key
value: "OBSOLETE"
\ No newline at end of file
value: "OBSOLETE"
- name: partition_service_endpoint
value: "http://partition/api/partition/v1"
- name: shared_partition
value: {{ .Values.default_tenant}}
- name: azure_istioauth_enabled
value: "true"
......@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
global:
replicaCount: 1
......@@ -19,4 +19,5 @@ image:
repository: community.opengroup.org:5555/osdu/platform/system/schema-service
branch: master
tag: latest
default_tenant: opendes
......@@ -11,12 +11,12 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
trigger:
batch: true
branches:
include:
- master
paths:
exclude:
- /**/*.md
......@@ -51,6 +51,8 @@ variables:
value: $(Pipeline.Workspace)/.m2/repository
- name: SKIP_TESTS
value: 'false'
- name: SKIP_DEPLOY
value: 'false'
stages:
- template: /devops/build-stage.yml@TemplateRepo
......
package org.opengroup.osdu.schema.provider.aws.security;
import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity;
import org.springframework.security.config.annotation.web.builders.HttpSecurity;
import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter;
@EnableWebSecurity
@EnableGlobalMethodSecurity(prePostEnabled = true)
public class WebSecurity extends WebSecurityConfigurerAdapter {
@Override
protected void configure(HttpSecurity http) throws Exception {
http.httpBasic().disable().csrf().disable(); // disable default authN. AuthN handled by endpoints proxy
}
}
......@@ -38,18 +38,21 @@ az keyvault secret show --vault-name $KEY_VAULT_NAME --name $KEY_VAULT_SECRET_NA
| `LOG_PREFIX` | `schema` | Logging prefix | no | - |
| `AUTHORIZE_API` | ex `https://foo-entitlements.azurewebsites.net` | Entitlements API endpoint | no | output of infrastructure deployment |
| `AUTHORIZE_API_KEY` | `********` | The API key clients will need to use when calling the entitlements | yes | -- |
| `partition_service_endpoint` | ex `https://foo-partition.azurewebsites.net` | Partition Service API endpoint | no | output of infrastructure deployment |
| `azure.activedirectory.app-resource-id` | `********` | AAD client application ID | yes | output of infrastructure deployment |
| `azure.application-insights.instrumentation-key` | `********` | API Key for App Insights | yes | output of infrastructure deployment |
| `azure.activedirectory.client-id` | `********` | AAD client application ID | yes | output of infrastructure deployment |
| `azure.activedirectory.AppIdUri` | `api://${azure.activedirectory.client-id}` | URI for AAD Application | no | -- |
| `azure.activedirectory.session-stateless` | `true` | Flag run in stateless mode (needed by AAD dependency) | no | -- |
| `cosmosdb_account` | ex `devintosdur2cosmosacct` | Cosmos account name | no | output of infrastructure deployment |
| `cosmosdb_database` | ex `dev-osdu-r2-db` | Cosmos database for storage documents | no | output of infrastructure deployment |
| `azure.storage.account-name` | ex `foo-storage-account` | Storage account for storing documents | no | output of infrastructure deployment |
| `azure.storage.enable-https` | `true` | Used by spring boot starter library | no | - |
| `KEYVAULT_URI` | ex `https://foo-keyvault.vault.azure.net/` | URI of KeyVault that holds application secrets | no | output of infrastructure deployment |
| `AZURE_CLIENT_ID` | `********` | Identity to run the service locally. This enables access to Azure resources. You only need this if running locally | yes | keyvault secret: `$KEYVAULT_URI/secrets/app-dev-sp-username` |
| `AZURE_TENANT_ID` | `********` | AD tenant to authenticate users from | yes | keyvault secret: `$KEYVAULT_URI/secrets/app-dev-sp-tenant-id` |
| `AZURE_CLIENT_SECRET` | `********` | Secret for `$AZURE_CLIENT_ID` | yes | keyvault secret: `$KEYVAULT_URI/secrets/app-dev-sp-password` |
| `partition_service_endpoint` | ex `https//foo-partition.azurewebsites.net/api/partition/v1` | Partition API endpoint | no | output of infrastructure deployment |
| `azure_istioauth_enabled` | `true` | Flag to Disable AAD auth | no | -- |
| `shared_partition` | `opendes` | Default Partition for Public Shared Schemas | no | -- |
| `server.port` | ex `8085` | port for schema service | no | -- |
......@@ -108,11 +111,11 @@ After configuring your environment as specified above, you can follow these step
```bash
mvn --projects schema-core,provider/schema-azure clean install
```
2. Run schema service in command line. We need to select which cloud vendor specific schema-service we want to run. For example, if we want to run schema-service for Azure, run the below command :
```bash
# Running Azure :
java -jar provider\schema-gcp\target\os-schema-azure-0.0.1-SNAPSHOT-spring-boot.jar
3. The port and path for the service endpoint can be configured in ```application.properties``` in the provider folder as following. If not specified, then the web container (ex. Tomcat) default is used:
2. Run schema service in command line. We need to select which cloud vendor specific schema-service we want to run. For example, if we want to run schema-service for Azure, run the below command :
```bash
# Running Azure :
java -jar provider/schema-azure/target/os-schema-azure-0.0.1-SNAPSHOT-spring-boot.jar
3. The port and path for the service endpoint can be configured in ```application.properties``` in the provider folder as following. If not specified, then the web container (ex. Tomcat) default is used:
```bash
server.servlet.contextPath=/api/schema-service/v1/
server.port=8080
......@@ -123,20 +126,29 @@ After configuring your environment as specified above, you can follow these step
After the service has started it should be accessible via a web browser by visiting [http://localhost:8080/swagger-ui.html](http://localhost:8080/swagger-ui.html). If the request does not fail, you can then run the integration tests.
### Running automated integration tests:
These tests validate functionality of schema service.
They can then be run/debugged directly in your IDE of choice using the GUI or via the commandline using below command from schema-core project.
Below command has to be run post building complete project.
cd testing/schema-test-core
```bash
# build + run Azure integration tests.
#
# Note: this assumes that the environment variables for integration tests as outlined
# above are already exported in your environment.
$ (cd testing/schema-test-core && mvn clean verify)
```
Additionally if you were trying to isolate specific variables the following can be executed
```bash
cd testing/schema-test-core
mvn verify -DVENDOR=azure -DHOST=http://localhost:8080 -DPRIVATE_TENANT1=opendes -DPRIVATE_TENANT2=tenant2 -DSHARED_TENANT=common -Dcucumber.options="--tags @SchemaService"
```
Below command can be run through azure-pipeline.yml after setting environment variables in the pipeline.
verify "-Dcucumber.options=--tags @SchemaService"
## Debugging
Jet Brains - the authors of Intellij IDEA, have written an [excellent guide](https://www.jetbrains.com/help/idea/debugging-your-first-java-application.html) on how to debug java programs.
......@@ -153,7 +165,7 @@ Copyright © Microsoft Corporation
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
You may obtain a copy of the License at
[http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0)
......
......@@ -15,140 +15,146 @@
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>os-schema</artifactId>
<groupId>org.opengroup.osdu</groupId>
<version>0.0.1</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>os-schema-azure</artifactId>
<version>0.0.1-SNAPSHOT</version>
<description>Azure related implementation staff.</description>
<packaging>jar</packaging>
<properties>
<osdu.corelibazure.version>0.0.40</osdu.corelibazure.version>
<osdu.oscorecommon.version>0.3.12</osdu.oscorecommon.version>
<osdu.os-schema-core.version>0.0.1</osdu.os-schema-core.version>
<mockito.version>1.10.19</mockito.version>
<cucumber.version>5.4.0</cucumber.version>
</properties>
<dependencyManagement>
<dependencies>
<!-- Inherit managed dependencies from core-lib-azure -->
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-azure</artifactId>
<version>${osdu.corelibazure.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>azure-active-directory-spring-boot-starter</artifactId>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-azure</artifactId>
<version>${osdu.corelibazure.version}</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
<version>${osdu.oscorecommon.version}</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-schema-core</artifactId>
<version>${osdu.os-schema-core.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
</exclusion>
</exclusions>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>${mockito.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.cucumber</groupId>
<artifactId>cucumber-java8</artifactId>
<version>${cucumber.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.cucumber</groupId>
<artifactId>cucumber-junit</artifactId>
<version>${cucumber.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.cucumber</groupId>
<artifactId>cucumber-guice</artifactId>
<version>${cucumber.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot</artifactId>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-beans</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
<configuration>
<classifier>spring-boot</classifier>
<mainClass>org.opengroup.osdu.schema.azure.SchemaApplication</mainClass>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>os-schema</artifactId>
<groupId>org.opengroup.osdu</groupId>
<version>0.0.1</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>os-schema-azure</artifactId>
<version>0.0.1-SNAPSHOT</version>
<description>Azure related implementation staff.</description>
<packaging>jar</packaging>
<properties>
<osdu.corelibazure.version>0.0.40</osdu.corelibazure.version>
<osdu.oscorecommon.version>0.3.18</osdu.oscorecommon.version>
<osdu.os-schema-core.version>0.0.1</osdu.os-schema-core.version>
<mockito.version>1.10.19</mockito.version>
<cucumber.version>5.4.0</cucumber.version>
</properties>
<dependencyManagement>
<dependencies>
<!-- Inherit managed dependencies from core-lib-azure -->
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-azure</artifactId>
<version>${osdu.corelibazure.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>azure-active-directory-spring-boot-starter</artifactId>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-azure</artifactId>
<version>${osdu.corelibazure.version}</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
<version>${osdu.oscorecommon.version}</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-schema-core</artifactId>
<version>${osdu.os-schema-core.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
</exclusion>
</exclusions>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>${mockito.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.cucumber</groupId>
<artifactId>cucumber-java8</artifactId>
<version>${cucumber.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.cucumber</groupId>
<artifactId>cucumber-junit</artifactId>
<version>${cucumber.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.cucumber</groupId>
<artifactId>cucumber-guice</artifactId>
<version>${cucumber.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot</artifactId>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-beans</artifactId>
</dependency>
<dependency>
<groupId>com.squareup.okio</groupId>
<artifactId>okio</artifactId>
<version>2.2.2</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
<configuration>
<classifier>spring-boot</classifier>
<mainClass>org.opengroup.osdu.schema.azure.SchemaApplication</mainClass>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
\ No newline at end of file
......@@ -22,7 +22,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.ComponentScan;
@SpringBootApplication(exclude = {
org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration.class })
org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration.class})
@ComponentScan({ "org.opengroup" })
public class SchemaApplication {
public static void main(String[] args)
......
......@@ -24,6 +24,5 @@ import org.opengroup.osdu.schema.model.Authority;
@NoArgsConstructor
public class AuthorityDoc {
private String id;
private String dataPartitionId;
private Authority authority;
}
......@@ -24,6 +24,5 @@ import org.opengroup.osdu.schema.model.EntityType;
@NoArgsConstructor
public class EntityTypeDoc {
private String id;
private String dataPartitionId;
private EntityType entityType;
}