From 32f3116dca2e44af9963ef23e2e11fc7709337c1 Mon Sep 17 00:00:00 2001 From: Aliaksandr Lubouski Date: Thu, 27 Aug 2020 12:27:37 +0000 Subject: [PATCH 01/10] AUTHORIZE_API_KEY changed with adding : --- .mvn/community-maven.settings.xml | 134 +++++++++++------- pom.xml | 12 +- provider/schema-gcp/pom.xml | 4 +- .../schema-gcp/src/main/appengine/app.yaml | 18 +++ schema-core/pom.xml | 2 +- .../util/EntitlementsClientFactory.java | 2 +- 6 files changed, 109 insertions(+), 63 deletions(-) create mode 100644 provider/schema-gcp/src/main/appengine/app.yaml diff --git a/.mvn/community-maven.settings.xml b/.mvn/community-maven.settings.xml index 75718a6d..b00721f7 100644 --- a/.mvn/community-maven.settings.xml +++ b/.mvn/community-maven.settings.xml @@ -1,55 +1,83 @@ - - - - GitLab-Authenticate-With-Job-Token - - true - - - community-maven-via-job-token - - - - - - GitLab-Authenticate-With-Private-Token - - - env.COMMUNITY_MAVEN_TOKEN - - - - community-maven-via-private-token - - - - - - - community-maven-via-job-token - - - - Job-Token - ${env.CI_JOB_TOKEN} - - - - - - - community-maven-via-private-token - - - - Private-Token - ${env.COMMUNITY_MAVEN_TOKEN} - - - - - + + + + GitLab-Authenticate-With-Job-Token + + true + + + community-maven-repo + community-maven-via-job-token + community-maven-via-job-token + + https://community.opengroup.org/api/v4/groups/17/-/packages/maven + https://community.opengroup.org/api/v4/projects/26/packages/maven + + https://community.opengroup.org/api/v4/projects/26/packages/maven + + + + + + + GitLab-Authenticate-With-Private-Token + + + env.COMMUNITY_MAVEN_TOKEN + + + + community-maven-repo + community-maven-via-private-token + community-maven-via-private-token + + https://community.opengroup.org/api/v4/groups/17/-/packages/maven + https://community.opengroup.org/api/v4/projects/26/packages/maven + + https://community.opengroup.org/api/v4/projects/26/packages/maven + + + + + + + + community-maven-via-job-token + + + + Job-Token + ${env.CI_JOB_TOKEN} + + + + + + + community-maven-via-private-token + + + + Private-Token + ${env.COMMUNITY_MAVEN_TOKEN} + + + + + + + azure-auth + + ${env.AZURE_TENANT_ID} + ${env.AZURE_CLIENT_ID} + ${env.AZURE_CLIENT_SECRET} + AZURE + + + + + diff --git a/pom.xml b/pom.xml index e19b79ff..cff8c23c 100644 --- a/pom.xml +++ b/pom.xml @@ -46,19 +46,19 @@ - ${gitlab-server} - https://community.opengroup.org/api/v4/groups/17/-/packages/maven + ${repo.releases.id} + ${repo.releases.url} - ${gitlab-server} - https://community.opengroup.org/api/v4/projects/26/packages/maven + ${publish.releases.id} + ${publish.releases.url} - ${gitlab-server} - https://community.opengroup.org/api/v4/projects/26/packages/maven + ${publish.snapshots.id} + ${publish.snapshots.url} diff --git a/provider/schema-gcp/pom.xml b/provider/schema-gcp/pom.xml index 00add0e1..6c1c7898 100644 --- a/provider/schema-gcp/pom.xml +++ b/provider/schema-gcp/pom.xml @@ -17,7 +17,7 @@ com.google.cloud google-cloud-storage - 1.105.0 + 1.108.0 @@ -131,4 +131,4 @@ - \ No newline at end of file + diff --git a/provider/schema-gcp/src/main/appengine/app.yaml b/provider/schema-gcp/src/main/appengine/app.yaml new file mode 100644 index 00000000..27562710 --- /dev/null +++ b/provider/schema-gcp/src/main/appengine/app.yaml @@ -0,0 +1,18 @@ +service: os-schema +runtime: java +env: flex + +resources: + cpu: 2 + memory_gb: 6 + +automatic_scaling: + min_num_instances: 2 + max_num_instances: 5 + cool_down_period_sec: 120 + cpu_utilization: + target_utilization: 0.7 + +env_variables: + SPRING_PROFILES_ACTIVE: 'ENVIRONMENT' + diff --git a/schema-core/pom.xml b/schema-core/pom.xml index a3650354..dff50370 100644 --- a/schema-core/pom.xml +++ b/schema-core/pom.xml @@ -15,7 +15,7 @@ 2.7.0 - 0.0.18 + 0.3.5 diff --git a/schema-core/src/main/java/org/opengroup/osdu/schema/util/EntitlementsClientFactory.java b/schema-core/src/main/java/org/opengroup/osdu/schema/util/EntitlementsClientFactory.java index 9f10cc7b..9ce2a5ec 100644 --- a/schema-core/src/main/java/org/opengroup/osdu/schema/util/EntitlementsClientFactory.java +++ b/schema-core/src/main/java/org/opengroup/osdu/schema/util/EntitlementsClientFactory.java @@ -40,4 +40,4 @@ public class EntitlementsClientFactory extends AbstractFactoryBean getObjectType() { return IEntitlementsFactory.class; } -} \ No newline at end of file +} -- GitLab From df984c100bc24861bac621ca5d0016069fdc23b5 Mon Sep 17 00:00:00 2001 From: Igor Filippov Date: Thu, 24 Sep 2020 15:04:02 +0400 Subject: [PATCH 02/10] GONRG-834 Make default data partition id for Schema configurable --- .../schemainfostore/GoogleSchemaInfoStore.java | 18 +++++++++++++++--- .../impl/schemastore/GoogleSchemaStore.java | 10 ++++++++++ .../src/main/resources/application.properties | 2 ++ 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/provider/schema-gcp/src/main/java/org/opengroup/osdu/schema/impl/schemainfostore/GoogleSchemaInfoStore.java b/provider/schema-gcp/src/main/java/org/opengroup/osdu/schema/impl/schemainfostore/GoogleSchemaInfoStore.java index 1759e9fd..bef82518 100644 --- a/provider/schema-gcp/src/main/java/org/opengroup/osdu/schema/impl/schemainfostore/GoogleSchemaInfoStore.java +++ b/provider/schema-gcp/src/main/java/org/opengroup/osdu/schema/impl/schemainfostore/GoogleSchemaInfoStore.java @@ -27,6 +27,7 @@ import org.opengroup.osdu.schema.model.SchemaRequest; import org.opengroup.osdu.schema.provider.interfaces.schemainfostore.ISchemaInfoStore; import org.opengroup.osdu.schema.util.VersionHierarchyUtil; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Repository; import com.google.cloud.Timestamp; @@ -65,6 +66,12 @@ public class GoogleSchemaInfoStore implements ISchemaInfoStore { @Autowired JaxRsDpsLog log; + @Value("${default.account.id}") + private String defaultAccountId; + + @Value("${common.account.id}") + private String commonAccountId; + /** * Method to get schemaInfo from google store * @@ -254,7 +261,12 @@ public class GoogleSchemaInfoStore implements ISchemaInfoStore { @Override public List getSchemaInfoList(QueryParams queryParams, String tenantId) throws ApplicationException { - Datastore datastore = dataStoreFactory.getDatastore(tenantId, SchemaConstants.NAMESPACE); + Datastore datastore; + if (tenantId.equalsIgnoreCase(commonAccountId)) { + datastore = dataStoreFactory.getDatastore(defaultAccountId, SchemaConstants.NAMESPACE); + } else { + datastore = dataStoreFactory.getDatastore(tenantId, SchemaConstants.NAMESPACE); + } List filterList = getFilters(queryParams); EntityQuery.Builder queryBuilder = Query.newEntityQueryBuilder().setNamespace(SchemaConstants.NAMESPACE) @@ -305,11 +317,11 @@ public class GoogleSchemaInfoStore implements ISchemaInfoStore { public boolean isUnique(String schemaId, String tenantId) throws ApplicationException { Set tenantList = new HashSet<>(); - tenantList.add(SchemaConstants.ACCOUNT_ID_COMMON_PROJECT); + tenantList.add(defaultAccountId); tenantList.add(tenantId); // code to call check uniqueness - if (tenantId.equalsIgnoreCase(SchemaConstants.ACCOUNT_ID_COMMON_PROJECT)) { + if (tenantId.equalsIgnoreCase(defaultAccountId)) { List privateTenantList = tenantFactory.listTenantInfo().stream().map(TenantInfo::getDataPartitionId) .collect(Collectors.toList()); tenantList.addAll(privateTenantList); diff --git a/provider/schema-gcp/src/main/java/org/opengroup/osdu/schema/impl/schemastore/GoogleSchemaStore.java b/provider/schema-gcp/src/main/java/org/opengroup/osdu/schema/impl/schemastore/GoogleSchemaStore.java index 1667f041..e2693504 100644 --- a/provider/schema-gcp/src/main/java/org/opengroup/osdu/schema/impl/schemastore/GoogleSchemaStore.java +++ b/provider/schema-gcp/src/main/java/org/opengroup/osdu/schema/impl/schemastore/GoogleSchemaStore.java @@ -9,6 +9,7 @@ import org.opengroup.osdu.schema.exceptions.ApplicationException; import org.opengroup.osdu.schema.exceptions.NotFoundException; import org.opengroup.osdu.schema.provider.interfaces.schemastore.ISchemaStore; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Repository; import com.google.cloud.storage.Blob; @@ -37,6 +38,12 @@ public class GoogleSchemaStore implements ISchemaStore { @Autowired JaxRsDpsLog log; + @Value("${default.account.id}") + private String defaultAccountId; + + @Value("${common.account.id}") + private String commonAccountId; + /** * Method to get schema from google Storage given Tenant ProjectInfo * @@ -49,6 +56,9 @@ public class GoogleSchemaStore implements ISchemaStore { */ @Override public String getSchema(String dataPartitionId, String filePath) throws ApplicationException, NotFoundException { + if (dataPartitionId.equalsIgnoreCase(commonAccountId)) { + dataPartitionId = defaultAccountId; + } filePath = filePath + SchemaConstants.JSON_EXTENSION; String bucketname = getSchemaBucketName(dataPartitionId); Storage storage = storageFactory.get(tenantFactory.getTenantInfo(dataPartitionId)); diff --git a/provider/schema-gcp/src/main/resources/application.properties b/provider/schema-gcp/src/main/resources/application.properties index 010ad0c5..fcda06bb 100644 --- a/provider/schema-gcp/src/main/resources/application.properties +++ b/provider/schema-gcp/src/main/resources/application.properties @@ -7,3 +7,5 @@ management.health.elasticsearch.enabled=false management.endpoints.web.base-path=/ management.endpoints.web.path-mapping.health=health LOG_PREFIX=schema +default.account.id=common +common.account.id=common -- GitLab From 0090499a7f3677e66c0d852fbe086004188898ce Mon Sep 17 00:00:00 2001 From: Stanislav Riabokon Date: Wed, 30 Sep 2020 15:38:38 +0400 Subject: [PATCH 03/10] Added google-cloud-core-http for google-cloud-datastore. --- provider/schema-gcp/pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/provider/schema-gcp/pom.xml b/provider/schema-gcp/pom.xml index 6c1c7898..f1ade260 100644 --- a/provider/schema-gcp/pom.xml +++ b/provider/schema-gcp/pom.xml @@ -25,6 +25,12 @@ com.google.cloud google-cloud-datastore 1.102.2 + + + com.google.cloud + google-cloud-core-http + + -- GitLab From 07e97b3a79a2bc11c54a24804aec41125f83bc5f Mon Sep 17 00:00:00 2001 From: Igor Filippov Date: Wed, 30 Sep 2020 15:47:26 +0400 Subject: [PATCH 04/10] GONRG-898: Schema unit tests failed --- .../schemainfostore/GoogleSchemaInfoStore.java | 16 ++++++++++++++++ .../GoogleSchemaInfoStoreTest.java | 2 ++ 2 files changed, 18 insertions(+) diff --git a/provider/schema-gcp/src/main/java/org/opengroup/osdu/schema/impl/schemainfostore/GoogleSchemaInfoStore.java b/provider/schema-gcp/src/main/java/org/opengroup/osdu/schema/impl/schemainfostore/GoogleSchemaInfoStore.java index bef82518..a0d4987e 100644 --- a/provider/schema-gcp/src/main/java/org/opengroup/osdu/schema/impl/schemainfostore/GoogleSchemaInfoStore.java +++ b/provider/schema-gcp/src/main/java/org/opengroup/osdu/schema/impl/schemainfostore/GoogleSchemaInfoStore.java @@ -340,4 +340,20 @@ public class GoogleSchemaInfoStore implements ISchemaInfoStore { } return true; } + + public String getDefaultAccountId() { + return defaultAccountId; + } + + public void setDefaultAccountId(String defaultAccountId) { + this.defaultAccountId = defaultAccountId; + } + + public String getCommonAccountId() { + return commonAccountId; + } + + public void setCommonAccountId(String commonAccountId) { + this.commonAccountId = commonAccountId; + } } \ No newline at end of file diff --git a/provider/schema-gcp/src/test/java/org/opengroup/osdu/schema/impl/schemainfostore/GoogleSchemaInfoStoreTest.java b/provider/schema-gcp/src/test/java/org/opengroup/osdu/schema/impl/schemainfostore/GoogleSchemaInfoStoreTest.java index 3fecc824..fdd40043 100644 --- a/provider/schema-gcp/src/test/java/org/opengroup/osdu/schema/impl/schemainfostore/GoogleSchemaInfoStoreTest.java +++ b/provider/schema-gcp/src/test/java/org/opengroup/osdu/schema/impl/schemainfostore/GoogleSchemaInfoStoreTest.java @@ -172,6 +172,8 @@ public class GoogleSchemaInfoStoreTest { KeyFactory storageKeyFactory = mock(KeyFactory.class); String schemaId = "schemaId"; String tenantId = "tenant"; + schemaInfoStore.setCommonAccountId(tenantId); + schemaInfoStore.setDefaultAccountId(tenantId); Mockito.when(tenantFactory.getTenantInfo("tenant")).thenReturn(tenantInfo); Mockito.when(tenantFactory.getTenantInfo("common")).thenReturn(tenantInfoCommon); Mockito.when(tenantInfo.getName()).thenReturn("tenant"); -- GitLab From 9ba707fc85abb9f14b30699936cc03f01a84bb42 Mon Sep 17 00:00:00 2001 From: Denis Karpenok Date: Fri, 9 Oct 2020 14:13:32 +0300 Subject: [PATCH 05/10] GO3NRG-933 Sonar --- provider/schema-gcp/pom.xml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/provider/schema-gcp/pom.xml b/provider/schema-gcp/pom.xml index 3fc5546f..43ef9de8 100644 --- a/provider/schema-gcp/pom.xml +++ b/provider/schema-gcp/pom.xml @@ -115,7 +115,25 @@ - + + org.jacoco + jacoco-maven-plugin + 0.7.7.201606060606 + + + + prepare-agent + + + + report + prepare-package + + report + + + + -- GitLab From c98078c66c17363cefd7f15048c9811aef7354d6 Mon Sep 17 00:00:00 2001 From: Rustam Lotsmanenko Date: Mon, 12 Oct 2020 11:13:56 +0000 Subject: [PATCH 06/10] GONRG-804 Actualize README.md for GCP --- README.md | 58 ++---------- provider/schema-gcp/README.md | 171 ++++++++++++++++++++++++++++++++++ 2 files changed, 179 insertions(+), 50 deletions(-) create mode 100644 provider/schema-gcp/README.md diff --git a/README.md b/README.md index 24945f27..62d98549 100644 --- a/README.md +++ b/README.md @@ -4,78 +4,36 @@ The Schema Service is a Maven multi-module project with each cloud implemention ### 1. GCP deployment -#### Prerequisite (Infra and access required) - -Schema service as per design uses two module from GCP. GCS or Google cloud storage to store actual schemas and Google cloud datastore to store schema metadata. It follows the multi tenancy -concept of DE, which means service is deployed in one GCP project and data is stored in client specific project. And permission to speccfic tenant project is decided based on data-partition-id user passes -as part of request header. So, to make it work from local we must have following setup done as prerequisite, - -1. GCP project setup is done and local gcloud sdk configured by activating the account/user and pointing to correct GCP project. You can follow the steps from [here](https://cloud.google.com/deployment-manager/docs/step-by-step-guide/installation-and-setup) - -2. Bucket with name -schema (e.g opendes-schema) is created in tenant GCS and tenant datafier service account has read/write access to that bucket. Steps to create bucket and grant access can be followed from [here](https://cloud.google.com/storage/docs/creating-buckets) - -3. Tenant datafier service account has read/write access to Google cloud datastore in tenant project. You can follow access control on datastore from [here](https://cloud.google.com/datastore/docs/access/iam). Permission required is ```roles/datastore.user``` - -4. Service-account/user activated as part of step 1 has service token creator role on datafier service-account of the data partition used. Details on service account creator role can be accessed from [here](https://cloud.google.com/iam/docs/service-accounts#the_service_account_token_creator_role) - -5. TenantInfo table should be present in service GCP datastore under namespace ```datascosystem``` and kind ```tenantInfo``` and has entry corresponding to data-partition-id passed. - -6. User/service-account that will be used to run the service has access to ```service.schema-service.editors``` group in the specified data-partition. - -### Local deployment Steps - -Once the above Prerequisite are done, we can follow the below steps to run the service locally, - -1. Navigate to the root of the schema project, os-schema. For building the project using command line, run below command : - ```bash - mvn clean install - ``` - This will build the core project as well as all the underlying projects. If we want to build projects for specific cloud vendor, we can use mvn --projects command. For example, if we want to build only for GCP(Google Cloud Platform), we can use below command : - ```bash - mvn --projects schema-core,provider/schema-gcp clean install - ``` -2. Run schema service in command line. We need to select which cloud vendor specific schema-service we want to run. For example, if we want to run schema-service for GCP, run the below command : - ```bash - # Running GCP : - java -jar provider\schema-gcp\target\os-schema-gcp-0.0.1-spring-boot.jar -3. The port and path for the service endpoint can be configured in ```application.properties``` in the provider folder as following. If not specified, then the web container (ex. Tomcat) default is used: - ```bash - server.servlet.contextPath=/api/schema-service/v1/ - server.port=8080 - ``` - -You can access the service APIs by following the service contract in [schema.yaml](https://dev.azure.com/slb-des-ext-collaboration/open-data-ecosystem/_git/os-schema?path=%2Fdocs%2Fapi%2Fschema.yaml) +Instructions for running the GCP implementation in the cloud can be found [here](./provider/schema-gcp/README.md). ### 2. Azure deployment Instructions for running the Azure implementation in the cloud can be found [here](https://community.opengroup.org/osdu/platform/system/schema-service/-/blob/master/provider/schema-azure/README.md). ## Running Automated Integration Test + DevSanity tests are located in a schema-core project in testing directory under the project root directory. 1. GCP - These tests validate functionality of schema service. They can then be run/debugged directly in your IDE of choice using the GUI or via the commandline using below command from schema-core project. Below command has to be run post building complete project. - - cd testing/schema-test-core - mvn verify -DVENDOR=gcp -DHOST=https://open.opendes.cloud.slb-ds.com -DPRIVATE_TENANT1=opendes -DPRIVATE_TENANT2=tenant2 -DSHARED_TENANT=common - +Instructions for running the GCP integration tests can be found [here](./provider/schema-gcp/README.md). + Below command can be run through azure-pipeline.yml after setting environment variables in the pipeline. verify ## Deploy Shared Schemas -Schema service as part of deployment deploys pre-defined OSDU schemas so end users can get community accepted schemas to refer. Such schemas are present in [folder](https://dev.azure.com/slb-des-ext-collaboration/open-data-ecosystem/_git/os-schema?path=%2Fdeployments%2Fshared-schemas%2Fosdu) and script to deploy the schema are present [here](https://dev.azure.com/slb-des-ext-collaboration/open-data-ecosystem/_git/os-schema?path=%2Fdeployments%2Fscripts). - -Details to deploy shared schemas can be found under [README.md](deployments/shared-schemas/README.md) - +Schema service as part of deployment deploys pre-defined OSDU schemas so end users can get community accepted schemas to refer. Such schemas are present in [folder](./deployments/shared-schemas/osdu) and script to deploy the schema are present [here](deployments/scripts). +Details to deploy shared schemas can be found under [README.md](./deployments/shared-schemas/README.md) + ## License + Copyright 2017-2020, Schlumberger Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/provider/schema-gcp/README.md b/provider/schema-gcp/README.md new file mode 100644 index 00000000..bdaed935 --- /dev/null +++ b/provider/schema-gcp/README.md @@ -0,0 +1,171 @@ +# Schema Service +schema-gcp is a Maven multi-module project service. + +## Getting Started +These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system. + +### Prerequisites (Infra and access required) +Pre-requisites + +* GCloud SDK with java (latest version) +* JDK 8 +* Lombok 1.16 or later +* Maven + +Schema service as per design uses two module from GCP. GCS or Google cloud storage to store actual schemas and Google cloud datastore to store schema metadata. It follows the multi tenancy +concept of DE, which means service is deployed in one GCP project and data is stored in client specific project. And permission to speccfic tenant project is decided based on data-partition-id user passes +as part of request header. So, to make it work from local we must have following setup done as prerequisite, + +1. GCP project setup is done and local gcloud sdk configured by activating the account/user and pointing to correct GCP project. You can follow the steps from [here](https://cloud.google.com/deployment-manager/docs/step-by-step-guide/installation-and-setup) + +2. Bucket with name -schema (e.g opendes-schema) is created in tenant GCS and tenant datafier service account has read/write access to that bucket. Steps to create bucket and grant access can be followed from [here](https://cloud.google.com/storage/docs/creating-buckets) + +3. Tenant datafier service account has read/write access to Google cloud datastore in tenant project. You can follow access control on datastore from [here](https://cloud.google.com/datastore/docs/access/iam). Permission required is ```roles/datastore.user``` + +4. Service-account/user activated as part of step 1 has service token creator role on datafier service-account of the data partition used. Details on service account creator role can be accessed from [here](https://cloud.google.com/iam/docs/service-accounts#the_service_account_token_creator_role) + +5. TenantInfo table should be present in service GCP datastore under namespace ```datascosystem``` and kind ```tenantInfo``` and has entry corresponding to data-partition-id passed. + +6. User/service-account that will be used to run the service has access to ```service.schema-service.editors``` group in the specified data-partition. + +### Installation +In order to run the service locally or remotely, you will need to have the following environment variables defined. + +| name | value | description | sensitive? | source | +| --- | --- | --- | --- | --- | +| `LOG_PREFIX` | `schema` | Logging prefix | no | - | +| `SERVER_SERVLET_CONTEXPATH` | `/api/schema-service/v1` | Servlet context path | no | - | +| `AUTHORIZE_API` | ex `https://entitlements.com/entitlements/v1` | Entitlements API endpoint | no | output of infrastructure deployment | +| `ACCOUNT_ID_COMMON_PROJECT` | ex `common` | Shared account id | no | - | +| `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts | +| `GCLOUD_PROJECT` | `******` | Cloud project id, you only need this if running locally | no | https://console.cloud.google.com | + +### Run Locally +Check that maven is installed: + +```bash +$ mvn --version +Apache Maven 3.6.0 +Maven home: /usr/share/maven +Java version: 1.8.0_212, vendor: AdoptOpenJDK, runtime: /usr/lib/jvm/jdk8u212-b04/jre +... +``` + +You may need to configure access to the remote maven repository that holds the OSDU dependencies. This file should live within `~/.mvn/community-maven.settings.xml`: + +```bash +$ cat ~/.m2/settings.xml + + + + + community-maven-via-private-token + + + + + + Private-Token + ${env.COMMUNITY_MAVEN_TOKEN} + + + + + + +``` +* Update the Google cloud SDK to the latest version: + +```bash +gcloud components update +``` +* Set Google Project Id: + +```bash +gcloud config set project +``` + +* Perform a basic authentication in the selected project: + +```bash +gcloud auth application-default login +``` +Once the above Prerequisite are done, we can follow the below steps to run the service locally, + +1. Navigate to the root of the schema project, os-schema. For building the project using command line, run below command : + ```bash + mvn clean install + ``` + This will build the core project as well as all the underlying projects. If we want to build projects for specific cloud vendor, we can use mvn --projects command. For example, if we want to build only for GCP(Google Cloud Platform), we can use below command : + ```bash + mvn --projects schema-core,provider/schema-gcp clean install + ``` +2. Run schema service in command line. We need to select which cloud vendor specific schema-service we want to run. For example, if we want to run schema-service for GCP, run the below command : + ```bash + # Running GCP : + java -jar provider\schema-gcp\target\os-schema-gcp-0.0.1-spring-boot.jar +3. The port and path for the service endpoint can be configured in ```application.properties``` in the provider folder as following. If not specified, then the web container (ex. Tomcat) default is used: + ```bash + server.servlet.contextPath=/api/schema-service/v1/ + server.port=8080 + +You can access the service APIs by following the service contract in [schema.yaml](docs/api/schema.yaml) + +## Testing + + ### Running E2E Tests + This section describes how to run cloud OSDU E2E tests (testing/schema-test-core). + + You will need to have the following environment variables defined. + + | name | value | description | sensitive? | source | + | --- | --- | --- | --- | --- | + | `INTEGRATION_TEST_AUDIENCE` | `*****.apps.googleusercontent.com` | client application ID | yes | https://console.cloud.google.com/apis/credentials | + | `VENDOR` | `gcp` | Use value 'gcp' to run gcp tests | no | - | + | `HOST` | ex`http://localhost:8080` | Schema service host | no | - | + | `INTEGRATION_TESTER` | `********` | Service account base64 encoded string for API calls. Note: this user must have entitlements configured already | yes | https://console.cloud.google.com/iam-admin/serviceaccounts | + | `PRIVATE_TENANT2` | ex`opendes` | OSDU tenant used for testing | no | - | + | `PRIVATE_TENANT1` | ex`osdu` | OSDU tenant used for testing | no | - | + | `SHARED_TENANT` | ex`common` | OSDU tenant used for testing | no | - | + + **Entitlements configuration for integration accounts** + + | INTEGRATION_TESTER | + | --- | + | users
service.entitlements.user
service.schema-service.viewers
service.schema-service.editors
data.integration.test
data.test1 | + + Execute following command to build code and run all the integration tests: + + ```bash + # Note: this assumes that the environment variables for integration tests as outlined + # above are already exported in your environment. + # build + install integration test core + $ (cd testing/schema-test-core/ && mvn clean test) + ``` + +## Deployment + +Schema Service is compatible with Cloud Run. + +* To deploy into Cloud run, please, use this documentation: +https://cloud.google.com/run/docs/quickstarts/build-and-deploy + +## License + +Copyright © Google LLC + +Copyright © EPAM Systems + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +[http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0) + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file -- GitLab From 4de5be35e4a44d0d2a0f9abe5860da1bba744f83 Mon Sep 17 00:00:00 2001 From: Stanislav Riabokon Date: Tue, 13 Oct 2020 16:22:45 +0400 Subject: [PATCH 07/10] Removed testing/schema-test-core --- pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/pom.xml b/pom.xml index f0cc9a90..d57d6a2e 100644 --- a/pom.xml +++ b/pom.xml @@ -44,7 +44,6 @@ provider/schema-gcp provider/schema-ibm provider/schema-azure - testing/schema-test-core -- GitLab From 879fa3f989b32b247ce564ecba658bae24a46798 Mon Sep 17 00:00:00 2001 From: Igor Filippov Date: Wed, 14 Oct 2020 12:59:56 +0400 Subject: [PATCH 08/10] GONRG-945: fix NPE --- .../schema/impl/schemainfostore/GoogleSchemaInfoStore.java | 6 ++++-- .../osdu/schema/impl/schemastore/GoogleSchemaStore.java | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/provider/schema-gcp/src/main/java/org/opengroup/osdu/schema/impl/schemainfostore/GoogleSchemaInfoStore.java b/provider/schema-gcp/src/main/java/org/opengroup/osdu/schema/impl/schemainfostore/GoogleSchemaInfoStore.java index 73249ff0..95064025 100644 --- a/provider/schema-gcp/src/main/java/org/opengroup/osdu/schema/impl/schemainfostore/GoogleSchemaInfoStore.java +++ b/provider/schema-gcp/src/main/java/org/opengroup/osdu/schema/impl/schemainfostore/GoogleSchemaInfoStore.java @@ -25,7 +25,6 @@ import org.opengroup.osdu.schema.model.SchemaIdentity; import org.opengroup.osdu.schema.model.SchemaInfo; import org.opengroup.osdu.schema.model.SchemaRequest; import org.opengroup.osdu.schema.provider.interfaces.schemainfostore.ISchemaInfoStore; -import org.opengroup.osdu.schema.util.VersionHierarchyUtil; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Repository; @@ -258,6 +257,10 @@ public class GoogleSchemaInfoStore implements ISchemaInfoStore { @Override public List getSchemaInfoList(QueryParams queryParams, String tenantId) throws ApplicationException { + List schemaList = new LinkedList<>(); + if (SchemaConstants.ACCOUNT_ID_COMMON_PROJECT.equals(tenantId)) { + return schemaList; + } Datastore datastore = dataStoreFactory.getDatastore(tenantId, SchemaConstants.NAMESPACE); List filterList = getFilters(queryParams); @@ -270,7 +273,6 @@ public class GoogleSchemaInfoStore implements ISchemaInfoStore { } QueryResults result = datastore.run(queryBuilder.build()); - List schemaList = new LinkedList<>(); while (result.hasNext()) { Entity entity = result.next(); schemaList.add(getSchemaInfoObject(entity, datastore)); diff --git a/provider/schema-gcp/src/main/java/org/opengroup/osdu/schema/impl/schemastore/GoogleSchemaStore.java b/provider/schema-gcp/src/main/java/org/opengroup/osdu/schema/impl/schemastore/GoogleSchemaStore.java index 073e7493..7dc586d8 100644 --- a/provider/schema-gcp/src/main/java/org/opengroup/osdu/schema/impl/schemastore/GoogleSchemaStore.java +++ b/provider/schema-gcp/src/main/java/org/opengroup/osdu/schema/impl/schemastore/GoogleSchemaStore.java @@ -9,7 +9,6 @@ import org.opengroup.osdu.schema.exceptions.ApplicationException; import org.opengroup.osdu.schema.exceptions.NotFoundException; import org.opengroup.osdu.schema.provider.interfaces.schemastore.ISchemaStore; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Repository; import com.google.cloud.storage.Blob; @@ -50,6 +49,9 @@ public class GoogleSchemaStore implements ISchemaStore { */ @Override public String getSchema(String dataPartitionId, String filePath) throws ApplicationException, NotFoundException { + if (SchemaConstants.ACCOUNT_ID_COMMON_PROJECT.equals(dataPartitionId)) { + throw new NotFoundException(SchemaConstants.SCHEMA_NOT_PRESENT); + } filePath = filePath + SchemaConstants.JSON_EXTENSION; String bucketname = getSchemaBucketName(dataPartitionId); Storage storage = storageFactory.get(tenantFactory.getTenantInfo(dataPartitionId)); -- GitLab From 3bd0ac66b11c2013b07e5450f7376556f0d4059c Mon Sep 17 00:00:00 2001 From: Rustam_Lotsmanenko Date: Wed, 14 Oct 2020 14:50:50 +0400 Subject: [PATCH 09/10] verify exclusion --- provider/schema-gcp/pom.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/provider/schema-gcp/pom.xml b/provider/schema-gcp/pom.xml index 43ef9de8..f4eea9a2 100644 --- a/provider/schema-gcp/pom.xml +++ b/provider/schema-gcp/pom.xml @@ -25,12 +25,6 @@ com.google.cloud google-cloud-datastore 1.102.2 - - - com.google.cloud - google-cloud-core-http - - -- GitLab From 6a6098909e5a183524479eca6c1523ffcd8311cc Mon Sep 17 00:00:00 2001 From: Igor Filippov Date: Fri, 16 Oct 2020 12:52:58 +0400 Subject: [PATCH 10/10] hide changes in pom and settings --- .mvn/community-maven.settings.xml | 34 +++---------------------------- pom.xml | 13 ++++++------ 2 files changed, 10 insertions(+), 37 deletions(-) diff --git a/.mvn/community-maven.settings.xml b/.mvn/community-maven.settings.xml index b00721f7..adcabd34 100644 --- a/.mvn/community-maven.settings.xml +++ b/.mvn/community-maven.settings.xml @@ -7,15 +7,7 @@ true - community-maven-repo - community-maven-via-job-token - community-maven-via-job-token - - https://community.opengroup.org/api/v4/groups/17/-/packages/maven - https://community.opengroup.org/api/v4/projects/26/packages/maven - - https://community.opengroup.org/api/v4/projects/26/packages/maven - + community-maven-via-job-token @@ -30,15 +22,7 @@ - community-maven-repo - community-maven-via-private-token - community-maven-via-private-token - - https://community.opengroup.org/api/v4/groups/17/-/packages/maven - https://community.opengroup.org/api/v4/projects/26/packages/maven - - https://community.opengroup.org/api/v4/projects/26/packages/maven - + community-maven-via-private-token @@ -67,17 +51,5 @@ - - - azure-auth - - ${env.AZURE_TENANT_ID} - ${env.AZURE_CLIENT_ID} - ${env.AZURE_CLIENT_SECRET} - AZURE - - - - - + \ No newline at end of file diff --git a/pom.xml b/pom.xml index d57d6a2e..52441901 100644 --- a/pom.xml +++ b/pom.xml @@ -44,22 +44,23 @@ provider/schema-gcp provider/schema-ibm provider/schema-azure + testing/schema-test-core - ${repo.releases.id} - ${repo.releases.url} + ${gitlab-server} + https://community.opengroup.org/api/v4/groups/17/-/packages/maven - ${publish.releases.id} - ${publish.releases.url} + ${gitlab-server} + https://community.opengroup.org/api/v4/projects/26/packages/maven - ${publish.snapshots.id} - ${publish.snapshots.url} + ${gitlab-server} + https://community.opengroup.org/api/v4/projects/26/packages/maven -- GitLab