Commit 7423572b authored by David Diederich's avatar David Diederich
Browse files

Merge remote-tracking branch 'origin/master' into update-pom-reference

Conflicts:
	pom.xml
parents c41f8c2f 858a3514
Pipeline #32944 passed with stages
in 20 minutes and 19 seconds
......@@ -5,7 +5,7 @@ version: 2
cli:
server: https://app.fossa.com
fetcher: custom
project: Dataset Registry
project: Dataset
analyze:
modules:
- name: os-dataset
......@@ -20,3 +20,11 @@ analyze:
type: mvn
target: provider/dataset-aws/pom.xml
path: .
- name: dataset-ibm
type: mvn
target: provider/dataset-ibm/pom.xml
path: .
- name: dataset-gcp
type: mvn
target: provider/dataset-gcp/pom.xml
path: .
......@@ -6,6 +6,14 @@ variables:
IBM_BUILD_SUBDIR: provider/dataset-ibm
# --- osdu-gcp specific variables ---
OSDU_GCP_SERVICE: dataset
OSDU_GCP_VENDOR: gcp
OSDU_GCP_APPLICATION_NAME: os-dataset
OSDU_GCP_FILE_DMS_BUCKET: "file-dms-bucket"
OSDU_GCP_EXPIRATION_DAYS: 1
OSDU_GCP_ENV_VARS: AUTHORIZE_API=$OSDU_GCP_ENTITLEMENTS_URL,GCP_STORAGE_API=$OSDU_GCP_STORAGE_URL,GCP_SCHEMA_API=$OSDU_GCP_SCHEMA_URL/api/schema-service/v1,FILE_DMS_BUCKET=$OSDU_GCP_FILE_DMS_BUCKET,EXPIRATION_DAYS=$OSDU_GCP_EXPIRATION_DAYS,REDIS_GROUP_HOST=$REDIS_GROUP_HOST,REDIS_GROUP_PORT=$OSDU_GCP_REDIS_SEARCH_PORT --vpc-connector=$OSDU_GCP_VPC_CONNECTOR
include:
- project: "osdu/platform/ci-cd-pipelines"
file: "standard-setup.yml"
......@@ -25,9 +33,23 @@ include:
- project: "osdu/platform/ci-cd-pipelines"
file: "scanners/gitlab-ultimate.yml"
- project: 'osdu/platform/ci-cd-pipelines'
file: 'cloud-providers/osdu-gcp-cloudrun.yml'
# disable the eslint scanner
# I think this is being generated from the presence of an HTML file, but there
# is no javascript to scan, so the job isn't helpful and just gets in the way
eslint-sast:
rules:
- when: never
osdu-gcp-test:
variables:
STORAGE_BASE_URL: $OSDU_GCP_STORAGE_URL
LEGAL_BASE_URL: $OSDU_GCP_LEGALTAG_API/
DATASET_BASE_URL: $OSDU_GCP_DATASET_URL
SCHEMA_API: $OSDU_GCP_SCHEMA_URL/api/schema-service/v1
PROVIDER_KEY: GCP
KIND_SUBTYPE: DatasetTest
LEGAL_TAG: datatest-int-tag
......@@ -7,14 +7,22 @@
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<gitlab-server>community-maven-via-job-token</gitlab-server>
<repo.releases.id>community-maven-repo</repo.releases.id>
<publish.snapshots.id>community-maven-via-job-token</publish.snapshots.id>
<publish.releases.id>community-maven-via-job-token</publish.releases.id>
<repo.releases.url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</repo.releases.url>
<publish.snapshots.url>https://community.opengroup.org/api/v4/projects/378/packages/maven
</publish.snapshots.url>
<publish.releases.url>https://community.opengroup.org/api/v4/projects/378/packages/maven
</publish.releases.url>
</properties>
</profile>
<profile>
<!-- This profile uses a personal token to authenticate, which is useful for local or manual runs. The
presence of the COMMUNITY_MAVEN_TOKEN variable triggers this and overrides the CI Token
based authentication -->
presence of the COMMUNITY_MAVEN_TOKEN variable triggers this and overrides the CI Token
based authentication -->
<id>GitLab-Authenticate-With-Private-Token</id>
<activation>
<property>
......@@ -22,7 +30,15 @@
</property>
</activation>
<properties>
<gitlab-server>community-maven-via-private-token</gitlab-server>
<repo.releases.id>community-maven-repo</repo.releases.id>
<publish.snapshots.id>community-maven-via-private-token</publish.snapshots.id>
<publish.releases.id>community-maven-via-private-token</publish.releases.id>
<repo.releases.url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</repo.releases.url>
<publish.snapshots.url>https://community.opengroup.org/api/v4/projects/378/packages/maven
</publish.snapshots.url>
<publish.releases.url>https://community.opengroup.org/api/v4/projects/378/packages/maven
</publish.releases.url>
</properties>
</profile>
</profiles>
......@@ -61,5 +77,6 @@
<environment>AZURE</environment>
</configuration>
</server>
</servers>
</settings>
This diff is collapsed.
......@@ -63,6 +63,10 @@ This section describes the deployment process for each cloud provider.
### AWS
TODO
### GCP
All documentation for the GCP implementation of `os-dataset` can be found [here](./provider/dataset-gcp/README.md)
## Running integration tests
......@@ -73,6 +77,10 @@ Integration tests are located in a separate project for each cloud in the ```tes
TODO
### GCP
All documentation for the GCP implementation of `os-dataset` can be found [here](./provider/dataset-gcp/README.md)
## License
Copyright © 2021 Amazon Web Services
......
......@@ -162,24 +162,6 @@
</dependency>
</dependencies>
<repositories>
<repository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</url>
</repository>
</repositories>
<distributionManagement>
<repository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/projects/378/packages/maven</url>
</repository>
<snapshotRepository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/projects/378/packages/maven</url>
</snapshotRepository>
</distributionManagement>
<build>
<plugins>
<plugin>
......
......@@ -15,9 +15,7 @@
package org.opengroup.osdu.dataset.di;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.RequiredArgsConstructor;
import org.opengroup.osdu.core.common.entitlements.EntitlementsAPIConfig;
import org.opengroup.osdu.core.common.entitlements.EntitlementsFactory;
import org.opengroup.osdu.core.common.entitlements.IEntitlementsFactory;
......@@ -27,10 +25,10 @@ import org.springframework.beans.factory.config.AbstractFactoryBean;
import org.springframework.stereotype.Component;
@Component
@RequiredArgsConstructor
public class EntitlementsClientFactory extends AbstractFactoryBean<IEntitlementsFactory> {
private final ObjectMapper objectMapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false);
private final HttpResponseBodyMapper bodyMapper = new HttpResponseBodyMapper(objectMapper);
private final HttpResponseBodyMapper bodyMapper;
@Value("${AUTHORIZE_API}")
private String AUTHORIZE_API;
......
......@@ -15,9 +15,7 @@
package org.opengroup.osdu.dataset.di;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.RequiredArgsConstructor;
import org.opengroup.osdu.core.common.http.json.HttpResponseBodyMapper;
import org.opengroup.osdu.dataset.schema.ISchemaFactory;
import org.opengroup.osdu.dataset.schema.SchemaAPIConfig;
......@@ -27,10 +25,10 @@ import org.springframework.beans.factory.config.AbstractFactoryBean;
import org.springframework.stereotype.Component;
@Component
@RequiredArgsConstructor
public class SchemaClientFactory extends AbstractFactoryBean<ISchemaFactory> {
private final ObjectMapper objectMapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false);
private final HttpResponseBodyMapper bodyMapper = new HttpResponseBodyMapper(objectMapper);
private final HttpResponseBodyMapper bodyMapper;
@Value("${SCHEMA_API}")
private String SCHEMA_API;
......
......@@ -15,9 +15,7 @@
package org.opengroup.osdu.dataset.di;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.RequiredArgsConstructor;
import org.opengroup.osdu.core.common.http.json.HttpResponseBodyMapper;
import org.opengroup.osdu.core.common.storage.IStorageFactory;
import org.opengroup.osdu.core.common.storage.StorageAPIConfig;
......@@ -27,10 +25,10 @@ import org.springframework.beans.factory.config.AbstractFactoryBean;
import org.springframework.stereotype.Component;
@Component
@RequiredArgsConstructor
public class StorageClientFactory extends AbstractFactoryBean<IStorageFactory> {
private final ObjectMapper objectMapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false);
private final HttpResponseBodyMapper bodyMapper = new HttpResponseBodyMapper(objectMapper);
private final HttpResponseBodyMapper bodyMapper;
@Value("${STORAGE_API}")
private String STORAGE_API;
......@@ -44,7 +42,7 @@ public class StorageClientFactory extends AbstractFactoryBean<IStorageFactory> {
protected IStorageFactory createInstance() throws Exception {
return new StorageFactory(StorageAPIConfig
.builder()
.rootUrl(STORAGE_API)
.rootUrl(STORAGE_API)
.build(),
bodyMapper);
}
......
......@@ -16,11 +16,14 @@ package org.opengroup.osdu.dataset.model.response;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import org.opengroup.osdu.core.common.model.storage.Record;
import lombok.Data;
@Data
@JsonIgnoreProperties(ignoreUnknown = true)
public class GetCreateUpdateDatasetRegistryResponse {
private List<Record> datasetRegistries;
......
......@@ -18,12 +18,8 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.inject.Inject;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.RequiredArgsConstructor;
import org.opengroup.osdu.core.common.http.json.HttpResponseBodyMapper;
import org.opengroup.osdu.core.common.http.json.HttpResponseBodyParsingException;
import org.opengroup.osdu.core.common.model.http.AppException;
......@@ -44,6 +40,7 @@ import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
@Service
@RequiredArgsConstructor
public class DatasetDmsServiceImpl implements DatasetDmsService {
@Inject
......@@ -55,12 +52,7 @@ public class DatasetDmsServiceImpl implements DatasetDmsService {
@Inject
private IDatasetDmsServiceMap dmsServiceMap;
private ObjectMapper jsonObjectMapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false);
private final HttpResponseBodyMapper bodyMapper = new HttpResponseBodyMapper(jsonObjectMapper);
public DatasetDmsServiceImpl() {
}
private final HttpResponseBodyMapper bodyMapper;
@Override
public GetDatasetStorageInstructionsResponse getStorageInstructions(String kindSubType) {
......
......@@ -18,12 +18,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.inject.Inject;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.RequiredArgsConstructor;
import org.opengroup.osdu.core.common.http.json.HttpResponseBodyMapper;
import org.opengroup.osdu.core.common.http.json.HttpResponseBodyParsingException;
import org.opengroup.osdu.core.common.model.http.AppException;
......@@ -45,6 +41,7 @@ import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
@Service
@RequiredArgsConstructor
public class DatasetRegistryServiceImpl implements DatasetRegistryService {
/**
......@@ -57,8 +54,7 @@ public class DatasetRegistryServiceImpl implements DatasetRegistryService {
*/
final String DATASET_KIND_REGEX = "^[\\w\\-\\.]+:[\\w\\-\\.]+:dataset--+[\\w\\-\\.]+:[0-9]+.[0-9]+.[0-9]+$";
private final ObjectMapper objectMapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false);
private final HttpResponseBodyMapper bodyMapper = new HttpResponseBodyMapper(objectMapper);
private final HttpResponseBodyMapper bodyMapper;
@Inject
private DpsHeaders headers;
......@@ -164,23 +160,23 @@ public class DatasetRegistryServiceImpl implements DatasetRegistryService {
}
//this should be in os-core-common, but placing here until it's able to be put inside the Record class
private boolean isOsduRecordIdValid(String recordId, String tenant, String kind) {
private boolean isOsduRecordIdValid(String recordId, String tenant, String kind) {
//Check format and tenant
if (!Record.isRecordIdValid(recordId, tenant, kind))
return false;
//id should be split by colons. ex: tenant:groupType--individualType:uniqueId
String[] recordIdSplitByColon = recordId.split(":");
//make sure groupType/individualType is correct
String[] kindSplitByColon = kind.split(":");
String kindSubType = kindSplitByColon[2]; //grab GroupType/IndividualType
if (!recordIdSplitByColon[1].equalsIgnoreCase(kindSubType))
return false;
return true;
return true;
}
private boolean validateDatasets(ISchemaService schemaService, List<Record> datasets) {
......
......@@ -56,7 +56,7 @@
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-dependencies</artifactId>
<version>2.1.16.RELEASE</version>
<version>2.3.4.RELEASE</version>
<type>pom</type>
<scope>import</scope>
</dependency>
......@@ -102,30 +102,51 @@
</dependency>
</dependencies>
<profiles>
<profile>
<id>Default</id>
<activation>
<property>
<name>!repo.releases.id</name>
</property>
</activation>
<properties>
<repo.releases.id>community-maven-repo</repo.releases.id>
<publish.snapshots.id>community-maven-via-job-token</publish.snapshots.id>
<publish.releases.id>community-maven-via-job-token</publish.releases.id>
<repo.releases.url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven
</repo.releases.url>
<publish.snapshots.url>https://community.opengroup.org/api/v4/projects/118/packages/maven
</publish.snapshots.url>
<publish.releases.url>https://community.opengroup.org/api/v4/projects/118/packages/maven
</publish.releases.url>
</properties>
</profile>
</profiles>
<repositories>
<repository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</url>
<id>${repo.releases.id}</id>
<url>${repo.releases.url}</url>
</repository>
</repositories>
<distributionManagement>
<distributionManagement>
<repository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/projects/118/packages/maven</url>
<id>${publish.releases.id}</id>
<url>${publish.releases.url}</url>
</repository>
<snapshotRepository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/projects/118/packages/maven</url>
</snapshotRepository>
</distributionManagement>
<id>${publish.snapshots.id}</id>
<url>${publish.snapshots.url}</url>
</snapshotRepository>
</distributionManagement>
<modules>
<module>dataset-core</module>
<module>provider/dataset-aws</module>
<module>provider/dataset-ibm</module>
<module>provider/dataset-gcp</module>
</modules>
</project>
......@@ -167,22 +167,4 @@
</plugins>
</build>
<repositories>
<repository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</url>
</repository>
</repositories>
<distributionManagement>
<repository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/projects/378/packages/maven</url>
</repository>
<snapshotRepository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/projects/378/packages/maven</url>
</snapshotRepository>
</distributionManagement>
</project>
# Dataset registry service
## Getting Started
These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system.
### Prerequisites
Pre-requisites
* GCloud SDK with java (latest version)
* JDK 8
* Lombok 1.16 or later
* Maven
### Installation
In order to run the service locally or remotely, you will need to have the following environment variables defined.
| name | value | description | sensitive? | source |
| --- | --- | --- | --- | --- |
| `GCP_SCHEMA_API` | ex `https://os-schema-jvmvia5dea-uc.a.run.app/api/schema-service/v1` | Schema API endpoint | no | output of infrastructure deployment |
| `GCP_STORAGE_API` | ex `https://os-storage-jvmvia5dea-uc.a.run.app/api/storage/v2` | Storage API endpoint | no | output of infrastructure deployment |
| `AUTHORIZE_API` | ex `https://os-entitlements-gcp-jvmvia5dea-uc.a.run.app/entitlements/v1` | Entitlements API endpoint | no | output of infrastructure deployment |
| `FILE_DMS_BUCKET` | ex `file-dms-bucket` | File bucket name postfix (full name represent by project-id + partition-id + GCP_FILE_DMS_BUCKET ex `osdu-cicd-epam-opendes-file-dms-bucket`) | no | output of infrastructure deployment |
| `EXPIRATION_DAYS` | ex `1` | expiration for signed urls & connection strings | no | |
| `REDIS_GROUP_HOST` | ex `127.0.0.1` | Redis host for groups | no | https://console.cloud.google.com/memorystore/redis/instances |
| `REDIS_GROUP_PORT` | ex `1111` | Redis port | no | https://console.cloud.google.com/memorystore/redis/instances |
### Run Locally
Check that maven is installed:
```bash
$ mvn --version
Apache Maven 3.6.0
Maven home: /usr/share/maven
Java version: 1.8.0_212, vendor: AdoptOpenJDK, runtime: /usr/lib/jvm/jdk8u212-b04/jre
...
```
You may need to configure access to the remote maven repository that holds the OSDU dependencies. This file should live within `~/.mvn/community-maven.settings.xml`:
```bash
$ cat ~/.m2/settings.xml
<?xml version="1.0" encoding="UTF-8"?>
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
<servers>
<server>
<id>community-maven-via-private-token</id>
<!-- Treat this auth token like a password. Do not share it with anyone, including Microsoft support. -->
<!-- The generated token expires on or before 11/14/2019 -->
<configuration>
<httpHeaders>
<property>
<name>Private-Token</name>
<value>${env.COMMUNITY_MAVEN_TOKEN}</value>
</property>
</httpHeaders>
</configuration>
</server>
</servers>
</settings>
```
* Update the Google cloud SDK to the latest version:
```bash
gcloud components update
```
* Set Google Project Id:
```bash
gcloud config set project <YOUR-PROJECT-ID>
```
* Perform a basic authentication in the selected project:
```bash
gcloud auth application-default login
```
* Navigate to Dataset service root folder and run:
```bash
mvn clean install
```
* If you wish to build the project without running tests
```bash
mvn clean install -DskipTests
```
After configuring your environment as specified above, you can follow these steps to build and run the application. These steps should be invoked from the *repository root.*
```bash
cd provider/dataset-gcp && mvn spring-boot:run
```
## Testing
### Running E2E Tests
This section describes how to run cloud OSDU E2E tests (testing/dataset-test-gcp).
You will need to have the following environment variables defined.
| name | value | description | sensitive? | source |
| --- | --- | --- | --- | --- |
| `DOMAIN` | ex `osdu-gcp.go3-nrg.projects.epam.com` | - | no | - |
| `STORAGE_BASE_URL` | ex `https://os-storage-jvmvia5dea-uc.a.run.app/api/storage/v2/` | Storage API endpoint | no | output of infrastructure deployment |
| `LEGAL_BASE_URL` | ex `https://os-legal-jvmvia5dea-uc.a.run.app/api/legal/v1/` | Legal API endpoint | no | output of infrastructure deployment |
| `DATASET_BASE_URL` | ex `http://localhost:8080/api/dataset/v1/` | Dataset API endpoint | no | output of infrastructure deployment |
| `SCHEMA_API` | ex `https://os-schema-jvmvia5dea-uc.a.run.app/api/schema-service/v1` | Schema API endpoint | no | output of infrastructure deployment |
| `PROVIDER_KEY` | `GCP` | required for response verification | no | - |
| `INTEGRATION_TEST_AUDIENCE` | ex `****.apps.googleusercontent.com;` | Client application ID | yes | https://console.cloud.google.com/apis/credentials |
| `INTEGRATION_TESTER` | `********` | Service account for API calls, passed as a filename or JSON content, plain or Base64 encoded. Note: this user must have entitlements configured already | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
| `GCP_DEPLOY_FILE` | `********` | Service account for test data tear down, passed as a filename or JSON content, plain or Base64 encoded. Must have cloud storage role configured | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
| `TENANT_NAME` | `opendes` | Tenant name | no | - |
| `KIND_SUBTYPE` | `DatasetTest` | Kind subtype that will be used in int tests, schema creation automated , result kind will be `TENANT_NAME::wks-test:dataset--FileCollection.KIND_SUBTYPE:1.0.0`| no | - |
| `LEGAL_TAG` | `public-usa-dataset-1` | Legal tag name, if tag with that name doesn't exist then it will be created during preparing step | no | - |
| `GCLOUD_PROJECT` | `osdu-cicd-epam` | Project id | no | - |
**Entitlements configuration for integration accounts**
| INTEGRATION_TESTER |
| --- |
| users<br/>service.entitlements.user<br/>service.storage.admin<br/>service.legal.user<br/>service.search.user<br/>service.delivery.viewer |
**Cloud roles configuration for integration accounts**
| GCP_DEPLOY_FILE|
| --- |
| storage.admin access to the Google Cloud Storage |
Execute following command to build code and run all the integration tests:
```bash
# Note: this assumes that the environment variables for integration tests as outlined
# above are already exported in your environment.
# build + install integration test core
$ (cd testing/dataset-test-core/ && mvn clean install)
```
```bash
# build + run GCP integration tests.
$ (cd testing/dataset-test-gcp/ && mvn clean test)
```
## Deployment
* To deploy into Cloud run, please, use this documentation:
https://cloud.google.com/run/docs/quickstarts/build-and-deploy
* To deploy into App Engine, please, use this documentation:
https://cloud.google.com/appengine/docs/flexible/java/quickstart
## License
Copyright 2021 Google LLC
Copyright 2021 EPAM Systems, Inc