Commit 3c705f52 authored by Abhishek Kumar (SLB)'s avatar Abhishek Kumar (SLB)
Browse files
parents 253e3b99 9010f743
......@@ -58,7 +58,10 @@ include:
file: "scanners/fossa-maven.yml"
- project: "osdu/platform/ci-cd-pipelines"
file: "cloud-providers/aws.yml"
file: "cloud-providers/aws-global.yml"
- project: "osdu/platform/ci-cd-pipelines"
file: "cloud-providers/aws-maven.yml"
- project: "osdu/platform/ci-cd-pipelines"
file: "cloud-providers/ibm-java-git.yml"
......
This diff is collapsed.
......@@ -14,6 +14,10 @@ osdu-gcp-bootstrap:
- export BEARER_TOKEN=$(python $OSDU_GCP_DEPLOYMENTS_SUBDIR/Token.py)
- python $OSDU_GCP_DEPLOYMENTS_SCRIPTS_SUBDIR/DeploySharedSchemas.py -u $OSDU_GCP_SCHEMA_URL/schema
rules:
- if: '$OSDU_GCP == "true" && $CI_COMMIT_BRANCH =~ /^release/'
when: never
- if: '$OSDU_GCP == "true" && $CI_COMMIT_TAG'
when: never
- if: '$OSDU_GCP == "true"'
when: always
......@@ -32,7 +36,7 @@ osdu-gcp-bootstrap-dev2:
- export BEARER_TOKEN=$(python $OSDU_GCP_DEPLOYMENTS_SUBDIR/Token.py)
- python $OSDU_GCP_DEPLOYMENTS_SCRIPTS_SUBDIR/DeploySharedSchemas.py -u $OSDU_GCP_SCHEMA_URL/schema
rules:
- if: '$OSDU_GCP == "true" && $CI_COMMIT_BRANCH =~ /^release$/'
- if: '$OSDU_GCP == "true" && $CI_COMMIT_BRANCH =~ /^release/'
when: always
- if: '$OSDU_GCP == "true" && $CI_COMMIT_TAG'
when: always
......@@ -42,19 +46,19 @@ osdu-gcp-bootstrap-dev2:
osdu-gcp-test:
needs: ["osdu-gcp-bootstrap"]
# Rules should be deleted here after common pipeline is updated with this rules
rules:
- if: '$OSDU_GCP == "true" && $OSDU_GCP_INT_TEST_TYPE != "python" && $CI_COMMIT_BRANCH =~ /^release/'
when: never
- if: '$OSDU_GCP == "true" && $OSDU_GCP_INT_TEST_TYPE != "python" && $CI_COMMIT_TAG'
when: never
- if: '$OSDU_GCP == "true" && $OSDU_GCP_INT_TEST_TYPE != "python"'
when: always
osdu-gcp-dev2-test:
needs: ["osdu-gcp-bootstrap-dev2"]
rules:
- if: '$OSDU_GCP == "true" && $CI_COMMIT_BRANCH =~ /^release$/'
when: always
- if: '$OSDU_GCP == "true" && $CI_COMMIT_TAG'
when: always
# The variable DEV2="true" should be specified manually in GitLab before running a pipeline to test this job against a protected branch
- if: '$OSDU_GCP == "true" && $DEV2 == "true"'
when: always
containerize_osdu_gcp_schema_bootstrap:
osdu-gcp-containerize-bootstrap-gitlab:
stage: containerize
image: docker:19.03.15
tags: ["osdu-medium"]
......@@ -65,10 +69,7 @@ containerize_osdu_gcp_schema_bootstrap:
- docker build -t $CI_REGISTRY_IMAGE/osdu-gcp-bootstrap:$CI_COMMIT_SHORT_SHA $EXTRA_DOCKER_TAG --file devops/$OSDU_GCP_VENDOR/bootstrap-osdu-module/Dockerfile .
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- docker push $CI_REGISTRY_IMAGE/osdu-gcp-bootstrap
only:
refs:
- master
- /.*bootstrap.*/
- tags
variables:
- $OSDU_GCP == 'true'
rules:
- if: '$OSDU_GCP == "true" && $CI_COMMIT_BRANCH =~ /^master$/'
- if: '$OSDU_GCP == "true" && $CI_COMMIT_BRANCH =~ /.*bootstrap.*/'
- if: '$OSDU_GCP == "true" && $CI_COMMIT_TAG'
......@@ -7,8 +7,10 @@
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<os-core-common.version>0.12.3</os-core-common.version>
<os-core-common.version>0.13.0</os-core-common.version>
<log4j2.version>2.17.1</log4j2.version>
<springfox.version>3.0.0</springfox.version>
<json-smart.version>2.4.7</json-smart.version>
</properties>
<licenses>
......@@ -28,6 +30,11 @@
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>net.minidev</groupId>
<artifactId>json-smart</artifactId>
<version>${json-smart.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
......@@ -42,7 +49,7 @@
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-schema</artifactId>
<version>0.13.0-SNAPSHOT</version>
<version>0.14.0-SNAPSHOT</version>
<packaging>pom</packaging>
<name>os-schema</name>
<description>os schema service </description>
......
......@@ -18,7 +18,7 @@
<parent>
<artifactId>os-schema</artifactId>
<groupId>org.opengroup.osdu</groupId>
<version>0.13.0-SNAPSHOT</version>
<version>0.14.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
......@@ -46,7 +46,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-schema-core</artifactId>
<version>0.13.0-SNAPSHOT</version>
<version>0.14.0-SNAPSHOT</version>
</dependency>
<!-- AWS managed packages -->
<dependency>
......@@ -78,7 +78,7 @@
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
<artifactId>os-core-lib-aws</artifactId>
<version>0.12.3</version>
<version>0.13.0</version>
</dependency>
<dependency>
......
......@@ -18,21 +18,21 @@
<parent>
<artifactId>os-schema</artifactId>
<groupId>org.opengroup.osdu</groupId>
<version>0.13.0-SNAPSHOT</version>
<version>0.14.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>os-schema-azure</artifactId>
<version>0.13.0-SNAPSHOT</version>
<version>0.14.0-SNAPSHOT</version>
<description>Azure related implementation staff.</description>
<packaging>jar</packaging>
<properties>
<azure.version>2.1.7</azure.version>
<osdu.corelibazure.version>0.12.4</osdu.corelibazure.version>
<osdu.oscorecommon.version>0.12.3</osdu.oscorecommon.version>
<osdu.os-schema-core.version>0.13.0-SNAPSHOT</osdu.os-schema-core.version>
<osdu.corelibazure.version>0.13.0-rc6</osdu.corelibazure.version>
<osdu.oscorecommon.version>0.13.0</osdu.oscorecommon.version>
<osdu.os-schema-core.version>0.14.0-SNAPSHOT</osdu.os-schema-core.version>
<mockito.version>1.10.19</mockito.version>
<cucumber.version>5.4.0</cucumber.version>
<nimbus-jose-jwt-azure.version>8.20.2</nimbus-jose-jwt-azure.version>
......@@ -64,6 +64,31 @@
<artifactId>spring-boot-starter-actuator</artifactId>
<version>2.4.5</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
<version>${log4j2.version}</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-to-slf4j</artifactId>
<version>${log4j2.version}</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>${log4j2.version}</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>${log4j2.version}</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-jul</artifactId>
<version>${log4j2.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
......
......@@ -3,7 +3,7 @@
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-schema</artifactId>
<version>0.13.0-SNAPSHOT</version>
<version>0.14.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
......@@ -20,10 +20,31 @@
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
<version>4.9.2</version>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>osm</artifactId>
<version>0.13.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>oqm</artifactId>
<version>0.13.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>obm</artifactId>
<version>0.13.1-SNAPSHOT</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.google.cloud/google-cloud-storage -->
<dependency>
<groupId>com.google.cloud</groupId>
......@@ -41,7 +62,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-schema-core</artifactId>
<version>0.13.0-SNAPSHOT</version>
<version>0.14.0-SNAPSHOT</version>
</dependency>
<dependency>
......@@ -67,7 +88,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-gcp</artifactId>
<version>0.9.0</version>
<version>0.13.0-rc3</version>
</dependency>
<dependency>
......
/*
* Copyright 2020-2022 Google LLC
* Copyright 2020-2022 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.schema.configuration;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
@Configuration
@ConditionalOnProperty(value = "obmDriver", havingValue = "minio")
@ConfigurationProperties("obm.minio")
@Getter
@Setter
public class MinioObmConfiguration {
private String partitionPropertiesPrefix = "obm.minio";
}
/*
* Copyright 2020-2022 Google LLC
* Copyright 2020-2022 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.schema.configuration;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
@Configuration
@ConditionalOnProperty(value = "osmDriver", havingValue = "postgres")
@ConfigurationProperties(prefix = "osm.postgres")
@Getter
@Setter
public class PostgresOsmConfiguration {
private String partitionPropertiesPrefix = "osm.postgres";
private Integer maximumPoolSize = 40;
private Integer minimumIdle = 0;
private Integer idleTimeout = 30000;
private Integer maxLifetime = 1800000;
private Integer connectionTimeout = 30000;
}
/*
* Copyright 2020-2022 Google LLC
* Copyright 2020-2022 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.schema.configuration;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
@Configuration
@ConditionalOnProperty(value = "oqmDriver", havingValue = "rabbitmq")
@ConfigurationProperties("oqm.rabbitmq")
@Getter
@Setter
public class RabbitMqOqmConfiguration {
private String partitionPropertiesPrefix = "oqm.rabbitmq";
}
/*
* Copyright 2020-2022 Google LLC
* Copyright 2020-2022 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.schema.configuration.mapper;
import com.google.cloud.Timestamp;
import com.google.cloud.datastore.Blob;
import com.google.cloud.datastore.Key;
import com.google.common.collect.ImmutableList;
import org.opengroup.osdu.core.gcp.osm.persistence.IdentityTranslator;
import org.opengroup.osdu.core.gcp.osm.translate.Instrumentation;
import org.opengroup.osdu.core.gcp.osm.translate.TypeMapper;
import org.opengroup.osdu.schema.constants.SchemaConstants;
import org.opengroup.osdu.schema.model.Authority;
import org.opengroup.osdu.schema.model.EntityType;
import org.opengroup.osdu.schema.model.SchemaRequest;
import org.opengroup.osdu.schema.model.Source;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
import java.util.Collections;
import java.util.HashMap;
import static org.springframework.beans.factory.config.BeanDefinition.SCOPE_SINGLETON;
@Component
@Scope(SCOPE_SINGLETON)
@ConditionalOnProperty("osmDriver")
public class TypeMapperImpl extends TypeMapper {
public TypeMapperImpl(){
super(ImmutableList.of(
new Instrumentation<>(
Authority.class,
new HashMap<String, String>(){{
put("authorityId", "name");
}},
new HashMap<>(),
new IdentityTranslator<>(
Authority::getAuthorityId,
(a, o) -> a.setAuthorityId(((Key) o).getName())
),
Collections.singletonList("name")
),
new Instrumentation<>(
EntityType.class,
new HashMap<String, String>(){{
put("entityTypeId", "name");
}},
new HashMap<>(),
new IdentityTranslator<>(
EntityType::getEntityTypeId,
(et, o) -> et.setEntityTypeId(((Key) o).getName())
),
Collections.singletonList("name")
),
new Instrumentation<>(
SchemaRequest.class,
new HashMap<String, String>(){{
put("schemaVersionMajor", SchemaConstants.MAJOR_VERSION);
put("schemaVersionMinor", SchemaConstants.MINOR_VERSION);
put("schemaVersionPatch", SchemaConstants.PATCH_VERSION);
}},
new HashMap<String, Class<?>>() {{
put("schema", Blob.class);
put("dateCreated", Timestamp.class);
}},
new IdentityTranslator<>(
(r) -> r.getSchemaInfo().getSchemaIdentity().getId(),
(r, o) -> r.getSchemaInfo().getSchemaIdentity().setId(((Key)o).getName())
),
Collections.singletonList("")
),
new Instrumentation<>(
Source.class,
new HashMap<String, String>() {{
put("sourceId", "name");
}},
new HashMap<>(),
new IdentityTranslator<>(
Source::getSourceId,
(s, o) -> s.setSourceId(((Key) o).getName())
),
Collections.singletonList("name")
)
));
}
}
/*
* Copyright 2020-2022 Google LLC
* Copyright 2020-2022 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.schema.destination.provider;
import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
import org.opengroup.osdu.core.gcp.oqm.model.OqmDestination;
public interface DestinationProvider<DestinationT> {
/**
* The method used only for destinations without kind (i.e. {@link OqmDestination})
* @param partitionId the id of partition
* @return the destination for OSDU Mappers
*/
DestinationT getDestination(String partitionId);
/**
* The method used only for destinations with kind (i.e. {@link org.opengroup.osdu.core.gcp.osm.model.Destination})
* You can also pass empty string for the kind name if you want to use this method for building other destinations
* @param partitionId the id of partition
* @param kindName the name of the kind
* @return the destination for OSDU Mappers
*/
DestinationT getDestination(String partitionId, String kindName);
/**
* The method used if tenantInfo already acknowledged and there is no need to call TenantFactory to get the info
*
* @param tenantInfo tenant info got from somewhere
* @param kindName the name of the kind
* @return the destination for OSDU Mappers
*/
DestinationT getDestination(TenantInfo tenantInfo, String kindName);
/**
* The method is used only for custom namespace and kind usage
*
* @param partitionId partitionId for destination
* @param namespace custom namespace
* @param kindName the name of the kind
* @return the destination for OSDU Mappers
*/
DestinationT getDestination(String partitionId, String namespace, String kindName);
}
/*
* Copyright 2020-2022 Google LLC
* Copyright 2020-2022 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.schema.destination.provider;
import lombok.extern.slf4j.Slf4j;
import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
import org.opengroup.osdu.core.gcp.osm.model.Kind;
import org.opengroup.osdu.core.gcp.osm.model.Namespace;
import org.opengroup.osdu.schema.destination.provider.model.DestinationInstructions;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@Service
@Slf4j
public abstract class DestinationProviderImpl<DestinationT> implements DestinationProvider<DestinationT> {
private final ITenantFactory tenantFactory;
@Autowired
public DestinationProviderImpl(ITenantFactory tenantFactory) {
this.tenantFactory = tenantFactory;
}
@Override
public DestinationT getDestination(String partitionId) {
TenantInfo tenantInfo = tenantFactory.getTenantInfo(partitionId);
return getDestination(tenantInfo, "");
}
@Override
public DestinationT getDestination(String partitionId, String kindName) {
TenantInfo tenantInfo = tenantFactory.getTenantInfo(partitionId);
return getDestination(tenantInfo, kindName);
}
@Override
public DestinationT getDestination(TenantInfo tenantInfo, String kindName) {
log.debug("Providing destination for the tenant: " + tenantInfo.getName());
String partitionId = tenantInfo.getDataPartitionId();