Skip to content

GitLab

Commit e3034e71 authored by Aalekh Jain's avatar Aalekh Jain
Browse files

Merge branch 'master' into Code-Coverage

parents cdddb4c1 9b3224c7
Pipeline #86882 failed with stages
in 41 minutes and 51 seconds
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
View file @ e3034e71
......@@ -9,7 +9,7 @@ variables:
IBM_BUILD_SUBDIR: provider/dataset-ibm
IBM_INT_TEST_SUBDIR: testing/dataset-test-ibm
# --- osdu-gcp specific variables ---
# --- osdu-gcp specific variables ---
OSDU_GCP_SERVICE: dataset
OSDU_GCP_VENDOR: gcp
OSDU_GCP_APPLICATION_NAME: dataset
......@@ -18,7 +18,7 @@ variables:
OSDU_GCP_TEST_SUBDIR: testing/$OSDU_GCP_SERVICE-test-$OSDU_GCP_VENDOR
OSDU_GCP_HELM_PACKAGE_CHARTS: "devops/gcp/deploy devops/gcp/configmap"
OSDU_GCP_HELM_CONFIG_SERVICE_VARS: "--set data.google_audiences=$GOOGLE_AUDIENCE --set data.redis_group_host=$REDIS_GROUP_HOST --set data.file_dms_bucket=$OSDU_GCP_FILE_DMS_BUCKET --set data.expiration_days=$OSDU_GCP_EXPIRATION_DAYS"
OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS: "--set data.image=$CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA --set data.serviceAccountName=workload-identity-dataset"
OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS: "--set data.image=$CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA --set data.serviceAccountName=$OSDU_GCP_SERVICE-k8s"
OSDU_GCP_HELM_CONFIG_SERVICE: dataset-config
OSDU_GCP_HELM_DEPLOYMENT_SERVICE: dataset-deploy
......@@ -37,14 +37,14 @@ include:
- project: "osdu/platform/ci-cd-pipelines"
file: "scanners/fossa-maven.yml"
- project: 'osdu/platform/ci-cd-pipelines'
file: 'cloud-providers/aws-global.yml'
- project: "osdu/platform/ci-cd-pipelines"
file: "cloud-providers/aws-global.yml"
- project: 'osdu/platform/ci-cd-pipelines'
file: 'cloud-providers/aws-maven.yml'
- project: "osdu/platform/ci-cd-pipelines"
file: "cloud-providers/aws-maven.yml"
- project: 'osdu/platform/ci-cd-pipelines'
file: 'cloud-providers/azure.yml'
- project: "osdu/platform/ci-cd-pipelines"
file: "cloud-providers/azure.yml"
- project: "osdu/platform/ci-cd-pipelines"
file: "cloud-providers/ibm.yml"
......@@ -52,9 +52,8 @@ include:
- project: "osdu/platform/ci-cd-pipelines"
file: "scanners/gitlab-ultimate.yml"
- project: 'osdu/platform/ci-cd-pipelines'
file: 'cloud-providers/osdu-gcp-gke.yml'
- project: "osdu/platform/ci-cd-pipelines"
file: "cloud-providers/osdu-gcp-gke.yml"
# disable the eslint scanner
# I think this is being generated from the presence of an HTML file, but there
......
dataset-core/pom.xml
View file @ e3034e71
......@@ -34,12 +34,8 @@
<java.version>1.8</java.version>
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
<springfox-version>2.7.0</springfox-version>
<tomcat_embed_core_version>9.0.54</tomcat_embed_core_version>
<log4j2.version>2.17.0</log4j2.version>
<!--
<springfox-version>2.9.2</springfox-version>
-->
</properties>
<dependencies>
......@@ -176,18 +172,6 @@
<version>3.0.0</version>
<scope>test</scope>
</dependency>
<!-- swagger dependencies -->
<dependency>
<groupId>io.springfox</groupId>
<artifactId>springfox-swagger2</artifactId>
<version>${springfox-version}</version>
</dependency>
<dependency>
<groupId>io.springfox</groupId>
<artifactId>springfox-swagger-ui</artifactId>
<version>${springfox-version}</version>
</dependency>
</dependencies>
<build>
......
dataset-core/src/main/java/org/opengroup/osdu/dataset/swagger/SpringfoxSwaggerHostResolver.java 0 → 100644
View file @ e3034e71
package org.opengroup.osdu.dataset.swagger;
import io.swagger.v3.oas.models.OpenAPI;
import io.swagger.v3.oas.models.servers.Server;
import org.springframework.core.Ordered;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
import springfox.documentation.oas.web.OpenApiTransformationContext;
import springfox.documentation.oas.web.WebMvcOpenApiTransformationFilter;
import springfox.documentation.spi.DocumentationType;
import javax.servlet.http.HttpServletRequest;
@Component
@Order(Ordered.LOWEST_PRECEDENCE)
public class SpringfoxSwaggerHostResolver implements WebMvcOpenApiTransformationFilter {
@Override
public boolean supports(DocumentationType delimiter) {
return delimiter == DocumentationType.OAS_30;
}
@Override
public OpenAPI transform(OpenApiTransformationContext<HttpServletRequest> context) {
OpenAPI swagger = context.getSpecification();
Server server = swagger.getServers().get(0);
if (server.getUrl().contains(":443")) {
// via the gateway
server.setUrl(server.getUrl().replace(":443",""));
}
return swagger;
}
}
\ No newline at end of file
dataset-core/src/main/java/org/opengroup/osdu/dataset/swagger/SwaggerDocumentationConfig.java
View file @ e3034e71
......@@ -19,25 +19,28 @@ import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Profile;
import springfox.documentation.builders.ParameterBuilder;
import springfox.documentation.builders.PathSelectors;
import springfox.documentation.builders.RequestHandlerSelectors;
import springfox.documentation.schema.ModelRef;
import springfox.documentation.builders.RequestParameterBuilder;
import springfox.documentation.oas.annotations.EnableOpenApi;
import springfox.documentation.service.ApiKey;
import springfox.documentation.service.AuthorizationScope;
import springfox.documentation.service.ParameterType;
import springfox.documentation.service.RequestParameter;
import springfox.documentation.service.SecurityReference;
import springfox.documentation.spi.DocumentationType;
import springfox.documentation.spi.service.contexts.SecurityContext;
import springfox.documentation.spring.web.plugins.Docket;
import springfox.documentation.swagger2.annotations.EnableSwagger2;
import springfox.documentation.service.Parameter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@Configuration
@EnableSwagger2
@EnableOpenApi
@Profile("!noswagger")
public class SwaggerDocumentationConfig {
public static final String AUTHORIZATION_HEADER = "Authorization";
......@@ -45,26 +48,23 @@ public class SwaggerDocumentationConfig {
@Bean
public Docket api() {
ParameterBuilder builder = new ParameterBuilder();
List<Parameter> parameters = new ArrayList<>();
RequestParameterBuilder builder = new RequestParameterBuilder();
List<RequestParameter> parameters = new ArrayList<>();
builder.name(DpsHeaders.DATA_PARTITION_ID)
.description("tenant")
.defaultValue("common")
.modelRef(new ModelRef("string"))
.parameterType("header")
.in(ParameterType.HEADER)
.required(true)
.build();
parameters.add(builder.build());
builder.name("frame-of-reference")
.description("reference")
.defaultValue("none")
.modelRef(new ModelRef("string"))
.parameterType("header")
.in(ParameterType.HEADER)
.required(true)
.build();
parameters.add(builder.build());
return new Docket(DocumentationType.SWAGGER_2)
.globalOperationParameters(parameters)
return new Docket(DocumentationType.OAS_30)
.globalRequestParameters(parameters)
.select()
.apis(RequestHandlerSelectors.basePackage("org.opengroup.osdu.dataset.api"))
.build()
......@@ -73,13 +73,13 @@ public class SwaggerDocumentationConfig {
}
private ApiKey apiKey() {
return new ApiKey("JWT", AUTHORIZATION_HEADER, "header");
return new ApiKey(AUTHORIZATION_HEADER, AUTHORIZATION_HEADER, "header");
}
private SecurityContext securityContext() {
return SecurityContext.builder()
.securityReferences(defaultAuth())
.forPaths(PathSelectors.regex(DEFAULT_INCLUDE_PATTERN))
.operationSelector(o -> PathSelectors.regex(DEFAULT_INCLUDE_PATTERN).test(o.requestMappingPattern()))
.build();
}
......@@ -89,6 +89,6 @@ public class SwaggerDocumentationConfig {
AuthorizationScope[] authorizationScopes =
new AuthorizationScope[]{authorizationScope};
return Collections.singletonList(
new SecurityReference("JWT", authorizationScopes));
new SecurityReference(AUTHORIZATION_HEADER, authorizationScopes));
}
}
pom.xml
View file @ e3034e71
......@@ -33,6 +33,7 @@
<commons-codec.version>1.14</commons-codec.version>
<nimbusds.version>7.9</nimbusds.version>
<log4j2.version>2.17.0</log4j2.version>
<springfox.version>3.0.0</springfox.version>
</properties>
<licenses>
......@@ -111,6 +112,13 @@
<version>1.18.8</version>
<scope>provided</scope>
</dependency>
<!-- swagger dependency -->
<dependency>
<groupId>io.springfox</groupId>
<artifactId>springfox-boot-starter</artifactId>
<version>${springfox.version}</version>
</dependency>
</dependencies>
<profiles>
......
provider/dataset-azure/src/main/java/org/opengroup/osdu/dataset/provider/azure/security/AADSecurityConfig.java
View file @ e3034e71
......@@ -42,11 +42,13 @@ public class AADSecurityConfig extends WebSecurityConfigurerAdapter {
.authorizeRequests()
.antMatchers("/", "/index.html",
"/v2/api-docs",
"/v3/api-docs",
"/configuration/ui",
"/swagger-resources/**",
"/configuration/security",
"/swagger",
"/swagger-ui.html",
"/swagger-ui/**",
"/info",
"/webjars/**").permitAll()
.anyRequest().authenticated()
......
provider/dataset-gcp/src/main/java/org/opengroup/osdu/dataset/provider/gcp/config/GcpConfigProperties.java
View file @ e3034e71
......@@ -31,9 +31,4 @@ public class GcpConfigProperties {
private String redisGroupHost;
private Integer redisGroupPort;
private String dmsApiBase;
/**
* TODO: remove property and fix PgOsmConfigurationProperties.partitionPropertiesPrefix after
* Partition service migration to new approach (https://jiraeu.epam.com/browse/GONRG-3881)
*/
private String partitionPropertiesDelimiter;
}
provider/dataset-gcp/src/main/java/org/opengroup/osdu/dataset/provider/gcp/mappers/osm/config/PgOsmConfigurationProperties.java
View file @ e3034e71
......@@ -29,7 +29,7 @@ import org.springframework.context.annotation.Configuration;
@Getter
@Setter
public class PgOsmConfigurationProperties {
private String partitionPropertiesPrefix = "osm_postgres";
private String partitionPropertiesPrefix = "osm.postgres";
private Integer maximumPoolSize = 40;
private Integer minimumIdle = 0;
private Integer idleTimeout = 30000;
......
provider/dataset-gcp/src/main/java/org/opengroup/osdu/dataset/provider/gcp/mappers/osm/config/PgTenantDestinationResolver.java
View file @ e3034e71
......@@ -35,7 +35,6 @@ import org.opengroup.osdu.core.gcp.osm.model.Destination;
import org.opengroup.osdu.core.gcp.osm.translate.TranslatorRuntimeException;
import org.opengroup.osdu.core.gcp.osm.translate.postgresql.PgDestinationResolution;
import org.opengroup.osdu.core.gcp.osm.translate.postgresql.PgDestinationResolver;
import org.opengroup.osdu.dataset.provider.gcp.config.GcpConfigProperties;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.jdbc.DataSourceBuilder;
import org.springframework.stereotype.Component;
......@@ -46,13 +45,12 @@ import org.springframework.stereotype.Component;
@Slf4j
public class PgTenantDestinationResolver implements PgDestinationResolver {
private static final String DATASOURCE = "datasource";
private static final String DATASOURCE = ".datasource.";
private static final String DRIVER_CLASS_NAME = "org.postgresql.Driver";
private final DpsHeaders dpsHeaders;
private final IPartitionFactory partitionFactory;
private final PgOsmConfigurationProperties pgOsmConfigurationProperties;
private final GcpConfigProperties gcpConfigProperties;
private final Map<String, DataSource> dataSourceCache = new HashMap<>();
@Override
......@@ -78,13 +76,12 @@ public class PgTenantDestinationResolver implements PgDestinationResolver {
Map<String, Property> partitionProperties = partitionInfo.getProperties();
String prefix = pgOsmConfigurationProperties.getPartitionPropertiesPrefix();
String delimiter = gcpConfigProperties.getPartitionPropertiesDelimiter();
String url = getPartitionProperty(partitionId, partitionProperties,
prefix + delimiter + DATASOURCE + delimiter + "url");
prefix + DATASOURCE + "url");
String username = getPartitionProperty(partitionId, partitionProperties,
prefix + delimiter + DATASOURCE + delimiter + "username");
prefix + DATASOURCE + "username");
String password = getPartitionProperty(partitionId, partitionProperties,
prefix + delimiter + DATASOURCE + delimiter + "password");
prefix + DATASOURCE + "password");
dataSource = DataSourceBuilder.create()
.driverClassName(DRIVER_CLASS_NAME)
......
provider/dataset-gcp/src/main/resources/application.properties
View file @ e3034e71
......@@ -56,5 +56,4 @@ ENTITLEMENTS_PATH=/api/entitlements/v2
ENTITLEMENTS_API=${ENTITLEMENTS_HOST}${ENTITLEMENTS_PATH}
osmDriver=datastore
obmDriver=gcs
PARTITION_PROPERTIES_DELIMITER=_
\ No newline at end of file
obmDriver=gcs
\ No newline at end of file
provider/dataset-ibm/pom.xml
View file @ e3034e71
......@@ -36,9 +36,27 @@
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-codec</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
<version>8.10.0</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec</artifactId>
<version>4.1.68.Final</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-to-slf4j</artifactId>
<version>${log4j2.version}</version>
......@@ -73,9 +91,17 @@
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-to-slf4j</artifactId>
</exclusion>
</exclusions>
<exclusion>
<groupId>org.apache.tomcat.embed</groupId>
<artifactId>tomcat-embed-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.tomcat.embed</groupId>
<artifactId>tomcat-embed-core</artifactId>
<version>9.0.54</version>
</dependency>
<!-- Third party Apache 2.0 license packages -->
<dependency>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment