Skip to content
GitLab
Commit 27efe467 authored by Dmitriy Rudko's avatar Dmitriy Rudko 💬
Browse files

Merge branch 'gcp-migrate-to-core-common-0.3.5' into 'master' 

GCP migrate Workflow to core-common 0.3.5

See merge request !8
parents 4bfb16a6 3d4b0ec5
Pipeline #5455 passed with stages
in 18 minutes and 29 seconds
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
View file @ 27efe467
variables:
OSDU_GCP_BUILD_SUBDIR: provider/file-gcp-datastore
# OSDU_GCP_INT_TEST_SUBDIR: testing/file-test-core
OSDU_GCP_SERVICE: workflow
OSDU_GCP_RECORDS_ROOT_URL: https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2
OSDU_GCP_VENDOR: gcp-datastore
OSDU_GCP_NAMESPACE: osdu-namespace
OSDU_GCP_APPLICATION_NAME: os-workflow
OSDU_GCP_ENV_VARS: SPRING_CLOUD_GCP_DATASTORE_NAMESPACE=$OSDU_GCP_NAMESPACE,GCP_AIRFLOW_URL=$OSDU_GCP_AIRFLOW_URL,OSDU_ENTITLEMENTS_URL=$OSDU_GCP_ENTITLEMENTS_URL --vpc-connector=$OSDU_GCP_VPC_CONNECTOR
OSDU_GCP_ENVIRONMENT: dev
OSDU_GCP_PORT: 8080
include:
- project: 'osdu/platform/ci-cd-pipelines'
ref: 'master'
......@@ -18,3 +30,7 @@ include:
- project: 'osdu/platform/ci-cd-pipelines'
ref: 'master'
file: 'publishing/pages.yml'
- project: "osdu/platform/ci-cd-pipelines"
ref: 'osdu-gcp-cloudrun'
file: 'cloud-providers/osdu-gcp-cloudrun.yml'
.mvn/community-maven.settings.xml
View file @ 27efe467
......@@ -7,7 +7,13 @@
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<gitlab-server>community-maven-via-job-token</gitlab-server>
<repo.releases.id>community-maven-repo</repo.releases.id>
<publish.snapshots.id>community-maven-via-job-token</publish.snapshots.id>
<publish.releases.id>community-maven-via-job-token</publish.releases.id>
<repo.releases.url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</repo.releases.url>
<publish.snapshots.url>https://community.opengroup.org/api/v4/projects/146/packages/maven</publish.snapshots.url>
<publish.releases.url>https://community.opengroup.org/api/v4/projects/146/packages/maven</publish.releases.url>
</properties>
</profile>
......@@ -22,7 +28,13 @@
</property>
</activation>
<properties>
<gitlab-server>community-maven-via-private-token</gitlab-server>
<repo.releases.id>community-maven-repo</repo.releases.id>
<publish.snapshots.id>community-maven-via-private-token</publish.snapshots.id>
<publish.releases.id>community-maven-via-private-token</publish.releases.id>
<repo.releases.url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</repo.releases.url>
<publish.snapshots.url>https://community.opengroup.org/api/v4/projects/146/packages/maven</publish.snapshots.url>
<publish.releases.url>https://community.opengroup.org/api/v4/projects/146/packages/maven</publish.releases.url>
</properties>
</profile>
</profiles>
......@@ -51,5 +63,17 @@
</httpHeaders>
</configuration>
</server>
<server>
<id>azure-auth</id>
<configuration>
<tenant>${env.AZURE_TENANT_ID}</tenant>
<client>${env.AZURE_CLIENT_ID}</client>
<key>${env.AZURE_CLIENT_SECRET}</key>
<environment>AZURE</environment>
</configuration>
</server>
</servers>
</settings>
pom.xml
View file @ 27efe467
......@@ -23,7 +23,7 @@
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.2.2.RELEASE</version>
<version>2.2.5.RELEASE</version>
<relativePath/>
</parent>
......@@ -47,13 +47,13 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<java.version>1.8</java.version>
<spring-boot.version>2.2.4.RELEASE</spring-boot.version>
<spring-cloud.version>Hoxton.SR2</spring-cloud.version>
<spring-boot.version>2.2.5.RELEASE</spring-boot.version>
<spring-cloud.version>Hoxton.SR3</spring-cloud.version>
<javax.inject.version>1</javax.inject.version>
<org.mapstruct.version>1.3.1.Final</org.mapstruct.version>
<maven-surefire-plugin.version>3.0.0-M4</maven-surefire-plugin.version>
<os-core-common.version>0.0.13-gcp</os-core-common.version>
<lombok.version>1.18.0</lombok.version>
<os-core-common.version>0.3.5</os-core-common.version>
<springfox.version>2.9.2</springfox.version>
</properties>
<licenses>
......@@ -77,6 +77,25 @@
</dependencyManagement>
<dependencies>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<scope>provided</scope>
</dependency>
<!--Swagger-->
<dependency>
<groupId>io.springfox</groupId>
<artifactId>springfox-swagger2</artifactId>
<version>${springfox.version}</version>
</dependency>
<dependency>
<groupId>io.springfox</groupId>
<artifactId>springfox-swagger-ui</artifactId>
<version>${springfox.version}</version>
</dependency>
<!-- OSDU -->
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
......@@ -100,6 +119,11 @@
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-json</artifactId>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
......@@ -284,19 +308,19 @@
<repositories>
<repository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</url>
<id>${repo.releases.id}</id>
<url>${repo.releases.url}</url>
</repository>
</repositories>
<distributionManagement>
<repository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/projects/146/packages/maven</url>
<id>${publish.releases.id}</id>
<url>${publish.releases.url}</url>
</repository>
<snapshotRepository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/projects/146/packages/maven</url>
<id>${publish.snapshots.id}</id>
<url>${publish.snapshots.url}</url>
</snapshotRepository>
</distributionManagement>
......
provider/workflow-azure/pom.xml
View file @ 27efe467
......@@ -109,35 +109,18 @@
</dependency>
</dependencies>
<repositories>
<repository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</url>
</repository>
</repositories>
<distributionManagement>
<repository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/projects/44/packages/maven</url>
</repository>
<snapshotRepository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/projects/44/packages/maven</url>
</snapshotRepository>
</distributionManagement>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
<configuration>
<classifier>spring-boot</classifier>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
<configuration>
<classifier>spring-boot</classifier>
<mainClass>org.opengroup.osdu.workflow.provider.azure.WorkflowApplication</mainClass>
</configuration>
</execution>
......
provider/workflow-gcp-datastore/cloudbuild/Dockerfile.cloudbuild 0 → 100644
View file @ 27efe467
# Use the official AdoptOpenJDK for a base image.
# https://hub.docker.com/_/openjdk
FROM openjdk:8-slim
WORKDIR /app
ARG PROVIDER_NAME
ENV PROVIDER_NAME $PROVIDER_NAME
# Copy the jar to the production image from the builder stage.
COPY provider/workflow-${PROVIDER_NAME}/target/workflow-${PROVIDER_NAME}-*.jar workflow-${PROVIDER_NAME}.jar
# Run the web service on container startup.
CMD java -Djava.security.egd=file:/dev/./urandom -Dserver.port=${PORT} -jar /app/workflow-${PROVIDER_NAME}.jar
provider/workflow-gcp-datastore/src/main/resources/application.yaml → provider/workflow-gcp-datastore/cloudbuild/cloudbuild.yaml
View file @ 27efe467
......@@ -14,20 +14,17 @@
# limitations under the License.
#
spring:
cloud:
gcp:
datastore:
# Local emulator
#project-id: a2ba07aca58-energy-osdu
#emulator:
#enabled: true
#port: 8484
#consistency: 1
steps:
- name: 'gcr.io/cloud-builders/docker'
args: [
'build',
'--build-arg', 'PROVIDER_NAME=${_PROVIDER_NAME}',
'--build-arg', 'PORT=${_PORT}',
'-t', 'gcr.io/$PROJECT_ID/os-workflow/workflow-${_PROVIDER_NAME}:${_SHORT_SHA}',
'-t', 'gcr.io/$PROJECT_ID/os-workflow/workflow-${_PROVIDER_NAME}:latest',
'-f', 'provider/workflow-${_PROVIDER_NAME}/cloudbuild/Dockerfile.cloudbuild',
'.'
]
images:
- 'gcr.io/$PROJECT_ID/os-workflow/workflow-${_PROVIDER_NAME}'
# TODO: remove it after defined tenant info and auth
namespace: odes-namespace
gcp:
airflow:
url: ${GCP_AIRFLOW_URL}
provider/workflow-gcp-datastore/pom.xml
View file @ 27efe467
......@@ -86,24 +86,6 @@
</dependencies>
<repositories>
<repository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</url>
</repository>
</repositories>
<distributionManagement>
<repository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/projects/146/packages/maven</url>
</repository>
<snapshotRepository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/projects/146/packages/maven</url>
</snapshotRepository>
</distributionManagement>
<build>
<plugins>
<plugin>
......
provider/workflow-gcp-datastore/src/main/java/org/opengroup/osdu/workflow/provider/gcp/service/GcpAuthenticationService.java deleted 100644 → 0
View file @ 4bfb16a6
/*
* Copyright 2020 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.workflow.provider.gcp.service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.opengroup.osdu.core.common.exception.UnauthorizedException;
import org.opengroup.osdu.workflow.provider.interfaces.IAuthenticationService;
import org.springframework.stereotype.Service;
@Service
@Slf4j
@RequiredArgsConstructor
public class GcpAuthenticationService implements IAuthenticationService {
@Override
public void checkAuthentication(String authorizationToken, String partitionID) {
log.debug("Start checking authentication. Authorization: {}, partitionID: {}",
authorizationToken, partitionID);
checkPreconditions(authorizationToken, partitionID);
// TODO: add check of user permissions
log.debug("Finished checking authentication.");
}
private void checkPreconditions(String authorizationToken, String partitionID) {
if (authorizationToken == null) {
throw new UnauthorizedException("Missing authorization token");
}
if (partitionID == null) {
throw new UnauthorizedException("Missing partitionID");
}
}
}
provider/workflow-gcp-datastore/src/main/java/org/opengroup/osdu/workflow/provider/gcp/service/SubmitIngestServiceImpl.java
View file @ 27efe467
......@@ -16,9 +16,15 @@
package org.opengroup.osdu.workflow.provider.gcp.service;
import static java.lang.String.format;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpResponse;
import com.google.api.client.http.HttpResponseException;
import java.io.IOException;
import java.util.Collections;
import java.util.Map;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.IOUtils;
......@@ -28,13 +34,6 @@ import org.opengroup.osdu.workflow.provider.gcp.property.AirflowProperties;
import org.opengroup.osdu.workflow.provider.interfaces.ISubmitIngestService;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.util.Collections;
import java.util.Map;
import static java.lang.String.format;
import static java.nio.charset.StandardCharsets.UTF_8;
@Service
@RequiredArgsConstructor
@Slf4j
......
provider/workflow-gcp-datastore/src/test/java/org/opengroup/osdu/workflow/provider/gcp/service/GcpAuthenticationServiceTest.java deleted 100644 → 0
View file @ 4bfb16a6
/*
* Copyright 2020 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.workflow.provider.gcp.service;
import org.junit.jupiter.api.DisplayNameGeneration;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.junit.jupiter.MockitoExtension;
import org.opengroup.osdu.core.common.exception.UnauthorizedException;
import org.opengroup.osdu.workflow.ReplaceCamelCase;
import org.opengroup.osdu.workflow.provider.interfaces.IAuthenticationService;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.catchThrowable;
@ExtendWith(MockitoExtension.class)
@DisplayNameGeneration(ReplaceCamelCase.class)
class GcpAuthenticationServiceTest {
private IAuthenticationService authenticationService = new GcpAuthenticationService();
private static final String AUTHORIZATION_TOKEN = "authToken";
private static final String PARTITION = "partition";
@Test
void shouldCheckAuthentication() {
// when
Throwable thrown = catchThrowable(() -> authenticationService.checkAuthentication(
AUTHORIZATION_TOKEN, PARTITION));
// then
assertThat(thrown).isNull();
}
@Test
void shouldThrowWhenNothingIsSpecified() {
// when
Throwable thrown = catchThrowable(() -> authenticationService.checkAuthentication(
null, null));
// then
assertThat(thrown)
.isInstanceOf(UnauthorizedException.class);
}
@Test
void shouldThrowWhenOnlyTokenIsSpecified() {
// when
Throwable thrown = catchThrowable(() -> authenticationService.checkAuthentication(
AUTHORIZATION_TOKEN, null));
// then
assertThat(thrown)
.isInstanceOf(UnauthorizedException.class)
.hasMessage("Missing partitionID");
}
@Test
void shouldThrowWhenOnlyPartitionIsSpecified() {
// when
Throwable thrown = catchThrowable(() -> authenticationService.checkAuthentication(
null, PARTITION));
// then
assertThat(thrown)
.isInstanceOf(UnauthorizedException.class)
.hasMessage("Missing authorization token");
}
}
provider/workflow-gcp/pom.xml
View file @ 27efe467
......@@ -87,24 +87,6 @@
</dependencies>
<repositories>
<repository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</url>
</repository>
</repositories>
<distributionManagement>
<repository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/projects/146/packages/maven</url>
</repository>
<snapshotRepository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/projects/146/packages/maven</url>
</snapshotRepository>
</distributionManagement>
<build>
<plugins>
<plugin>
......
provider/workflow-gcp/src/main/java/org/opengroup/osdu/workflow/provider/gcp/service/GcpAuthenticationService.java deleted 100644 → 0
View file @ 4bfb16a6
/*
* Copyright 2020 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.workflow.provider.gcp.service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.opengroup.osdu.core.common.exception.UnauthorizedException;
import org.opengroup.osdu.workflow.provider.interfaces.IAuthenticationService;
import org.springframework.stereotype.Service;
@Service
@Slf4j
@RequiredArgsConstructor
public class GcpAuthenticationService implements IAuthenticationService {
@Override
public void checkAuthentication(String authorizationToken, String partitionID) {
log.debug("Start checking authentication. Authorization: {}, partitionID: {}",
authorizationToken, partitionID);
checkPreconditions(authorizationToken, partitionID);
// TODO: add check of user permissions
log.debug("Finished checking authentication.");
}
private void checkPreconditions(String authorizationToken, String partitionID) {
if (authorizationToken == null) {
throw new UnauthorizedException("Missing authorization token");
}
if (partitionID == null) {
throw new UnauthorizedException("Missing partitionID");
}
}
}
provider/workflow-gcp/src/main/resources/application.yaml deleted 100644 → 0
View file @ 4bfb16a6
#
# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
gcp:
airflow:
url: ${GCP_AIRFLOW_URL}
provider/workflow-gcp/src/test/java/org/opengroup/osdu/workflow/provider/gcp/service/GcpAuthenticationServiceTest.java deleted 100644 → 0
View file @ 4bfb16a6
/*
* Copyright 2020 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.workflow.provider.gcp.service;
import org.junit.jupiter.api.DisplayNameGeneration;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.junit.jupiter.MockitoExtension;
import org.opengroup.osdu.core.common.exception.UnauthorizedException;
import org.opengroup.osdu.workflow.ReplaceCamelCase;
import org.opengroup.osdu.workflow.provider.interfaces.IAuthenticationService;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.catchThrowable;
@ExtendWith(MockitoExtension.class)
@DisplayNameGeneration(ReplaceCamelCase.class)
class GcpAuthenticationServiceTest {
private IAuthenticationService authenticationService = new GcpAuthenticationService();