Commit 1b949e78 authored by Maksim Malkov's avatar Maksim Malkov Committed by Swapnil
Browse files

Add GSM

parent aa52a2b7
......@@ -116,6 +116,7 @@ The following software have components provided under the terms of this license:
- JSON Web Token support for the JVM (from https://repo1.maven.org/maven2/io/jsonwebtoken/jjwt)
- JSON library from Android SDK (from http://developer.android.com/sdk)
- JSONassert (from https://github.com/skyscreamer/JSONassert)
- JSR107 API and SPI (from https://github.com/jsr107/jsr107spec)
- Jackson 2 extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-jackson2)
- Jackson dataformat: CBOR (from http://github.com/FasterXML/jackson-dataformats-binary)
- Jackson dataformat: CBOR (from http://github.com/FasterXML/jackson-dataformats-binary)
......@@ -160,8 +161,8 @@ The following software have components provided under the terms of this license:
- Lucene Grouping (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-grouping)
- Lucene Highlighter (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-highlighter)
- Lucene Join (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-join)
- Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-memory)
- Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-backward-codecs)
- Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-memory)
- Lucene Miscellaneous (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-misc)
- Lucene Queries (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-queries)
- Lucene QueryParsers (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-queryparser)
......@@ -267,6 +268,8 @@ The following software have components provided under the terms of this license:
- Spring Web MVC (from https://github.com/spring-projects/spring-framework)
- Spring Web MVC (from https://github.com/spring-projects/spring-framework)
- T-Digest (from https://github.com/tdunning/t-digest)
- Vavr (from http://vavr.io)
- Vavr Match (from http://vavr.io)
- Woodstox (from https://github.com/FasterXML/woodstox)
- Zipkin Core Library (from https://repo1.maven.org/maven2/io/zipkin/zipkin2/zipkin)
- Zipkin Reporter: Core (from https://repo1.maven.org/maven2/io/zipkin/reporter2/zipkin-reporter)
......@@ -328,6 +331,14 @@ The following software have components provided under the terms of this license:
- proto-google-common-protos (from https://github.com/googleapis/api-client-staging)
- proto-google-iam-v1 (from https://github.com/googleapis/java-iam/proto-google-iam-v1)
- rank-eval (from https://github.com/elastic/elasticsearch)
- resilience4j (from https://resilience4j.readme.io)
- resilience4j (from https://resilience4j.readme.io)
- resilience4j (from https://resilience4j.readme.io)
- resilience4j (from https://github.com/resilience4j/resilience4j)
- resilience4j (from https://github.com/resilience4j/resilience4j)
- resilience4j (from https://resilience4j.readme.io)
- resilience4j (from https://resilience4j.readme.io)
- resilience4j (from https://resilience4j.readme.io)
- rest (from https://github.com/elastic/elasticsearch)
- rest-high-level (from https://github.com/elastic/elasticsearch)
- rxjava (from https://github.com/ReactiveX/RxJava)
......@@ -340,6 +351,7 @@ The following software have components provided under the terms of this license:
- spring-boot-dependencies (from https://spring.io/projects/spring-boot)
- spring-boot-starter (from https://spring.io/projects/spring-boot)
- spring-boot-starter-actuator (from https://spring.io/projects/spring-boot)
- spring-boot-starter-aop (from https://spring.io/projects/spring-boot)
- spring-boot-starter-json (from https://spring.io/projects/spring-boot)
- spring-boot-starter-logging (from https://spring.io/projects/spring-boot)
- spring-boot-starter-security (from https://spring.io/projects/spring-boot)
......@@ -382,8 +394,8 @@ The following software have components provided under the terms of this license:
- swagger-annotations (from https://repo1.maven.org/maven2/io/swagger/core/v3/swagger-annotations)
- swagger-annotations (from https://repo1.maven.org/maven2/io/swagger/swagger-annotations)
- swagger-jaxrs (from )
- swagger-models (from https://repo1.maven.org/maven2/io/swagger/core/v3/swagger-models)
- swagger-models (from https://repo1.maven.org/maven2/io/swagger/swagger-models)
- swagger-models (from https://repo1.maven.org/maven2/io/swagger/core/v3/swagger-models)
- tomcat (from http://tomcat.apache.org/)
- tomcat-annotations-api (from https://tomcat.apache.org/)
- tomcat-api (from http://tomcat.apache.org/)
......@@ -532,6 +544,7 @@ EPL-1.0
========================================================================
The following software have components provided under the terms of this license:
- AspectJ Weaver (from https://www.eclipse.org/aspectj/)
- JUnit Jupiter (Aggregator) (from https://junit.org/junit5/)
- JUnit Jupiter (Aggregator) (from https://junit.org/junit5/)
- JUnit Jupiter API (from https://junit.org/junit5/)
......@@ -869,4 +882,3 @@ The following software have components provided under the terms of this license:
- Jakarta XML Binding API (from https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api)
- Spongy Castle (from http://rtyley.github.io/spongycastle/)
......@@ -49,7 +49,7 @@
<javax.inject.version>1</javax.inject.version>
<org.mapstruct.version>1.3.1.Final</org.mapstruct.version>
<maven-surefire-plugin.version>3.0.0-M4</maven-surefire-plugin.version>
<os-core-common.version>0.11.0-rc4</os-core-common.version>
<os-core-common.version>0.11.0-SNAPSHOT</os-core-common.version>
<springfox.version>3.0.0</springfox.version>
</properties>
......@@ -214,18 +214,6 @@
</compilerArgs>
</configuration>
</plugin>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<executions>
<execution>
<id>build-info</id>
<goals>
<goal>build-info</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</pluginManagement>
......@@ -301,26 +289,6 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>pl.project13.maven</groupId>
<artifactId>git-commit-id-plugin</artifactId>
<version>4.0.5</version>
<executions>
<execution>
<goals>
<goal>revision</goal>
</goals>
</execution>
</executions>
<configuration>
<verbose>true</verbose>
<dateFormat>yyyy-MM-dd'T'HH:mm:ssZ</dateFormat>
<generateGitPropertiesFile>true</generateGitPropertiesFile>
<generateGitPropertiesFilename>
${project.build.outputDirectory}/git.properties
</generateGitPropertiesFilename>
</configuration>
</plugin>
</plugins>
</build>
......
......@@ -31,7 +31,7 @@
<properties>
<azure.version>2.1.7</azure.version>
<osdu.corelibazure.version>0.10.0</osdu.corelibazure.version>
<osdu.corelibazure.version>0.11.0-rc4</osdu.corelibazure.version>
<azure.appservice.resourcegroup />
<azure.appservice.plan />
<azure.appservice.appname />
......
......@@ -14,6 +14,10 @@
package org.opengroup.osdu.workflow.provider.azure.config;
import com.azure.cosmos.CosmosClient;
import com.azure.cosmos.CosmosClientBuilder;
import com.azure.security.keyvault.secrets.SecretClient;
import org.opengroup.osdu.azure.KeyVaultFacade;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
......@@ -35,6 +39,16 @@ public class AzureBootstrapConfig {
return keyVaultURL;
}
/*This is done to support Single partition support for slb. Once implementation is complete for multi-partition we can remove this method */
@Bean
public CosmosClient buildCosmosClient(SecretClient kv) {
final String partitionId = getPartitionId();
final String cosmosEndpoint = KeyVaultFacade.getSecretWithValidation(kv, String.format("%s-cosmos-endpoint", partitionId));
final String cosmosPrimaryKey = KeyVaultFacade.getSecretWithValidation(kv, String.format("%s-cosmos-primary-key", partitionId));
return new CosmosClientBuilder().endpoint(cosmosEndpoint).key(cosmosPrimaryKey).buildClient();
}
public String getPartitionId() {
return this.partitionId;
}
......
package org.opengroup.osdu.workflow.provider.azure.config;
import com.azure.cosmos.CosmosClient;
import org.opengroup.osdu.azure.cosmosdb.ICosmosClientFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
@Component
@Primary
public class SinglePartitionCosmosClientFactory implements ICosmosClientFactory {
@Autowired
CosmosClient cosmosClient;
@Override
public CosmosClient getClient(final String s) {
return cosmosClient;
}
//Adding dummy class to implement interface.
@Override
public CosmosClient getSystemClient() {
return cosmosClient;
}
}
\ No newline at end of file
package org.opengroup.osdu.workflow.provider.azure.gsm;
import com.microsoft.azure.eventgrid.models.EventGridEvent;
import lombok.RequiredArgsConstructor;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.joda.time.DateTime;
import org.opengroup.osdu.azure.eventgrid.EventGridTopicStore;
import org.opengroup.osdu.core.common.exception.CoreException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.status.Message;
import org.opengroup.osdu.core.common.status.IEventPublisher;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
@Slf4j
@Service
@RequiredArgsConstructor
public class AzureEventGridPublisher implements IEventPublisher {
private static final String STATUS_CHANGED = "status-changed";
private static final String EVENT_DATA_VERSION = "1.0";
@Setter
@Value("${azure.eventGrid.topicName}")
private String topicName;
@Setter
@Value("${azure.eventGrid.enabled}")
private Boolean isEventGridEnabled;
private final EventGridTopicStore eventGridTopicStore;
@Override
public void publish(Message[] messages, Map<String, String> attributesMap) throws CoreException {
validateEventGrid();
validateInput(messages, attributesMap);
Map<String, Object> message = createMessageMap(messages, attributesMap);
List<EventGridEvent> eventsList = createEventGridEventList(message);
String dataPartitionId = attributesMap.get(DpsHeaders.DATA_PARTITION_ID);
eventGridTopicStore.publishToEventGridTopic(dataPartitionId, topicName, eventsList);
String correlationId = attributesMap.get(DpsHeaders.CORRELATION_ID);
String logMsg = String.format(
"Event published successfully to topic='%s' with dataPartitionId='%s' and correlationId='%s'.",
topicName, dataPartitionId, correlationId
);
log.info(logMsg);
}
private List<EventGridEvent> createEventGridEventList(Map<String, Object> message) {
String messageId = UUID.randomUUID().toString();
EventGridEvent eventGridEvent = new EventGridEvent(messageId, STATUS_CHANGED, message, STATUS_CHANGED,
DateTime.now(), EVENT_DATA_VERSION);
return Collections.singletonList(eventGridEvent);
}
private Map<String, Object> createMessageMap(Message[] messages, Map<String, String> attributesMap) {
String dataPartitionId = attributesMap.get(DpsHeaders.DATA_PARTITION_ID);
String correlationId = attributesMap.get(DpsHeaders.CORRELATION_ID);
Map<String, Object> message = new HashMap<>();
message.put("data", messages);
message.put(DpsHeaders.DATA_PARTITION_ID, dataPartitionId);
message.put(DpsHeaders.CORRELATION_ID, correlationId);
return message;
}
private void validateEventGrid() throws CoreException {
if (!isEventGridEnabled) {
throw new CoreException("Event grid is not enabled");
}
}
private void validateInput(Message[] messages, Map<String, String> attributesMap) throws CoreException {
validateMsg(messages);
validateAttributesMap(attributesMap);
}
private void validateMsg(Message[] messages) throws CoreException {
if (messages == null || messages.length == 0) {
throw new CoreException("Nothing in message to publish");
}
}
private void validateAttributesMap(Map<String, String> attributesMap) throws CoreException {
if (attributesMap == null || attributesMap.isEmpty()) {
throw new CoreException("data-partition-id and correlation-id are required to publish status event");
} else if (attributesMap.get(DpsHeaders.DATA_PARTITION_ID) == null) {
throw new CoreException("data-partition-id is required to publish status event");
} else if (attributesMap.get(DpsHeaders.CORRELATION_ID) == null) {
throw new CoreException("correlation-id is required to publish status event");
}
}
}
......@@ -3,10 +3,11 @@ package org.opengroup.osdu.workflow.provider.azure.repository;
import com.azure.cosmos.CosmosException;
import com.azure.cosmos.models.SqlParameter;
import com.azure.cosmos.models.SqlQuerySpec;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.azure.cosmosdb.CosmosStore;
import org.opengroup.osdu.azure.query.CosmosStorePageRequest;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.workflow.exception.WorkflowRunNotFoundException;
......@@ -18,9 +19,8 @@ import org.opengroup.osdu.workflow.provider.azure.consts.WorkflowRunConstants;
import org.opengroup.osdu.workflow.provider.azure.model.WorkflowRunDoc;
import org.opengroup.osdu.workflow.provider.azure.utils.CursorUtils;
import org.opengroup.osdu.workflow.provider.interfaces.IWorkflowRunRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.stereotype.Component;
import org.springframework.stereotype.Repository;
import java.util.ArrayList;
import java.util.List;
......@@ -29,28 +29,20 @@ import java.util.Optional;
import static org.opengroup.osdu.workflow.model.WorkflowStatusType.getCompletedStatusTypes;
@Component
@Slf4j
@Repository
@RequiredArgsConstructor
public class WorkflowRunRepository implements IWorkflowRunRepository {
private static final String LOGGER_NAME = WorkflowRunRepository.class.getName();
private final CosmosConfig cosmosConfig;
@Autowired
private CosmosConfig cosmosConfig;
private final CosmosStore cosmosStore;
@Autowired
private CosmosStore cosmosStore;
private final DpsHeaders dpsHeaders;
@Autowired
private DpsHeaders dpsHeaders;
private final CursorUtils cursorUtils;
@Autowired
private JaxRsDpsLog logger;
@Autowired
private CursorUtils cursorUtils;
@Autowired
private WorkflowTasksSharingRepository workflowTasksSharingRepository;
private final WorkflowTasksSharingRepository workflowTasksSharingRepository;
@Override
public WorkflowRun saveWorkflowRun(final WorkflowRun workflowRun) {
......@@ -72,7 +64,7 @@ public class WorkflowRunRepository implements IWorkflowRunRepository {
if (!workflowRunDoc.isPresent()) {
final String errorMessage = String.format("WorkflowRun: %s for Workflow: %s doesn't exist",
runId, workflowName);
logger.error(LOGGER_NAME, errorMessage);
log.error(errorMessage);
throw new WorkflowRunNotFoundException(errorMessage);
} else {
return buildWorkflowRun(workflowRunDoc.get());
......@@ -160,7 +152,7 @@ public class WorkflowRunRepository implements IWorkflowRunRepository {
@Override
public WorkflowRun updateWorkflowRun(final WorkflowRun workflowRun) {
logger.info(LOGGER_NAME, String.format("Update called for workflow id: %s, run id: %s",
log.info(String.format("Update called for workflow id: %s, run id: %s",
workflowRun.getWorkflowId(), workflowRun.getRunId()));
final WorkflowRunDoc workflowRunDoc = buildWorkflowRunDoc(workflowRun);
cosmosStore.replaceItem(dpsHeaders.getPartitionId(),
......@@ -169,14 +161,16 @@ public class WorkflowRunRepository implements IWorkflowRunRepository {
workflowRunDoc.getId(),
workflowRunDoc.getPartitionKey(),
workflowRunDoc);
logger.info(LOGGER_NAME, String.format("Updated workflowRun with id : %s of workflowId: %s",
log.info(String.format("Updated workflowRun with id : %s of workflowId: %s",
workflowRunDoc.getId(), workflowRunDoc.getWorkflowName()));
// TODO [aaljain]: The feature for deleting container needs to be moved to service folder later
// TODO 19.03.2021 (expires after 19.09.2021)[aaljain]:
// The feature for deleting container needs to be moved to service folder later
final WorkflowStatusType currentStatusType = workflowRun.getStatus();
if (getCompletedStatusTypes().contains(currentStatusType)) {
workflowTasksSharingRepository.deleteTasksSharingInfoContainer(dpsHeaders.getPartitionId(), workflowRun.getWorkflowName(), workflowRun.getRunId());
}
return getWorkflowRun(workflowRun.getWorkflowId(), workflowRun.getRunId());
}
......
......@@ -41,11 +41,10 @@ public class WorkflowTasksSharingRepository implements IWorkflowTasksSharingRepo
@Override
public String getSignedUrl(String workflowName, String runId) {
final String dataPartitionId = dpsHeaders.getPartitionId();
// TODO : Add support for using user provided expiry time and permissions (?)
final OffsetDateTime startTime = OffsetDateTime.now();
final int expiryDays = 7;
final OffsetDateTime expiryTime = OffsetDateTime.now().plusDays(expiryDays);
// TODO : Add support for custom permission (?)
// TODO 18.03.21 (expires at 18.09.21): Add support for custom permission (?)
final BlobContainerSasPermission permissions = new BlobContainerSasPermission()
.setCreatePermission(true)
.setReadPermission(true)
......
......@@ -39,7 +39,7 @@ public class AuthenticationServiceImpl implements IAuthenticationService {
checkPreconditions(authorizationToken, partitionID);
// TODO: add check of user permissions
// TODO 22.06.21 (expires at 22.12.21): add check of user permissions
logger.log(Level.INFO, "Finished checking authentication.");
}
......
......@@ -79,3 +79,7 @@ shared.tenant.name=system
# Dagcontent config
osdu.azure.airflow.ignoreDagContent=${ignore_dagContent}
# Azure Event Grid Configuration
azure.eventGrid.enabled=${event_grid_enabled:true}
azure.eventGrid.topicName=${event_grid_topic:statuschangedtopic}
package org.opengroup.osdu.workflow.provider.azure.gsm;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.opengroup.osdu.azure.eventgrid.EventGridTopicStore;
import org.opengroup.osdu.core.common.exception.CoreException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.status.Message;
import org.opengroup.osdu.core.common.model.status.StatusDetails;
import java.util.HashMap;
import java.util.Map;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
/**
* Tests for {@link AzureEventGridPublisher}
*/
@ExtendWith(MockitoExtension.class)
class AzureEventGridPublisherTest {
private final static String TOPIC_NAME = "status_topic";
private final static String CORRELATION_ID = "CORRELATION_ID";
private final static String DATA_PARTITION_ID = "DATA_PARTITION_ID";
@Mock
private EventGridTopicStore eventGridTopicStore;
private AzureEventGridPublisher publisher;
@BeforeEach
void setUp() {
publisher = new AzureEventGridPublisher(eventGridTopicStore);
publisher.setTopicName(TOPIC_NAME);
publisher.setIsEventGridEnabled(true);
}
@Test
void shouldPublishGSMMessage() {
//given
doNothing().when(eventGridTopicStore).publishToEventGridTopic(any(), any(), any());
Message[] messages = new Message[]{new StatusDetails()};
Map<String, String> attributesMap = new HashMap<String, String>() {{
put(DpsHeaders.CORRELATION_ID, CORRELATION_ID);
put(DpsHeaders.DATA_PARTITION_ID, DATA_PARTITION_ID);
}};
//when
publisher.publish(messages, attributesMap);
//then
verify(eventGridTopicStore, times(1))
.publishToEventGridTopic(any(), any(), any());
}
@Test
void shouldTrowCoreExceptionOnDisabledPublishing() {
//given
publisher.setIsEventGridEnabled(false);
Message[] messages = new Message[]{new StatusDetails()};
Map<String, String> attributesMap = new HashMap<String, String>() {{
put(DpsHeaders.CORRELATION_ID, CORRELATION_ID);
put(DpsHeaders.DATA_PARTITION_ID, DATA_PARTITION_ID);
}};
//when & then
Assertions.assertThrows(CoreException.class,
() -> publisher.publish(messages, attributesMap)
);
//then
verify(eventGridTopicStore, times(0))
.publishToEventGridTopic(any(), any(), any());
}
@Test
void shouldTrowCoreExceptionOnMissingMessages() {
//given
Map<String, String> attributesMap = new HashMap<String, String>() {{
put(DpsHeaders.CORRELATION_ID, CORRELATION_ID);
put(DpsHeaders.DATA_PARTITION_ID, DATA_PARTITION_ID);
}};
//when & then
Assertions.assertThrows(CoreException.class,
() -> publisher.publish(null, attributesMap)
);
//then
verify(eventGridTopicStore, times(0))
.publishToEventGridTopic(any(), any(), any());
}
@Test
void shouldTrowCoreExceptionOnEmptyMessages() {
//given
Message[] messages = new Message[]{};
Map<String, String> attributesMap = new HashMap<String, String>() {{
put(DpsHeaders.CORRELATION_ID, CORRELATION_ID);
put(DpsHeaders.DATA_PARTITION_ID, DATA_PARTITION_ID);
}};
//when & then
Assertions.assertThrows(CoreException.class,
() -> publisher.publish(messages, attributesMap)
);
//then
verify(eventGridTopicStore, times(0))
.publishToEventGridTopic(any(), any(), any());
}
@Test
void shouldTrowCoreExceptionOnMissingAttributes() {
//given
Message[] messages = new Message[]{new StatusDetails()};
//when & then
Assertions.assertThrows(CoreException.class,
() -> publisher.publish(messages, null)
);
//then
verify(eventGridTopicStore, times(0))
.publishToEventGridTopic(any(), any(), any());
}
@Test
void shouldTrowCoreExceptionOnEmptyAttributes() {
//given
Message[] messages = new Message[]{new StatusDetails()};
Map<String, String> attributesMap = new HashMap<>();