Commit 92988757 authored by Aman Verma's avatar Aman Verma
Browse files

merging with maste

parents d9fb0e66 1d860ade
Pipeline #45914 passed with stages
in 23 minutes and 41 seconds
......@@ -11,6 +11,8 @@ variables:
IBM_BUILD_SUBDIR: provider/schema-ibm
IBM_INT_TEST_SUBDIR: testing/schema-test-core
IBM_TEST_CMD: verify
IBM_DEPLOYMENTS_SUBDIR: deployments/scripts/ibm
IBM_DEPLOYMENTS_SCRIPTS_SUBDIR: deployments/scripts
AZURE_SERVICE: schema
AZURE_BUILD_SUBDIR: provider/schema-azure
......@@ -35,7 +37,7 @@ variables:
OSDU_GCP_VENDOR: gcp
OSDU_GCP_APPLICATION_NAME: os-schema
OSDU_GCP_TESTS_SUBDIR: testing/schema-test-core
OSDU_GCP_ENV_VARS: AUTHORIZE_API=$OSDU_GCP_ENTITLEMENTS_URL,AUTHORIZE_API_KEY=$OSDU_GCP_AUTHORIZE_API_KEY,LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_WEB=DEBUG,SHARED_TENANT_NAME=$TENANT --vpc-connector=$OSDU_GCP_VPC_CONNECTOR
OSDU_GCP_ENV_VARS: AUTHORIZE_API=$OSDU_GCP_ENTITLEMENTS_URL,AUTHORIZE_API_KEY=$OSDU_GCP_AUTHORIZE_API_KEY,LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_WEB=DEBUG,SHARED_TENANT_NAME=$TENANT,PARTITION_API=$OSDU_GCP_PARTITION_API,GOOGLE_AUDIENCES=$GOOGLE_AUDIENCE --vpc-connector=$OSDU_GCP_VPC_CONNECTOR
include:
- project: "osdu/platform/ci-cd-pipelines"
......@@ -65,6 +67,7 @@ include:
- local: "/devops/azure/gitlab-bootstrap.yml"
- local: "/devops/aws/bootstrap.yaml"
- local: "/devops/ibm/bootstrap.yaml"
osdu-gcp-test:
image: maven:3.6.3-jdk-8
......@@ -83,4 +86,4 @@ osdu-gcp-test:
- $OSDU_GCP_TESTS_SUBDIR/target/*/TEST-*.xml
reports:
junit:
- $OSDU_GCP_TESTS_SUBDIR/target/*/TEST-*.xml
\ No newline at end of file
- $OSDU_GCP_TESTS_SUBDIR/target/*/TEST-*.xml
......@@ -249,6 +249,7 @@ The following software have components provided under the terms of this license:
- AWS Java SDK for Amazon Rekognition (from https://aws.amazon.com/sdkforjava)
- AWS Java SDK for Amazon Route 53 Auto Naming (from https://aws.amazon.com/sdkforjava)
- AWS Java SDK for Amazon Route 53 Resolver (from https://aws.amazon.com/sdkforjava)
- AWS Java SDK for Amazon Route53 (from https://aws.amazon.com/sdkforjava)
- AWS Java SDK for Amazon S3 (from https://aws.amazon.com/sdkforjava)
- AWS Java SDK for Amazon S3 on Outposts (from https://aws.amazon.com/sdkforjava)
- AWS Java SDK for Amazon SES (from https://aws.amazon.com/sdkforjava)
......@@ -352,6 +353,7 @@ The following software have components provided under the terms of this license:
- Guava ListenableFuture only (from )
- Guava: Google Core Libraries for Java (from https://github.com/google/guava.git)
- Guava: Google Core Libraries for Java (from https://github.com/google/guava.git)
- Guava: Google Core Libraries for Java (from https://github.com/google/guava.git)
- HPPC Collections (from http://labs.carrotsearch.com)
- Hibernate Validator Engine (from )
- Hibernate Validator Engine (from )
......@@ -433,8 +435,8 @@ The following software have components provided under the terms of this license:
- Microsoft Azure Java Core Library (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java)
- Mockito (from http://www.mockito.org)
- Mockito (from http://mockito.org)
- Mockito (from http://www.mockito.org)
- Netty Reactive Streams HTTP support (from )
- Netty Reactive Streams Implementation (from )
- Netty/Buffer (from http://netty.io/)
......@@ -535,6 +537,7 @@ The following software have components provided under the terms of this license:
- elasticsearch-core (from https://github.com/elastic/elasticsearch)
- elasticsearch-geo (from https://github.com/elastic/elasticsearch)
- error-prone annotations (from )
- error-prone annotations (from )
- google-auth (from https://github.com/GoogleCloudPlatform/google-auth-library-python)
- io.grpc:grpc-alts (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-api (from https://github.com/grpc/grpc-java)
......@@ -839,6 +842,7 @@ The following software have components provided under the terms of this license:
- Azure Spring Boot AutoConfigure (from https://github.com/Microsoft/azure-spring-boot)
- Checker Qual (from https://checkerframework.org)
- Checker Qual (from https://checkerframework.org)
- Checker Qual (from https://checkerframework.org)
- Cucumber Expressions (from https://github.com/cucumber/cucumber-expressions-java)
- Cucumber Tag Expressions (from https://github.com/cucumber/tag-expressions-java)
- Cucumber-JVM: Core (from )
......@@ -874,8 +878,8 @@ The following software have components provided under the terms of this license:
- Microsoft Azure client library for KeyVault Secrets (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure common module for Storage (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure internal Avro module for Storage (from https://github.com/Azure/azure-sdk-for-java)
- Mockito (from http://www.mockito.org)
- Mockito (from http://mockito.org)
- Mockito (from http://www.mockito.org)
- Netty/Codec/HTTP (from )
- Netty/Codec/HTTP (from )
- Netty/Common (from )
......@@ -944,6 +948,7 @@ SPL-1.0
========================================================================
The following software have components provided under the terms of this license:
- Checker Qual (from https://checkerframework.org)
- Checker Qual (from https://checkerframework.org)
========================================================================
......@@ -964,6 +969,7 @@ The following software have components provided under the terms of this license:
- Asynchronous Http Client (from )
- Guava: Google Core Libraries for Java (from https://github.com/google/guava.git)
- Guava: Google Core Libraries for Java (from https://github.com/google/guava.git)
- Guava: Google Core Libraries for Java (from https://github.com/google/guava.git)
- HdrHistogram (from http://hdrhistogram.github.io/HdrHistogram/)
- HdrHistogram (from http://hdrhistogram.github.io/HdrHistogram/)
- Joda-Time (from http://www.joda.org/joda-time/)
......@@ -987,6 +993,7 @@ unknown
The following software have components provided under the terms of this license:
- Byte Buddy (without dependencies) (from )
- Checker Qual (from https://checkerframework.org)
- JSON in Java (from https://github.com/douglascrockford/JSON-java)
- JUnit (from http://junit.org)
- JUnit (from http://junit.org)
......
import json
import requests
import http.client
import os;
class IBMToken(object):
def get_ibm_id_token(self):
url = 'https://' + os.getenv('IBM_KEYCLOAK_URL') + '/auth/realms/OSDU/protocol/openid-connect/token'
clientId = os.getenv('IBM_KEYCLOAK_CLIENT_ID')
clientSecret = os.getenv('IBM_KEYCLOAK_CLIENT_SECRET')
grantType = 'password'
userName = os.getenv('IBM_AUTH_USER_ACCESS')
password = os.getenv('IBM_AUTH_USER_ACCESS_PASSWORD')
scope = 'openid'
if url is None:
print('Please pass url to generate token')
exit(1)
if clientId is None:
print('Please pass client id to generate token')
exit(1)
if clientSecret is None:
print('Please pass client secret to generate token')
exit(1)
if grantType is None:
print('Please pass client secret to generate token')
exit(1)
if userName is None:
print('Please pass client secret to generate token')
exit(1)
if password is None:
print('Please pass client secret to generate token')
exit(1)
if scope is None:
print('Please pass client secret to generate token')
exit(1)
try:
response = requests.post(url, data={"grant_type": grantType,
"client_id": clientId,
"client_secret" : clientSecret,
"username": userName,
"password": password, "scope": scope})
result = response.json()
token = 'Bearer ' + result['access_token']
print(token)
except Exception as e:
print(e)
if __name__ == '__main__':
IBMToken().get_ibm_id_token()
export IBM_SCHEMA_URL=$IBM_SCHEMA_HOST/api/schema-service/v1/schema
BEARER_TOKEN=`python $IBM_DEPLOYMENTS_SUBDIR/Token.py`
export BEARER_TOKEN=$BEARER_TOKEN
echo $BEARER_TOKEN
python $IBM_DEPLOYMENTS_SCRIPTS_SUBDIR/DeploySharedSchemas.py -u $IBM_SCHEMA_URL
......@@ -106,5 +106,9 @@ spec:
value: {{ .Values.default_tenant}}
- name: azure_istioauth_enabled
value: "true"
- name: event_grid_enabled
value: "false"
- name: event_grid_topic
value: "schemachangedtopic"
ibm_bootstrap:
stage: bootstrap
image: python:3.8
needs: ["ibm-deploy"]
extends:
- .ibm_variables
script:
- pip install -r $IBM_DEPLOYMENTS_SUBDIR/requirements.txt
- chmod +x $IBM_DEPLOYMENTS_SUBDIR/bootstrap.sh
- $IBM_DEPLOYMENTS_SUBDIR/bootstrap.sh
only:
variables:
- $IBM == 'true'
ibm-test:
needs: ["ibm_bootstrap"]
only:
variables:
- $IBM == 'true'
......@@ -31,7 +31,7 @@
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-schema</artifactId>
<version>0.9.0-SNAPSHOT</version>
<version>0.10.0-SNAPSHOT</version>
<packaging>pom</packaging>
<name>os-schema</name>
<description>os schema service </description>
......
......@@ -18,7 +18,7 @@
<parent>
<artifactId>os-schema</artifactId>
<groupId>org.opengroup.osdu</groupId>
<version>0.9.0-SNAPSHOT</version>
<version>0.10.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
......@@ -35,7 +35,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-schema-core</artifactId>
<version>0.9.0-SNAPSHOT</version>
<version>0.10.0-SNAPSHOT</version>
</dependency>
<!-- AWS managed packages -->
<dependency>
......@@ -67,7 +67,7 @@
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
<artifactId>os-core-lib-aws</artifactId>
<version>0.9.1-SNAPSHOT</version>
<version>0.9.0</version>
</dependency>
<dependency>
......
package org.opengroup.osdu.schema.provider.aws.impl.messagebus;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.schema.provider.interfaces.messagebus.IMessageBus;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@Component
public class MessageBusImpl implements IMessageBus{
@Autowired
private JaxRsDpsLog logger;
@Override
public void publishMessage(String schemaId, String eventType) {
// TODO Auto-generated method stub
logger.warning("publish message not implemented yet");
}
}
......@@ -18,20 +18,20 @@
<parent>
<artifactId>os-schema</artifactId>
<groupId>org.opengroup.osdu</groupId>
<version>0.9.0-SNAPSHOT</version>
<version>0.10.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>os-schema-azure</artifactId>
<version>0.9.0-SNAPSHOT</version>
<version>0.10.0-SNAPSHOT</version>
<description>Azure related implementation staff.</description>
<packaging>jar</packaging>
<properties>
<osdu.corelibazure.version>0.6.1</osdu.corelibazure.version>
<osdu.oscorecommon.version>0.6.9</osdu.oscorecommon.version>
<osdu.os-schema-core.version>0.9.0-SNAPSHOT</osdu.os-schema-core.version>
<osdu.os-schema-core.version>0.10.0-SNAPSHOT</osdu.os-schema-core.version>
<mockito.version>1.10.19</mockito.version>
<cucumber.version>5.4.0</cucumber.version>
<nimbus-jose-jwt-azure.version>7.9</nimbus-jose-jwt-azure.version>
......
......@@ -17,6 +17,7 @@ package org.opengroup.osdu.schema.azure;
import org.opengroup.osdu.azure.dependencies.AzureOSDUConfig;
import org.opengroup.osdu.schema.azure.di.AzureBootstrapConfig;
import org.opengroup.osdu.schema.azure.di.CosmosContainerConfig;
import org.opengroup.osdu.schema.azure.di.EventGridConfig;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.ComponentScan;
......@@ -31,7 +32,8 @@ public class SchemaApplication {
SchemaApplication.class,
AzureBootstrapConfig.class,
AzureOSDUConfig.class,
CosmosContainerConfig.class
CosmosContainerConfig.class,
EventGridConfig.class
};
SpringApplication.run(sources, args);
}
......
/*
* Copyright 2021 Schlumberger
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.schema.azure.di;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
import com.azure.cosmos.implementation.apachecommons.lang.StringUtils;
@Configuration
public class EventGridConfig {
private boolean eventGridEnabled;
private String eventGridCustomTopic;
public boolean isEventGridEnabled() {
return eventGridEnabled;
}
public String getCustomTopicName() {
return eventGridCustomTopic;
}
public EventGridConfig(@Value("#{new Boolean('${azure.eventGrid.enabled:false}')}") boolean publish,
@Value("#{new String('${azure.eventGrid.topicName:schemachangedtopic}')}") String topicName) {
if (publish && StringUtils.isEmpty(topicName)) {
throw new RuntimeException("Missing EventGrid Configuration");
}
this.eventGridEnabled = publish;
this.eventGridCustomTopic = topicName;
}
}
/*
* Copyright 2021 Schlumberger
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.schema.azure.impl.messagebus;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.UUID;
import org.joda.time.DateTime;
import org.opengroup.osdu.azure.eventgrid.EventGridTopicStore;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.schema.azure.di.EventGridConfig;
import org.opengroup.osdu.schema.azure.impl.messagebus.model.SchemaPubSubInfo;
import org.opengroup.osdu.schema.constants.SchemaConstants;
import org.opengroup.osdu.schema.logging.AuditLogger;
import org.opengroup.osdu.schema.provider.interfaces.messagebus.IMessageBus;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.microsoft.azure.eventgrid.models.EventGridEvent;
@Component
public class MessageBusImpl implements IMessageBus {
@Autowired
private EventGridTopicStore eventGridTopicStore;
@Autowired
private JaxRsDpsLog logger;
@Autowired
private EventGridConfig eventGridConfig;
@Autowired
private AuditLogger auditLogger;
@Autowired
DpsHeaders headers;
private final static String EVENT_DATA_VERSION = "1.0";
@Override
public void publishMessage(String schemaId, String eventType) {
if (eventGridConfig.isEventGridEnabled()) {
logger.info("Generating event of type {}",eventType);
try {
publishToEventGrid(schemaId, eventType);
auditLogger.schemaNotificationSuccess(Collections.singletonList(schemaId));
}catch (AppException ex) {
//We do not want to fail schema creation if notification delivery has failed, hence just logging the exception
auditLogger.schemaNotificationFailure(Collections.singletonList(schemaId));
logger.warning(SchemaConstants.SCHEMA_NOTIFICATION_FAILED);
}
}else {
logger.info(SchemaConstants.SCHEMA_NOTIFICATION_IS_DISABLED);
}
}
private void publishToEventGrid(String schemaId, String eventType) {
String messageId = UUID.randomUUID().toString();
SchemaPubSubInfo[] schemaPubSubMsgs = new SchemaPubSubInfo [1];
schemaPubSubMsgs[0]=new SchemaPubSubInfo(schemaId,eventType);
List<EventGridEvent> eventsList = new ArrayList<>();
HashMap<String, Object> message = new HashMap<>();
message.put("data", schemaPubSubMsgs);
message.put(DpsHeaders.ACCOUNT_ID, headers.getPartitionIdWithFallbackToAccountId());
message.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
message.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
//EventGridEvent supports array of messages to be triggered in a batch but at present we do not support
//schema creation in bulk so generating one event at a time.
EventGridEvent eventGridEvent = new EventGridEvent(
messageId,
SchemaConstants.EVENT_SUBJECT,
message,
eventType,
DateTime.now(),
EVENT_DATA_VERSION
);
eventsList.add(eventGridEvent);
logger.info("Schema event created: " + messageId);
eventGridTopicStore.publishToEventGridTopic(headers.getPartitionId(), eventGridConfig.getCustomTopicName(), eventsList);
logger.info("Schema event generated successfully");
}
}
package org.opengroup.osdu.schema.azure.impl.messagebus.model;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@AllArgsConstructor
@NoArgsConstructor
public class SchemaPubSubInfo {
private String kind;
private String op;
}
......@@ -65,4 +65,9 @@ azure.activedirectory.app-resource-id=${aad_client_id}
# Use this property to name your shared tenant name
# shared.tenant.name=${shared_partition}
shared.tenant.name=system
# Azure Event Grid Configuration
azure.eventGrid.enabled=${event_grid_enabled}
azure.eventGrid.topicName=${event_grid_topic}
/*
* Copyright 2021 Schlumberger
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.schema.azure.impl.messagebus;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.mockito.MockitoAnnotations.initMocks;
import java.util.ArrayList;
import java.util.HashMap;
import org.junit.Before;
import org.junit.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.opengroup.osdu.azure.eventgrid.EventGridTopicStore;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.schema.azure.di.EventGridConfig;
import org.opengroup.osdu.schema.azure.impl.messagebus.model.SchemaPubSubInfo;
import org.opengroup.osdu.schema.logging.AuditLogger;
import com.microsoft.azure.eventgrid.models.EventGridEvent;
import com.microsoft.azure.servicebus.primitives.ServiceBusException;
@ExtendWith(MockitoExtension.class)
public class MessageBusImplTest {
private static final String DATA_PARTITION_WITH_FALLBACK_ACCOUNT_ID = "data-partition-account-id";
private static final String CORRELATION_ID = "correlation-id";
private static final String PARTITION_ID = "partition-id";
@Mock
private EventGridTopicStore eventGridTopicStore;
@Mock
private EventGridConfig eventGridConfig;
@Mock
private DpsHeaders dpsHeaders;
@Mock
private JaxRsDpsLog logger;
@Mock
private AuditLogger auditLogger;
@InjectMocks
private MessageBusImpl messageBusImpl;
@Before
public void init() throws ServiceBusException, InterruptedException {
initMocks(this);
doReturn(DATA_PARTITION_WITH_FALLBACK_ACCOUNT_ID).when(dpsHeaders).getPartitionIdWithFallbackToAccountId();
doReturn(PARTITION_ID).when(dpsHeaders).getPartitionId();
doReturn(CORRELATION_ID).when(dpsHeaders).getCorrelationId();
}
@Test
public void should_publishToEventGrid_WhenFlagIsFalse() {
//The schema-notification is turned off
when(this.eventGridConfig.isEventGridEnabled()).thenReturn(false);
//Call publish Message
messageBusImpl.publishMessage("dummy", "dummy");
//Assert that eventGridTopicStore is not called even once
verify(this.eventGridTopicStore, times(0)).publishToEventGridTopic(any(), any(), anyList());
}
@Test
public void should_publishToEventGrid_WhenFlagIsTrue() {
//The schema-notification is turned off
when(this.eventGridConfig.isEventGridEnabled()).thenReturn(true);
//The schema-notification is turned off
when(this.eventGridConfig.getCustomTopicName()).thenReturn("dummy-topic");
//The schema-notification is turned off
doNothing().when(this.eventGridTopicStore).publishToEventGridTopic(anyString(), anyString(), anyList());;