Commit 1dfe433d authored by Abhishek Kumar's avatar Abhishek Kumar Committed by ethiraj krishnamanaidu
Browse files

Externalizing the shared partition name so that it can be configured

differently by each vendor.
parent 0f99d3f8
......@@ -31,7 +31,7 @@ variables:
OSDU_GCP_SERVICE: schema
OSDU_GCP_VENDOR: gcp
OSDU_GCP_APPLICATION_NAME: os-schema
OSDU_GCP_ENV_VARS: AUTHORIZE_API=$OSDU_GCP_ENTITLEMENTS_URL,AUTHORIZE_API_KEY=$OSDU_GCP_AUTHORIZE_API_KEY,LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_WEB=DEBUG,ACCOUNT_ID_COMMON_PROJECT=$TENANT --vpc-connector=$OSDU_GCP_VPC_CONNECTOR
OSDU_GCP_ENV_VARS: AUTHORIZE_API=$OSDU_GCP_ENTITLEMENTS_URL,AUTHORIZE_API_KEY=$OSDU_GCP_AUTHORIZE_API_KEY,LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_WEB=DEBUG,SHARED_TENANT_NAME=$TENANT --vpc-connector=$OSDU_GCP_VPC_CONNECTOR
include:
- project: "osdu/platform/ci-cd-pipelines"
......
......@@ -524,7 +524,7 @@ The following software have components provided under the terms of this license:
- jakarta.xml.bind-api (from )
========================================================================
CC-BY-3.0
CC-BY-2.5
========================================================================
The following software have components provided under the terms of this license:
......@@ -552,7 +552,6 @@ CDDL-1.0
========================================================================
The following software have components provided under the terms of this license:
- JavaBeans(TM) Activation Framework (from http://java.sun.com/javase/technologies/desktop/javabeans/jaf/index.jsp)
- JavaMail API (from )
- javax.annotation-api (from http://jcp.org/en/jsr/detail?id=250)
......@@ -562,14 +561,21 @@ CDDL-1.1
The following software have components provided under the terms of this license:
- JavaBeans Activation Framework (from )
- JavaBeans(TM) Activation Framework (from http://java.sun.com/javase/technologies/desktop/javabeans/jaf/index.jsp)
- tomcat-embed-core (from http://tomcat.apache.org/)
========================================================================
EPL-1.0
CPL-1.0
========================================================================
The following software have components provided under the terms of this license:
- JUnit (from http://junit.org)
========================================================================
EPL-1.0
========================================================================
The following software have components provided under the terms of this license:
- JUnit Jupiter (Aggregator) (from https://junit.org/junit5/)
- Logback Classic Module (from )
- Logback Core Module (from )
......@@ -666,7 +672,6 @@ LGPL-2.1-or-later
========================================================================
The following software have components provided under the terms of this license:
- Java Native Access (from https://github.com/java-native-access/jna)
- Java Native Access Platform (from https://github.com/java-native-access/jna)
- SnakeYAML (from http://www.snakeyaml.org)
......@@ -773,7 +778,6 @@ Public-Domain
========================================================================
The following software have components provided under the terms of this license:
- AOP alliance (from http://aopalliance.sourceforge.net)
- HdrHistogram (from http://hdrhistogram.github.io/HdrHistogram/)
- LatencyUtils (from http://latencyutils.github.io/LatencyUtils/)
- Spongy Castle (from http://rtyley.github.io/spongycastle/)
......@@ -797,6 +801,7 @@ public-domain
========================================================================
The following software have components provided under the terms of this license:
- AOP alliance (from http://aopalliance.sourceforge.net)
- AWS SDK for Java - Models (from https://aws.amazon.com/sdkforjava)
- Asynchronous Http Client (from )
- Guava: Google Core Libraries for Java (from https://github.com/google/guava.git)
......@@ -811,6 +816,7 @@ The following software have components provided under the terms of this license:
- Project Lombok (from https://projectlombok.org)
- Project Lombok (from https://projectlombok.org)
- Spring Web (from https://github.com/spring-projects/spring-framework)
- StAX API (from http://stax.codehaus.org/)
- msal4j (from https://github.com/AzureAD/microsoft-authentication-library-for-java)
- reactive-streams (from http://www.reactive-streams.org/)
......@@ -821,6 +827,7 @@ The following software have components provided under the terms of this license:
- Byte Buddy (without dependencies) (from )
- JSON in Java (from https://github.com/douglascrockford/JSON-java)
- JUnit (from http://junit.org)
- JUnit Jupiter (Aggregator) (from https://junit.org/junit5/)
- JavaBeans Activation Framework API jar (from )
- JavaMail API (from )
......
......@@ -13,8 +13,21 @@
// limitations under the License.
package org.opengroup.osdu.schema.provider.aws.impl.schemainfostore;
import com.amazonaws.services.dynamodbv2.datamodeling.PaginatedQueryList;
import com.amazonaws.services.dynamodbv2.model.AttributeValue;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.stream.Collectors;
import javax.annotation.PostConstruct;
import javax.inject.Inject;
import org.joda.time.DateTime;
import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelper;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
......@@ -35,13 +48,11 @@ import org.opengroup.osdu.schema.provider.interfaces.schemainfostore.ISchemaInfo
import org.opengroup.osdu.schema.provider.interfaces.schemastore.ISchemaStore;
import org.opengroup.osdu.schema.util.VersionHierarchyUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Repository;
import javax.annotation.PostConstruct;
import javax.inject.Inject;
import java.text.MessageFormat;
import java.util.*;
import java.util.stream.Collectors;
import com.amazonaws.services.dynamodbv2.datamodeling.PaginatedQueryList;
import com.amazonaws.services.dynamodbv2.model.AttributeValue;
@Repository
public class AwsSchemaInfoStore implements ISchemaInfoStore {
......@@ -62,6 +73,9 @@ public class AwsSchemaInfoStore implements ISchemaInfoStore {
private ISchemaStore schemaStore;
private DynamoDBQueryHelper queryHelper;
@Value("${shared.tenant.name:common}")
private String sharedTenant;
@PostConstruct
public void init() {
......@@ -242,11 +256,11 @@ public class AwsSchemaInfoStore implements ISchemaInfoStore {
@Override
public boolean isUnique(String schemaId, String tenantId) throws ApplicationException {
Set<String> tenantList = new HashSet<>();
tenantList.add(SchemaConstants.ACCOUNT_ID_COMMON_PROJECT);
tenantList.add(sharedTenant);
tenantList.add(tenantId);
// code to call check uniqueness
if (tenantId.equalsIgnoreCase(SchemaConstants.ACCOUNT_ID_COMMON_PROJECT)) {
if (tenantId.equalsIgnoreCase(sharedTenant)) {
List<String> privateTenantList = tenantFactory.listTenantInfo().stream().map(TenantInfo::getDataPartitionId)
.collect(Collectors.toList());
tenantList.addAll(privateTenantList);
......
......@@ -36,3 +36,6 @@ aws.dynamodb.endpoint=dynamodb.${AWS_REGION}.amazonaws.com
# if this is turned on then the service tries to connect to elastic search
management.health.elasticsearch.enabled=false
# Use this property to name your shared tenant name
shared.tenant.name=common
......@@ -13,6 +13,8 @@
// limitations under the License.
package org.opengroup.osdu.schema.provider.aws.impl.schemainfostore;
import static org.junit.Assert.assertEquals;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
......@@ -25,52 +27,56 @@ import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.schema.exceptions.ApplicationException;
import org.opengroup.osdu.schema.provider.aws.config.AwsServiceConfig;
import static org.junit.Assert.assertEquals;
import org.springframework.test.util.ReflectionTestUtils;
@RunWith(MockitoJUnitRunner.class)
public class AwsSchemaInfoStoreTest {
@InjectMocks
private AwsSchemaInfoStore schemaInfoStore;
@InjectMocks
private AwsSchemaInfoStore schemaInfoStore;
@Mock
private AwsServiceConfig serviceConfig;
@Mock
private DpsHeaders headers;
@Mock
private DynamoDBQueryHelper queryHelper;
@Mock
private JaxRsDpsLog logger;
@Mock
private AwsServiceConfig serviceConfig;
@Before
public void setUp() throws Exception {
serviceConfig.amazonRegion = "us-east-1";
serviceConfig.s3DataBucket = "bucket";
serviceConfig.environment = "test";
serviceConfig.s3Endpoint = "s3endpoint";
serviceConfig.dynamoDbEndpoint = "dbendpoint";
serviceConfig.dynamoDbTablePrefix = "pre-";
@Mock
private DpsHeaders headers;
ReflectionTestUtils.setField(schemaInfoStore, "sharedTenant", "common");
@Mock
private DynamoDBQueryHelper queryHelper;
@Mock
private JaxRsDpsLog logger;
@Before
public void setUp() throws Exception {
serviceConfig.amazonRegion = "us-east-1";
serviceConfig.s3DataBucket = "bucket";
serviceConfig.environment = "test";
serviceConfig.s3Endpoint = "s3endpoint";
serviceConfig.dynamoDbEndpoint = "dbendpoint";
serviceConfig.dynamoDbTablePrefix = "pre-";
}
}
@Test
public void isUnique_ifNotExists_returnTrue() throws ApplicationException {
String partitionId = "partitionId";
String schemaId = "schemaId";
Mockito.when(queryHelper.keyExistsInTable(Mockito.any(), Mockito.any())).thenReturn(false);
Boolean actual = schemaInfoStore.isUnique(schemaId, partitionId);
assertEquals(true, actual);
}
@Test
public void isUnique_ifNotExists_returnTrue() throws ApplicationException {
String partitionId = "partitionId";
String schemaId = "schemaId";
Mockito.when(queryHelper.keyExistsInTable(Mockito.any(), Mockito.any())).thenReturn(false);
Boolean actual = schemaInfoStore.isUnique(schemaId, partitionId);
assertEquals(true, actual);
}
@Test
public void isUnique_ifExists_returnFalse() throws ApplicationException {
String partitionId = "partitionId";
String schemaId = "schemaId";
Mockito.when(queryHelper.keyExistsInTable(Mockito.any(), Mockito.any())).thenReturn(true);
Boolean actual = schemaInfoStore.isUnique(schemaId, partitionId);
assertEquals(false, actual);
}
@Test
public void isUnique_ifExists_returnFalse() throws ApplicationException {
String partitionId = "partitionId";
String schemaId = "schemaId";
Mockito.when(queryHelper.keyExistsInTable(Mockito.any(), Mockito.any())).thenReturn(true);
Boolean actual = schemaInfoStore.isUnique(schemaId, partitionId);
assertEquals(false, actual);
}
}
\ No newline at end of file
......@@ -44,6 +44,7 @@ import org.opengroup.osdu.schema.model.SchemaRequest;
import org.opengroup.osdu.schema.provider.interfaces.schemainfostore.ISchemaInfoStore;
import org.opengroup.osdu.schema.util.VersionHierarchyUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Repository;
import lombok.extern.java.Log;
......@@ -69,6 +70,9 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore {
@Autowired
private String cosmosDBName;
@Value("${shared.tenant.name:common}")
private String sharedTenant;
@Autowired
JaxRsDpsLog log;
......@@ -326,13 +330,13 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore {
@Override
public boolean isUnique(String schemaId, String tenantId) throws ApplicationException {
Set<String> tenantList = new HashSet<>();
tenantList.add(SchemaConstants.ACCOUNT_ID_COMMON_PROJECT);
tenantList.add(sharedTenant);
tenantList.add(tenantId);
/* TODO : Below code enables uniqueness check across tenants and is redundant now. This will be handled/updated as part
of data partition changes.
*/
if (tenantId.equalsIgnoreCase(SchemaConstants.ACCOUNT_ID_COMMON_PROJECT)) {
if (tenantId.equalsIgnoreCase(sharedTenant)) {
List<String> privateTenantList = tenantFactory.listTenantInfo().stream().map(TenantInfo::getName)
.collect(Collectors.toList());
tenantList.addAll(privateTenantList);
......
......@@ -54,3 +54,6 @@ logging.slf4jlogger.enabled=true
tenantFactoryImpl.required=true
azure.blobStore.required=true
# Use this property to name your shared tenant name
shared.tenant.name=common
......@@ -42,6 +42,7 @@ import org.opengroup.osdu.schema.model.QueryParams;
import org.opengroup.osdu.schema.model.SchemaIdentity;
import org.opengroup.osdu.schema.model.SchemaInfo;
import org.opengroup.osdu.schema.model.SchemaRequest;
import org.springframework.test.util.ReflectionTestUtils;
import java.io.IOException;
import java.util.*;
......@@ -93,6 +94,7 @@ public class AzureSchemaInfoStoreTest {
@Before
public void init() {
initMocks(this);
ReflectionTestUtils.setField(schemaInfoStore, "sharedTenant", "common");
doReturn(dataPartitionId).when(headers).getPartitionId();
}
......
......@@ -65,8 +65,8 @@ public class GoogleSchemaInfoStore implements ISchemaInfoStore {
@Autowired
JaxRsDpsLog log;
@Value("${account.id.common.project}")
private String commonAccountId;
@Value("${shared.tenant.name:common}")
private String sharedTenant;
/**
* Method to get schemaInfo from google store
......@@ -258,9 +258,6 @@ public class GoogleSchemaInfoStore implements ISchemaInfoStore {
@Override
public List<SchemaInfo> getSchemaInfoList(QueryParams queryParams, String tenantId) throws ApplicationException {
List<SchemaInfo> schemaList = new LinkedList<>();
if (SchemaConstants.ACCOUNT_ID_COMMON_PROJECT.equals(tenantId)) {
return schemaList;
}
Datastore datastore = dataStoreFactory.getDatastore(tenantId, SchemaConstants.NAMESPACE);
List<Filter> filterList = getFilters(queryParams);
......@@ -311,11 +308,11 @@ public class GoogleSchemaInfoStore implements ISchemaInfoStore {
public boolean isUnique(String schemaId, String tenantId) throws ApplicationException {
Set<String> tenantList = new HashSet<>();
tenantList.add(commonAccountId);
tenantList.add(sharedTenant);
tenantList.add(tenantId);
// code to call check uniqueness
if (tenantId.equalsIgnoreCase(commonAccountId)) {
if (tenantId.equalsIgnoreCase(sharedTenant)) {
List<String> privateTenantList = tenantFactory.listTenantInfo().stream().map(TenantInfo::getDataPartitionId)
.collect(Collectors.toList());
tenantList.addAll(privateTenantList);
......@@ -335,11 +332,12 @@ public class GoogleSchemaInfoStore implements ISchemaInfoStore {
return true;
}
public String getCommonAccountId() {
return commonAccountId;
}
public String getSharedTenant() {
return sharedTenant;
}
public void setSharedTenant(String sharedTenant) {
this.sharedTenant = sharedTenant;
}
public void setCommonAccountId(String commonAccountId) {
this.commonAccountId = commonAccountId;
}
}
\ No newline at end of file
......@@ -9,6 +9,7 @@ import org.opengroup.osdu.schema.exceptions.ApplicationException;
import org.opengroup.osdu.schema.exceptions.NotFoundException;
import org.opengroup.osdu.schema.provider.interfaces.schemastore.ISchemaStore;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Repository;
import com.google.cloud.storage.Blob;
......@@ -33,6 +34,9 @@ public class GoogleSchemaStore implements ISchemaStore {
@Autowired
TenantFactory tenantFactory;
@Value("${shared.tenant.name:common}")
private String sharedTenant;
@Autowired
JaxRsDpsLog log;
......@@ -49,9 +53,6 @@ public class GoogleSchemaStore implements ISchemaStore {
*/
@Override
public String getSchema(String dataPartitionId, String filePath) throws ApplicationException, NotFoundException {
if (SchemaConstants.ACCOUNT_ID_COMMON_PROJECT.equals(dataPartitionId)) {
throw new NotFoundException(SchemaConstants.SCHEMA_NOT_PRESENT);
}
filePath = filePath + SchemaConstants.JSON_EXTENSION;
String bucketname = getSchemaBucketName(dataPartitionId);
Storage storage = storageFactory.get(tenantFactory.getTenantInfo(dataPartitionId));
......
......@@ -7,4 +7,6 @@ management.health.elasticsearch.enabled=false
management.endpoints.web.base-path=/
management.endpoints.web.path-mapping.health=health
LOG_PREFIX=schema
account.id.common.project=common
# Use this property to name your shared tenant name
shared.tenant.name=common
......@@ -9,6 +9,7 @@ import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
......@@ -32,6 +33,7 @@ import org.opengroup.osdu.schema.model.SchemaIdentity;
import org.opengroup.osdu.schema.model.SchemaInfo;
import org.opengroup.osdu.schema.model.SchemaRequest;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.test.util.ReflectionTestUtils;
import com.google.cloud.Timestamp;
import com.google.cloud.datastore.Blob;
......@@ -104,6 +106,11 @@ public class GoogleSchemaInfoStoreTest {
@Rule
public ExpectedException expectedException = ExpectedException.none();
@Before
public void setUp() {
ReflectionTestUtils.setField(schemaInfoStore, "sharedTenant", "common");
}
@Test
public void testGetLatestMinorVersion_ReturnNull() throws NotFoundException, ApplicationException {
......@@ -172,7 +179,7 @@ public class GoogleSchemaInfoStoreTest {
KeyFactory storageKeyFactory = mock(KeyFactory.class);
String schemaId = "schemaId";
String tenantId = "tenant";
schemaInfoStore.setCommonAccountId(tenantId);
schemaInfoStore.setSharedTenant(tenantId);
Mockito.when(tenantFactory.getTenantInfo("tenant")).thenReturn(tenantInfo);
Mockito.when(tenantFactory.getTenantInfo("common")).thenReturn(tenantInfoCommon);
Mockito.when(tenantInfo.getName()).thenReturn("tenant");
......
package org.opengroup.osdu.schema.impl.schemastore;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
......@@ -17,6 +18,7 @@ import org.opengroup.osdu.schema.constants.SchemaConstants;
import org.opengroup.osdu.schema.exceptions.ApplicationException;
import org.opengroup.osdu.schema.exceptions.NotFoundException;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.test.util.ReflectionTestUtils;
import com.google.cloud.storage.Blob;
import com.google.cloud.storage.BlobId;
......@@ -59,6 +61,11 @@ public class GoogleSchemaStoreTest {
private static final String FILE_PATH = "/test-folder/test-file";
private static final String CONTENT = "Hello World";
@Before
public void setUp() {
ReflectionTestUtils.setField(schemaStore, "sharedTenant", "common");
}
@Test
public void testCreateSchema() throws ApplicationException {
......
......@@ -35,6 +35,7 @@ import org.opengroup.osdu.schema.provider.interfaces.schemainfostore.ISchemaInfo
import org.opengroup.osdu.schema.provider.interfaces.schemastore.ISchemaStore;
import org.opengroup.osdu.schema.util.VersionHierarchyUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Repository;
import org.springframework.web.context.annotation.RequestScope;
......@@ -43,7 +44,6 @@ import com.cloudant.client.api.query.Expression;
import com.cloudant.client.api.query.QueryBuilder;
import com.cloudant.client.api.query.QueryResult;
import com.cloudant.client.api.query.Selector;
import com.google.gson.Gson;
/**
* Repository class to to register Schema in IBM Document Store.
......@@ -67,6 +67,9 @@ public class IbmSchemaInfoStore extends IbmDocumentStore implements ISchemaInfoS
@Autowired
private ISchemaStore schemaStore;
@Value("${shared.tenant.name:common}")
private String sharedTenant;
@PostConstruct
......@@ -78,11 +81,11 @@ public class IbmSchemaInfoStore extends IbmDocumentStore implements ISchemaInfoS
@Override
public boolean isUnique(String schemaId, String tenantId) throws ApplicationException {
Set<String> tenantList = new HashSet<>();
tenantList.add(SchemaConstants.ACCOUNT_ID_COMMON_PROJECT);
tenantList.add(sharedTenant);
tenantList.add(tenantId);
// code to call check uniqueness
if (tenantId.equalsIgnoreCase(SchemaConstants.ACCOUNT_ID_COMMON_PROJECT)) {
if (tenantId.equalsIgnoreCase(sharedTenant)) {
List<String> privateTenantList = tenantFactory.listTenantInfo().stream().map(TenantInfo::getDataPartitionId)
.collect(Collectors.toList());
tenantList.addAll(privateTenantList);
......
......@@ -20,3 +20,7 @@ ibm.tenant.db.password=TODO
ibm.tenant.db.name=TODO
spring.main.allow-bean-definition-overriding=true
# Use this property to name your shared tenant name
shared.tenant.name=common
\ No newline at end of file
......@@ -5,13 +5,15 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import java.net.MalformedURLException;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;
import static org.mockito.ArgumentMatchers.*;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
......@@ -24,7 +26,6 @@ import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
import org.opengroup.osdu.core.ibm.cloudant.IBMCloudantClientFactory;
import org.opengroup.osdu.core.ibm.multitenancy.TenantFactory;
import org.opengroup.osdu.core.ibm.objectstorage.CloudObjectStorageFactory;
import org.opengroup.osdu.schema.constants.SchemaConstants;
import org.opengroup.osdu.schema.enums.SchemaScope;
import org.opengroup.osdu.schema.enums.SchemaStatus;
......@@ -38,11 +39,11 @@ import org.opengroup.osdu.schema.model.SchemaInfo;
import org.opengroup.osdu.schema.model.SchemaRequest;
import org.opengroup.osdu.schema.provider.ibm.SchemaDoc;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.test.util.ReflectionTestUtils;
import com.cloudant.client.api.Database;
import com.cloudant.client.api.query.QueryResult;
import com.cloudant.client.org.lightcouch.DocumentConflictException;
import com.ibm.cloud.objectstorage.services.s3.AmazonS3;
@RunWith(SpringJUnit4ClassRunner.class)
......@@ -95,6 +96,10 @@ public class IbmSchemaInfoStoreTest {
private static final String dataPartitionId = "testPartitionId";
@Before
public void setUp() {
ReflectionTestUtils.setField(schemaInfoStore, "sharedTenant", "common");
}
@Test
public void testGetLatestMinorVersion_ReturnNull() throws NotFoundException, ApplicationException, MalformedURLException {
......