Commit d623fcbf authored by ethiraj krishnamanaidu's avatar ethiraj krishnamanaidu
Browse files

Merge branch 'slb-decouple-shared-tenant' into 'master'

Externalizing the shared partition name

See merge request !60
parents 0f99d3f8 1dfe433d
Pipeline #19691 failed with stages
in 68 minutes and 18 seconds
......@@ -31,7 +31,7 @@ variables:
OSDU_GCP_SERVICE: schema
OSDU_GCP_VENDOR: gcp
OSDU_GCP_APPLICATION_NAME: os-schema
OSDU_GCP_ENV_VARS: AUTHORIZE_API=$OSDU_GCP_ENTITLEMENTS_URL,AUTHORIZE_API_KEY=$OSDU_GCP_AUTHORIZE_API_KEY,LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_WEB=DEBUG,ACCOUNT_ID_COMMON_PROJECT=$TENANT --vpc-connector=$OSDU_GCP_VPC_CONNECTOR
OSDU_GCP_ENV_VARS: AUTHORIZE_API=$OSDU_GCP_ENTITLEMENTS_URL,AUTHORIZE_API_KEY=$OSDU_GCP_AUTHORIZE_API_KEY,LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_WEB=DEBUG,SHARED_TENANT_NAME=$TENANT --vpc-connector=$OSDU_GCP_VPC_CONNECTOR
include:
- project: "osdu/platform/ci-cd-pipelines"
......
......@@ -524,7 +524,7 @@ The following software have components provided under the terms of this license:
- jakarta.xml.bind-api (from )
========================================================================
CC-BY-3.0
CC-BY-2.5
========================================================================
The following software have components provided under the terms of this license:
......@@ -552,7 +552,6 @@ CDDL-1.0
========================================================================
The following software have components provided under the terms of this license:
- JavaBeans(TM) Activation Framework (from http://java.sun.com/javase/technologies/desktop/javabeans/jaf/index.jsp)
- JavaMail API (from )
- javax.annotation-api (from http://jcp.org/en/jsr/detail?id=250)
......@@ -562,14 +561,21 @@ CDDL-1.1
The following software have components provided under the terms of this license:
- JavaBeans Activation Framework (from )
- JavaBeans(TM) Activation Framework (from http://java.sun.com/javase/technologies/desktop/javabeans/jaf/index.jsp)
- tomcat-embed-core (from http://tomcat.apache.org/)
========================================================================
EPL-1.0
CPL-1.0
========================================================================
The following software have components provided under the terms of this license:
- JUnit (from http://junit.org)
========================================================================
EPL-1.0
========================================================================
The following software have components provided under the terms of this license:
- JUnit Jupiter (Aggregator) (from https://junit.org/junit5/)
- Logback Classic Module (from )
- Logback Core Module (from )
......@@ -666,7 +672,6 @@ LGPL-2.1-or-later
========================================================================
The following software have components provided under the terms of this license:
- Java Native Access (from https://github.com/java-native-access/jna)
- Java Native Access Platform (from https://github.com/java-native-access/jna)
- SnakeYAML (from http://www.snakeyaml.org)
......@@ -773,7 +778,6 @@ Public-Domain
========================================================================
The following software have components provided under the terms of this license:
- AOP alliance (from http://aopalliance.sourceforge.net)
- HdrHistogram (from http://hdrhistogram.github.io/HdrHistogram/)
- LatencyUtils (from http://latencyutils.github.io/LatencyUtils/)
- Spongy Castle (from http://rtyley.github.io/spongycastle/)
......@@ -797,6 +801,7 @@ public-domain
========================================================================
The following software have components provided under the terms of this license:
- AOP alliance (from http://aopalliance.sourceforge.net)
- AWS SDK for Java - Models (from https://aws.amazon.com/sdkforjava)
- Asynchronous Http Client (from )
- Guava: Google Core Libraries for Java (from https://github.com/google/guava.git)
......@@ -811,6 +816,7 @@ The following software have components provided under the terms of this license:
- Project Lombok (from https://projectlombok.org)
- Project Lombok (from https://projectlombok.org)
- Spring Web (from https://github.com/spring-projects/spring-framework)
- StAX API (from http://stax.codehaus.org/)
- msal4j (from https://github.com/AzureAD/microsoft-authentication-library-for-java)
- reactive-streams (from http://www.reactive-streams.org/)
......@@ -821,6 +827,7 @@ The following software have components provided under the terms of this license:
- Byte Buddy (without dependencies) (from )
- JSON in Java (from https://github.com/douglascrockford/JSON-java)
- JUnit (from http://junit.org)
- JUnit Jupiter (Aggregator) (from https://junit.org/junit5/)
- JavaBeans Activation Framework API jar (from )
- JavaMail API (from )
......
......@@ -13,8 +13,21 @@
// limitations under the License.
package org.opengroup.osdu.schema.provider.aws.impl.schemainfostore;
import com.amazonaws.services.dynamodbv2.datamodeling.PaginatedQueryList;
import com.amazonaws.services.dynamodbv2.model.AttributeValue;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.stream.Collectors;
import javax.annotation.PostConstruct;
import javax.inject.Inject;
import org.joda.time.DateTime;
import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelper;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
......@@ -35,13 +48,11 @@ import org.opengroup.osdu.schema.provider.interfaces.schemainfostore.ISchemaInfo
import org.opengroup.osdu.schema.provider.interfaces.schemastore.ISchemaStore;
import org.opengroup.osdu.schema.util.VersionHierarchyUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Repository;
import javax.annotation.PostConstruct;
import javax.inject.Inject;
import java.text.MessageFormat;
import java.util.*;
import java.util.stream.Collectors;
import com.amazonaws.services.dynamodbv2.datamodeling.PaginatedQueryList;
import com.amazonaws.services.dynamodbv2.model.AttributeValue;
@Repository
public class AwsSchemaInfoStore implements ISchemaInfoStore {
......@@ -62,6 +73,9 @@ public class AwsSchemaInfoStore implements ISchemaInfoStore {
private ISchemaStore schemaStore;
private DynamoDBQueryHelper queryHelper;
@Value("${shared.tenant.name:common}")
private String sharedTenant;
@PostConstruct
public void init() {
......@@ -242,11 +256,11 @@ public class AwsSchemaInfoStore implements ISchemaInfoStore {
@Override
public boolean isUnique(String schemaId, String tenantId) throws ApplicationException {
Set<String> tenantList = new HashSet<>();
tenantList.add(SchemaConstants.ACCOUNT_ID_COMMON_PROJECT);
tenantList.add(sharedTenant);
tenantList.add(tenantId);
// code to call check uniqueness
if (tenantId.equalsIgnoreCase(SchemaConstants.ACCOUNT_ID_COMMON_PROJECT)) {
if (tenantId.equalsIgnoreCase(sharedTenant)) {
List<String> privateTenantList = tenantFactory.listTenantInfo().stream().map(TenantInfo::getDataPartitionId)
.collect(Collectors.toList());
tenantList.addAll(privateTenantList);
......
......@@ -36,3 +36,6 @@ aws.dynamodb.endpoint=dynamodb.${AWS_REGION}.amazonaws.com
# if this is turned on then the service tries to connect to elastic search
management.health.elasticsearch.enabled=false
# Use this property to name your shared tenant name
shared.tenant.name=common
......@@ -13,6 +13,8 @@
// limitations under the License.
package org.opengroup.osdu.schema.provider.aws.impl.schemainfostore;
import static org.junit.Assert.assertEquals;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
......@@ -25,52 +27,56 @@ import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.schema.exceptions.ApplicationException;
import org.opengroup.osdu.schema.provider.aws.config.AwsServiceConfig;
import static org.junit.Assert.assertEquals;
import org.springframework.test.util.ReflectionTestUtils;
@RunWith(MockitoJUnitRunner.class)
public class AwsSchemaInfoStoreTest {
@InjectMocks
private AwsSchemaInfoStore schemaInfoStore;
@InjectMocks
private AwsSchemaInfoStore schemaInfoStore;
@Mock
private AwsServiceConfig serviceConfig;
@Mock
private DpsHeaders headers;
@Mock
private DynamoDBQueryHelper queryHelper;
@Mock
private JaxRsDpsLog logger;
@Mock
private AwsServiceConfig serviceConfig;
@Before
public void setUp() throws Exception {
serviceConfig.amazonRegion = "us-east-1";
serviceConfig.s3DataBucket = "bucket";
serviceConfig.environment = "test";
serviceConfig.s3Endpoint = "s3endpoint";
serviceConfig.dynamoDbEndpoint = "dbendpoint";
serviceConfig.dynamoDbTablePrefix = "pre-";
@Mock
private DpsHeaders headers;
ReflectionTestUtils.setField(schemaInfoStore, "sharedTenant", "common");
@Mock
private DynamoDBQueryHelper queryHelper;
@Mock
private JaxRsDpsLog logger;
@Before
public void setUp() throws Exception {
serviceConfig.amazonRegion = "us-east-1";
serviceConfig.s3DataBucket = "bucket";
serviceConfig.environment = "test";
serviceConfig.s3Endpoint = "s3endpoint";
serviceConfig.dynamoDbEndpoint = "dbendpoint";
serviceConfig.dynamoDbTablePrefix = "pre-";
}
}
@Test
public void isUnique_ifNotExists_returnTrue() throws ApplicationException {
String partitionId = "partitionId";
String schemaId = "schemaId";
Mockito.when(queryHelper.keyExistsInTable(Mockito.any(), Mockito.any())).thenReturn(false);
Boolean actual = schemaInfoStore.isUnique(schemaId, partitionId);
assertEquals(true, actual);
}
@Test
public void isUnique_ifNotExists_returnTrue() throws ApplicationException {
String partitionId = "partitionId";
String schemaId = "schemaId";
Mockito.when(queryHelper.keyExistsInTable(Mockito.any(), Mockito.any())).thenReturn(false);
Boolean actual = schemaInfoStore.isUnique(schemaId, partitionId);
assertEquals(true, actual);
}
@Test
public void isUnique_ifExists_returnFalse() throws ApplicationException {
String partitionId = "partitionId";
String schemaId = "schemaId";
Mockito.when(queryHelper.keyExistsInTable(Mockito.any(), Mockito.any())).thenReturn(true);
Boolean actual = schemaInfoStore.isUnique(schemaId, partitionId);
assertEquals(false, actual);
}
@Test
public void isUnique_ifExists_returnFalse() throws ApplicationException {
String partitionId = "partitionId";
String schemaId = "schemaId";
Mockito.when(queryHelper.keyExistsInTable(Mockito.any(), Mockito.any())).thenReturn(true);
Boolean actual = schemaInfoStore.isUnique(schemaId, partitionId);
assertEquals(false, actual);
}
}
\ No newline at end of file
......@@ -44,6 +44,7 @@ import org.opengroup.osdu.schema.model.SchemaRequest;
import org.opengroup.osdu.schema.provider.interfaces.schemainfostore.ISchemaInfoStore;
import org.opengroup.osdu.schema.util.VersionHierarchyUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Repository;
import lombok.extern.java.Log;
......@@ -69,6 +70,9 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore {
@Autowired
private String cosmosDBName;
@Value("${shared.tenant.name:common}")
private String sharedTenant;
@Autowired
JaxRsDpsLog log;
......@@ -326,13 +330,13 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore {
@Override
public boolean isUnique(String schemaId, String tenantId) throws ApplicationException {
Set<String> tenantList = new HashSet<>();
tenantList.add(SchemaConstants.ACCOUNT_ID_COMMON_PROJECT);
tenantList.add(sharedTenant);
tenantList.add(tenantId);
/* TODO : Below code enables uniqueness check across tenants and is redundant now. This will be handled/updated as part
of data partition changes.
*/
if (tenantId.equalsIgnoreCase(SchemaConstants.ACCOUNT_ID_COMMON_PROJECT)) {
if (tenantId.equalsIgnoreCase(sharedTenant)) {
List<String> privateTenantList = tenantFactory.listTenantInfo().stream().map(TenantInfo::getName)
.collect(Collectors.toList());
tenantList.addAll(privateTenantList);
......
......@@ -54,3 +54,6 @@ logging.slf4jlogger.enabled=true
tenantFactoryImpl.required=true
azure.blobStore.required=true
# Use this property to name your shared tenant name
shared.tenant.name=common
......@@ -42,6 +42,7 @@ import org.opengroup.osdu.schema.model.QueryParams;
import org.opengroup.osdu.schema.model.SchemaIdentity;
import org.opengroup.osdu.schema.model.SchemaInfo;
import org.opengroup.osdu.schema.model.SchemaRequest;
import org.springframework.test.util.ReflectionTestUtils;
import java.io.IOException;
import java.util.*;
......@@ -93,6 +94,7 @@ public class AzureSchemaInfoStoreTest {
@Before
public void init() {
initMocks(this);
ReflectionTestUtils.setField(schemaInfoStore, "sharedTenant", "common");
doReturn(dataPartitionId).when(headers).getPartitionId();
}
......
......@@ -65,8 +65,8 @@ public class GoogleSchemaInfoStore implements ISchemaInfoStore {
@Autowired
JaxRsDpsLog log;
@Value("${account.id.common.project}")
private String commonAccountId;
@Value("${shared.tenant.name:common}")
private String sharedTenant;
/**
* Method to get schemaInfo from google store
......@@ -258,9 +258,6 @@ public class GoogleSchemaInfoStore implements ISchemaInfoStore {
@Override
public List<SchemaInfo> getSchemaInfoList(QueryParams queryParams, String tenantId) throws ApplicationException {
List<SchemaInfo> schemaList = new LinkedList<>();
if (SchemaConstants.ACCOUNT_ID_COMMON_PROJECT.equals(tenantId)) {
return schemaList;
}
Datastore datastore = dataStoreFactory.getDatastore(tenantId, SchemaConstants.NAMESPACE);
List<Filter> filterList = getFilters(queryParams);
......@@ -311,11 +308,11 @@ public class GoogleSchemaInfoStore implements ISchemaInfoStore {
public boolean isUnique(String schemaId, String tenantId) throws ApplicationException {
Set<String> tenantList = new HashSet<>();
tenantList.add(commonAccountId);
tenantList.add(sharedTenant);
tenantList.add(tenantId);
// code to call check uniqueness
if (tenantId.equalsIgnoreCase(commonAccountId)) {
if (tenantId.equalsIgnoreCase(sharedTenant)) {
List<String> privateTenantList = tenantFactory.listTenantInfo().stream().map(TenantInfo::getDataPartitionId)
.collect(Collectors.toList());
tenantList.addAll(privateTenantList);
......@@ -335,11 +332,12 @@ public class GoogleSchemaInfoStore implements ISchemaInfoStore {
return true;
}
public String getCommonAccountId() {
return commonAccountId;
}
public String getSharedTenant() {
return sharedTenant;
}
public void setSharedTenant(String sharedTenant) {
this.sharedTenant = sharedTenant;
}
public void setCommonAccountId(String commonAccountId) {
this.commonAccountId = commonAccountId;
}
}
\ No newline at end of file
......@@ -9,6 +9,7 @@ import org.opengroup.osdu.schema.exceptions.ApplicationException;
import org.opengroup.osdu.schema.exceptions.NotFoundException;
import org.opengroup.osdu.schema.provider.interfaces.schemastore.ISchemaStore;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Repository;
import com.google.cloud.storage.Blob;
......@@ -33,6 +34,9 @@ public class GoogleSchemaStore implements ISchemaStore {
@Autowired
TenantFactory tenantFactory;
@Value("${shared.tenant.name:common}")
private String sharedTenant;
@Autowired
JaxRsDpsLog log;
......@@ -49,9 +53,6 @@ public class GoogleSchemaStore implements ISchemaStore {
*/
@Override
public String getSchema(String dataPartitionId, String filePath) throws ApplicationException, NotFoundException {
if (SchemaConstants.ACCOUNT_ID_COMMON_PROJECT.equals(dataPartitionId)) {
throw new NotFoundException(SchemaConstants.SCHEMA_NOT_PRESENT);
}
filePath = filePath + SchemaConstants.JSON_EXTENSION;
String bucketname = getSchemaBucketName(dataPartitionId);
Storage storage = storageFactory.get(tenantFactory.getTenantInfo(dataPartitionId));
......
......@@ -7,4 +7,6 @@ management.health.elasticsearch.enabled=false
management.endpoints.web.base-path=/
management.endpoints.web.path-mapping.health=health
LOG_PREFIX=schema
account.id.common.project=common
# Use this property to name your shared tenant name
shared.tenant.name=common
......@@ -9,6 +9,7 @@ import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
......@@ -32,6 +33,7 @@ import org.opengroup.osdu.schema.model.SchemaIdentity;
import org.opengroup.osdu.schema.model.SchemaInfo;
import org.opengroup.osdu.schema.model.SchemaRequest;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.test.util.ReflectionTestUtils;
import com.google.cloud.Timestamp;
import com.google.cloud.datastore.Blob;
......@@ -104,6 +106,11 @@ public class GoogleSchemaInfoStoreTest {
@Rule
public ExpectedException expectedException = ExpectedException.none();
@Before
public void setUp() {
ReflectionTestUtils.setField(schemaInfoStore, "sharedTenant", "common");
}
@Test
public void testGetLatestMinorVersion_ReturnNull() throws NotFoundException, ApplicationException {
......@@ -172,7 +179,7 @@ public class GoogleSchemaInfoStoreTest {
KeyFactory storageKeyFactory = mock(KeyFactory.class);
String schemaId = "schemaId";
String tenantId = "tenant";
schemaInfoStore.setCommonAccountId(tenantId);
schemaInfoStore.setSharedTenant(tenantId);
Mockito.when(tenantFactory.getTenantInfo("tenant")).thenReturn(tenantInfo);
Mockito.when(tenantFactory.getTenantInfo("common")).thenReturn(tenantInfoCommon);
Mockito.when(tenantInfo.getName()).thenReturn("tenant");
......
package org.opengroup.osdu.schema.impl.schemastore;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
......@@ -17,6 +18,7 @@ import org.opengroup.osdu.schema.constants.SchemaConstants;
import org.opengroup.osdu.schema.exceptions.ApplicationException;
import org.opengroup.osdu.schema.exceptions.NotFoundException;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.test.util.ReflectionTestUtils;
import com.google.cloud.storage.Blob;
import com.google.cloud.storage.BlobId;
......@@ -59,6 +61,11 @@ public class GoogleSchemaStoreTest {
private static final String FILE_PATH = "/test-folder/test-file";
private static final String CONTENT = "Hello World";
@Before
public void setUp() {
ReflectionTestUtils.setField(schemaStore, "sharedTenant", "common");
}
@Test
public void testCreateSchema() throws ApplicationException {
......
......@@ -35,6 +35,7 @@ import org.opengroup.osdu.schema.provider.interfaces.schemainfostore.ISchemaInfo
import org.opengroup.osdu.schema.provider.interfaces.schemastore.ISchemaStore;
import org.opengroup.osdu.schema.util.VersionHierarchyUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Repository;
import org.springframework.web.context.annotation.RequestScope;
......@@ -43,7 +44,6 @@ import com.cloudant.client.api.query.Expression;
import com.cloudant.client.api.query.QueryBuilder;
import com.cloudant.client.api.query.QueryResult;
import com.cloudant.client.api.query.Selector;
import com.google.gson.Gson;
/**
* Repository class to to register Schema in IBM Document Store.
......@@ -67,6 +67,9 @@ public class IbmSchemaInfoStore extends IbmDocumentStore implements ISchemaInfoS
@Autowired
private ISchemaStore schemaStore;
@Value("${shared.tenant.name:common}")
private String sharedTenant;
@PostConstruct
......@@ -78,11 +81,11 @@ public class IbmSchemaInfoStore extends IbmDocumentStore implements ISchemaInfoS
@Override
public boolean isUnique(String schemaId, String tenantId) throws ApplicationException {
Set<String> tenantList = new HashSet<>();
tenantList.add(SchemaConstants.ACCOUNT_ID_COMMON_PROJECT);
tenantList.add(sharedTenant);
tenantList.add(tenantId);
// code to call check uniqueness
if (tenantId.equalsIgnoreCase(SchemaConstants.ACCOUNT_ID_COMMON_PROJECT)) {
if (tenantId.equalsIgnoreCase(sharedTenant)) {
List<String> privateTenantList = tenantFactory.listTenantInfo().stream().map(TenantInfo::getDataPartitionId)
.collect(Collectors.toList());
tenantList.addAll(privateTenantList);
......
......@@ -20,3 +20,7 @@ ibm.tenant.db.password=TODO
ibm.tenant.db.name=TODO
spring.main.allow-bean-definition-overriding=true
# Use this property to name your shared tenant name
shared.tenant.name=common
\ No newline at end of file
......@@ -5,13 +5,15 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import java.net.MalformedURLException;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;
import static org.mockito.ArgumentMatchers.*;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
......@@ -24,7 +26,6 @@ import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
import org.opengroup.osdu.core.ibm.cloudant.IBMCloudantClientFactory;
import org.opengroup.osdu.core.ibm.multitenancy.TenantFactory;
import org.opengroup.osdu.core.ibm.objectstorage.CloudObjectStorageFactory;
import org.opengroup.osdu.schema.constants.SchemaConstants;
import org.opengroup.osdu.schema.enums.SchemaScope;
import org.opengroup.osdu.schema.enums.SchemaStatus;
......@@ -38,11 +39,11 @@ import org.opengroup.osdu.schema.model.SchemaInfo;
import org.opengroup.osdu.schema.model.SchemaRequest;
import org.opengroup.osdu.schema.provider.ibm.SchemaDoc;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.test.util.ReflectionTestUtils;
import com.cloudant.client.api.Database;
import com.cloudant.client.api.query.QueryResult;
import com.cloudant.client.org.lightcouch.DocumentConflictException;
import com.ibm.cloud.objectstorage.services.s3.AmazonS3;
@RunWith(SpringJUnit4ClassRunner.class)
......@@ -95,6 +96,10 @@ public class IbmSchemaInfoStoreTest {
private static final String dataPartitionId = "testPartitionId";
@Before
public void setUp() {
ReflectionTestUtils.setField(schemaInfoStore, "sharedTenant", "common");
}