From 71a0c01e8adc16d59110426270b344d58c2e1859 Mon Sep 17 00:00:00 2001 From: Aman Verma Date: Tue, 10 Aug 2021 18:35:06 +0530 Subject: [PATCH 01/24] implementing new interface methods for azure --- .../schemainfostore/AzureAuthorityStore.java | 37 +++++++++++++-- .../AzureAuthorityStoreTest.java | 47 ++++++++++++++++++- 2 files changed, 79 insertions(+), 5 deletions(-) diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureAuthorityStore.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureAuthorityStore.java index 494fb590..31e3419f 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureAuthorityStore.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureAuthorityStore.java @@ -77,8 +77,7 @@ public class AzureAuthorityStore implements IAuthorityStore { AuthorityDoc authorityDoc; if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { - authorityDoc = cosmosStore.findItem(systemResourceConfig.getCosmosDatabase(), authorityContainer, id, authorityId, AuthorityDoc.class) - .orElseThrow(() -> new NotFoundException("bad input parameter")); + return this.getSystemAuthority(authorityId); } else { authorityDoc = cosmosStore.findItem(headers.getPartitionId(), cosmosDBName, authorityContainer, id, authorityId, AuthorityDoc.class) .orElseThrow(() -> new NotFoundException("bad input parameter")); @@ -87,6 +86,15 @@ public class AzureAuthorityStore implements IAuthorityStore { return authorityDoc.getAuthority(); } + @Override + public Authority getSystemAuthority(String authorityId) throws NotFoundException, ApplicationException { + AuthorityDoc authorityDoc; + authorityDoc = cosmosStore.findItem(systemResourceConfig.getCosmosDatabase(), authorityContainer, authorityId, authorityId, AuthorityDoc.class) + .orElseThrow(() -> new NotFoundException("bad input parameter")); + + return authorityDoc.getAuthority(); + } + /** * Method to register an authority in azure store. * @param authority @@ -101,7 +109,7 @@ public class AzureAuthorityStore implements IAuthorityStore { try { AuthorityDoc authorityDoc = new AuthorityDoc(id, authority); if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { - cosmosStore.createItem(systemResourceConfig.getCosmosDatabase(), authorityContainer, id, authorityDoc); + return this.createSystemAuthority(authority); } else { cosmosStore.createItem(headers.getPartitionId(), cosmosDBName, authorityContainer, id, authorityDoc); } @@ -119,4 +127,27 @@ public class AzureAuthorityStore implements IAuthorityStore { log.info(SchemaConstants.AUTHORITY_CREATED); return authority; } + + @Override + public Authority createSystemAuthority(Authority authority) throws ApplicationException, BadRequestException { + try { + AuthorityDoc authorityDoc = new AuthorityDoc(authority.getAuthorityId(), authority); + if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { + cosmosStore.createItem(systemResourceConfig.getCosmosDatabase(), authorityContainer, authority.getAuthorityId(), authorityDoc); + } else { + cosmosStore.createItem(headers.getPartitionId(), cosmosDBName, authorityContainer, authority.getAuthorityId(), authorityDoc); + } + } catch (AppException ex) { + if (ex.getError().getCode() == 409) { + log.warning(SchemaConstants.AUTHORITY_EXISTS_ALREADY_REGISTERED); + throw new BadRequestException(MessageFormat.format(SchemaConstants.AUTHORITY_EXISTS_EXCEPTION, + authority.getAuthorityId())); + } else { + log.error(MessageFormat.format(SchemaConstants.OBJECT_INVALID, ex.getMessage())); + throw new ApplicationException(SchemaConstants.INVALID_INPUT); + } + } + + return authority; + } } diff --git a/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureAuthorityStoreTest.java b/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureAuthorityStoreTest.java index 7010741d..4deb654e 100644 --- a/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureAuthorityStoreTest.java +++ b/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureAuthorityStoreTest.java @@ -108,12 +108,17 @@ public class AzureAuthorityStoreTest { .findItem( eq(systemCosmosDBName), any(), - eq(sharedTenantId + ":" + authorityId), + eq(authorityId), eq(partitionKey), any()); + assertNotNull(store.getSystemAuthority(authorityId)); + assertEquals(authorityId, store.getSystemAuthority(authorityId).getAuthorityId()); + + // This is temporary and will be removed once schema-core starts consuming *system* methods assertNotNull(store.get(authorityId)); assertEquals(authorityId, store.get(authorityId).getAuthorityId()); + } @Test @@ -151,6 +156,18 @@ public class AzureAuthorityStoreTest { eq(sharedTenantId + ":" + ""), eq(dataPartitionId), any()); + + try { + store.getSystemAuthority(""); + fail("Should not succeed"); + } catch (NotFoundException e) { + assertEquals("bad input parameter", e.getMessage()); + + } catch (Exception e) { + fail("Should not get different exception"); + } + + // This is temporary and will be removed once schema-core starts consuming *system* methods try { store.get(""); fail("Should not succeed"); @@ -172,6 +189,10 @@ public class AzureAuthorityStoreTest { public void testCreateAuthority_PublicSchemas() throws ApplicationException, BadRequestException { Mockito.when(headers.getPartitionId()).thenReturn(sharedTenantId); doNothing().when(cosmosStore).createItem(eq(systemCosmosDBName), any(), eq(partitionKey), any()); + + assertNotNull(store.createSystemAuthority(mockAuthority)); + + // This is temporary and will be removed once schema-core starts consuming *system* methods assertNotNull(store.create(mockAuthority)); } @@ -197,8 +218,19 @@ public class AzureAuthorityStoreTest { throws NotFoundException, ApplicationException, BadRequestException, IOException { Mockito.when(headers.getPartitionId()).thenReturn(sharedTenantId); AppException exception = getMockAppException(409); - doThrow(exception).when(cosmosStore).createItem(eq(systemCosmosDBName), any(), eq("common:testAuthorityId"), any()); + doThrow(exception).when(cosmosStore).createItem(eq(systemCosmosDBName), any(), eq("testAuthorityId"), any()); + + try { + store.createSystemAuthority(mockAuthority); + fail("Should not succeed"); + } catch (BadRequestException e) { + assertEquals("Authority already registered with Id: testAuthorityId", e.getMessage()); + + } catch (Exception e) { + fail("Should not get different exception"); + } + // This is temporary and will be removed once schema-core starts consuming *system* methods try { store.create(mockAuthority); fail("Should not succeed"); @@ -230,6 +262,17 @@ public class AzureAuthorityStoreTest { Mockito.when(headers.getPartitionId()).thenReturn(sharedTenantId); AppException exception = getMockAppException(500); doThrow(exception).when(cosmosStore).createItem(systemCosmosDBName, any(), eq(partitionKey), any()); + + try { + store.createSystemAuthority(mockAuthority); + fail("Should not succeed"); + } catch (ApplicationException e) { + assertEquals(SchemaConstants.INVALID_INPUT, e.getMessage()); + } catch (Exception e) { + fail("Should not get different exception"); + } + + // This is temporary and will be removed once schema-core starts consuming *system* methods try { store.create(mockAuthority); fail("Should not succeed"); -- GitLab From f435cbfe12ac8a0cfac937e2334f0f36e16bdacf Mon Sep 17 00:00:00 2001 From: Aman Verma Date: Wed, 11 Aug 2021 11:58:17 +0530 Subject: [PATCH 02/24] refactoring the exception handling --- .../schemainfostore/AzureAuthorityStore.java | 29 +++++++++---------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureAuthorityStore.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureAuthorityStore.java index 31e3419f..c99d131f 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureAuthorityStore.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureAuthorityStore.java @@ -114,14 +114,7 @@ public class AzureAuthorityStore implements IAuthorityStore { cosmosStore.createItem(headers.getPartitionId(), cosmosDBName, authorityContainer, id, authorityDoc); } } catch (AppException ex) { - if (ex.getError().getCode() == 409) { - log.warning(SchemaConstants.AUTHORITY_EXISTS_ALREADY_REGISTERED); - throw new BadRequestException(MessageFormat.format(SchemaConstants.AUTHORITY_EXISTS_EXCEPTION, - authority.getAuthorityId())); - } else { - log.error(MessageFormat.format(SchemaConstants.OBJECT_INVALID, ex.getMessage())); - throw new ApplicationException(SchemaConstants.INVALID_INPUT); - } + handleAppException(ex, authority); } log.info(SchemaConstants.AUTHORITY_CREATED); @@ -138,16 +131,20 @@ public class AzureAuthorityStore implements IAuthorityStore { cosmosStore.createItem(headers.getPartitionId(), cosmosDBName, authorityContainer, authority.getAuthorityId(), authorityDoc); } } catch (AppException ex) { - if (ex.getError().getCode() == 409) { - log.warning(SchemaConstants.AUTHORITY_EXISTS_ALREADY_REGISTERED); - throw new BadRequestException(MessageFormat.format(SchemaConstants.AUTHORITY_EXISTS_EXCEPTION, - authority.getAuthorityId())); - } else { - log.error(MessageFormat.format(SchemaConstants.OBJECT_INVALID, ex.getMessage())); - throw new ApplicationException(SchemaConstants.INVALID_INPUT); - } + handleAppException(ex, authority); } return authority; } + + private void handleAppException(AppException ex, Authority authority) throws ApplicationException, BadRequestException { + if (ex.getError().getCode() == 409) { + log.warning(SchemaConstants.AUTHORITY_EXISTS_ALREADY_REGISTERED); + throw new BadRequestException(MessageFormat.format(SchemaConstants.AUTHORITY_EXISTS_EXCEPTION, + authority.getAuthorityId())); + } else { + log.error(MessageFormat.format(SchemaConstants.OBJECT_INVALID, ex.getMessage())); + throw new ApplicationException(SchemaConstants.INVALID_INPUT); + } + } } -- GitLab From 7d96dded7c18c633ab66f941da4084b0767d20e4 Mon Sep 17 00:00:00 2001 From: Aman Verma Date: Wed, 11 Aug 2021 12:04:25 +0530 Subject: [PATCH 03/24] removing redundant if check --- .../azure/impl/schemainfostore/AzureAuthorityStore.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureAuthorityStore.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureAuthorityStore.java index c99d131f..d3332c20 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureAuthorityStore.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureAuthorityStore.java @@ -125,15 +125,12 @@ public class AzureAuthorityStore implements IAuthorityStore { public Authority createSystemAuthority(Authority authority) throws ApplicationException, BadRequestException { try { AuthorityDoc authorityDoc = new AuthorityDoc(authority.getAuthorityId(), authority); - if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { - cosmosStore.createItem(systemResourceConfig.getCosmosDatabase(), authorityContainer, authority.getAuthorityId(), authorityDoc); - } else { - cosmosStore.createItem(headers.getPartitionId(), cosmosDBName, authorityContainer, authority.getAuthorityId(), authorityDoc); - } + cosmosStore.createItem(systemResourceConfig.getCosmosDatabase(), authorityContainer, authority.getAuthorityId(), authorityDoc); } catch (AppException ex) { handleAppException(ex, authority); } + log.info(SchemaConstants.AUTHORITY_CREATED); return authority; } -- GitLab From 166565ca5bd5b74a622d30b54969198c88cb0ea7 Mon Sep 17 00:00:00 2001 From: Aman Verma Date: Wed, 11 Aug 2021 12:37:43 +0530 Subject: [PATCH 04/24] implementing other interfaces --- .../schemainfostore/AzureEntityTypeStore.java | 47 +++++++++++++----- .../schemainfostore/AzureSourceStore.java | 48 ++++++++++++++----- .../AzureEntityTypeStoreTest.java | 48 +++++++++++++++++-- .../schemainfostore/AzureSourceStoreTest.java | 19 ++++++-- 4 files changed, 131 insertions(+), 31 deletions(-) diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureEntityTypeStore.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureEntityTypeStore.java index 7a88a690..71c55d76 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureEntityTypeStore.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureEntityTypeStore.java @@ -74,8 +74,7 @@ public class AzureEntityTypeStore implements IEntityTypeStore { EntityTypeDoc entityTypeDoc; if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { - entityTypeDoc = cosmosStore.findItem(systemResourceConfig.getCosmosDatabase(), entityTypeContainer, id, entityTypeId, EntityTypeDoc.class) - .orElseThrow(() -> new NotFoundException("bad input parameter")); + return this.getSystemEntity(entityTypeId); } else { entityTypeDoc = cosmosStore.findItem(headers.getPartitionId(), cosmosDBName, entityTypeContainer, id, entityTypeId, EntityTypeDoc.class) .orElseThrow(() -> new NotFoundException("bad input parameter")); @@ -84,6 +83,15 @@ public class AzureEntityTypeStore implements IEntityTypeStore { return entityTypeDoc.getEntityType(); } + @Override + public EntityType getSystemEntity(String entityTypeId) throws NotFoundException, ApplicationException { + EntityTypeDoc entityTypeDoc; + entityTypeDoc = cosmosStore.findItem(systemResourceConfig.getCosmosDatabase(), entityTypeContainer, entityTypeId, entityTypeId, EntityTypeDoc.class) + .orElseThrow(() -> new NotFoundException("bad input parameter")); + + return entityTypeDoc.getEntityType(); + } + /** * Method to create entityType in azure store * @param entityType @@ -98,22 +106,39 @@ public class AzureEntityTypeStore implements IEntityTypeStore { try { EntityTypeDoc entityTypeDoc = new EntityTypeDoc(id, entityType); if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { - cosmosStore.createItem(systemResourceConfig.getCosmosDatabase(), entityTypeContainer, id, entityTypeDoc); + this.createSystemEntity(entityType); } else { cosmosStore.createItem(headers.getPartitionId(), cosmosDBName, entityTypeContainer, id, entityTypeDoc); } } catch (AppException ex) { - if (ex.getError().getCode() == 409) { - log.warning(SchemaConstants.ENTITY_TYPE_EXISTS); - throw new BadRequestException(MessageFormat.format(SchemaConstants.ENTITY_TYPE_EXISTS_EXCEPTION, - entityType.getEntityTypeId())); - } else { - log.error(MessageFormat.format(SchemaConstants.OBJECT_INVALID, ex.getMessage())); - throw new ApplicationException(SchemaConstants.INVALID_INPUT); - } + handleAppException(ex, entityType); + } + + log.info(SchemaConstants.ENTITY_TYPE_CREATED); + return entityType; + } + + @Override + public EntityType createSystemEntity(EntityType entityType) throws BadRequestException, ApplicationException { + try { + EntityTypeDoc entityTypeDoc = new EntityTypeDoc(entityType.getEntityTypeId(), entityType); + cosmosStore.createItem(systemResourceConfig.getCosmosDatabase(), entityTypeContainer, entityType.getEntityTypeId(), entityTypeDoc); + } catch (AppException ex) { + handleAppException(ex, entityType); } log.info(SchemaConstants.ENTITY_TYPE_CREATED); return entityType; } + + private void handleAppException(AppException ex, EntityType entityType) throws BadRequestException, ApplicationException { + if (ex.getError().getCode() == 409) { + log.warning(SchemaConstants.ENTITY_TYPE_EXISTS); + throw new BadRequestException(MessageFormat.format(SchemaConstants.ENTITY_TYPE_EXISTS_EXCEPTION, + entityType.getEntityTypeId())); + } else { + log.error(MessageFormat.format(SchemaConstants.OBJECT_INVALID, ex.getMessage())); + throw new ApplicationException(SchemaConstants.INVALID_INPUT); + } + } } diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSourceStore.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSourceStore.java index 2b6be778..bc72c99d 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSourceStore.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSourceStore.java @@ -72,8 +72,7 @@ public class AzureSourceStore implements ISourceStore { SourceDoc sourceDoc; if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { - sourceDoc = cosmosStore.findItem(systemResourceConfig.getCosmosDatabase(), sourceContainer, id, sourceId, SourceDoc.class) - .orElseThrow(() -> new NotFoundException("bad input parameter")); + return this.getSystemSource(sourceId); } else { sourceDoc = cosmosStore.findItem(headers.getPartitionId(), cosmosDBName, sourceContainer, id, sourceId, SourceDoc.class) .orElseThrow(() -> new NotFoundException("bad input parameter")); @@ -82,6 +81,16 @@ public class AzureSourceStore implements ISourceStore { return sourceDoc.getSource(); } + @Override + public Source getSystemSource(String sourceId) throws NotFoundException, ApplicationException { + SourceDoc sourceDoc; + + sourceDoc = cosmosStore.findItem(systemResourceConfig.getCosmosDatabase(), sourceContainer, sourceId, sourceId, SourceDoc.class) + .orElseThrow(() -> new NotFoundException("bad input parameter")); + + return sourceDoc.getSource(); + } + /** * Method to create Source in azure store * @param source @@ -96,22 +105,39 @@ public class AzureSourceStore implements ISourceStore { try { SourceDoc sourceDoc = new SourceDoc(id, source); if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { - cosmosStore.createItem(systemResourceConfig.getCosmosDatabase(), sourceContainer, id, sourceDoc); + return this.createSystemSource(source); } else { cosmosStore.createItem(headers.getPartitionId(), cosmosDBName, sourceContainer, id, sourceDoc); } } catch (AppException ex) { - if (ex.getError().getCode() == 409) { - log.warning(SchemaConstants.SOURCE_EXISTS); - throw new BadRequestException(MessageFormat.format(SchemaConstants.SOURCE_EXISTS_EXCEPTION, - source.getSourceId())); - } else { - log.error(MessageFormat.format(SchemaConstants.OBJECT_INVALID, ex.getMessage())); - throw new ApplicationException(SchemaConstants.INVALID_INPUT); - } + handleAppException(ex, source); } log.info(SchemaConstants.SOURCE_CREATED); return source; } + + @Override + public Source createSystemSource(Source source) throws BadRequestException, ApplicationException { + try { + SourceDoc sourceDoc = new SourceDoc(source.getSourceId(), source); + cosmosStore.createItem(systemResourceConfig.getCosmosDatabase(), sourceContainer, source.getSourceId(), sourceDoc); + } catch (AppException ex) { + handleAppException(ex, source); + } + + log.info(SchemaConstants.SOURCE_CREATED); + return source; + } + + private void handleAppException(AppException ex, Source source) throws BadRequestException, ApplicationException { + if (ex.getError().getCode() == 409) { + log.warning(SchemaConstants.SOURCE_EXISTS); + throw new BadRequestException(MessageFormat.format(SchemaConstants.SOURCE_EXISTS_EXCEPTION, + source.getSourceId())); + } else { + log.error(MessageFormat.format(SchemaConstants.OBJECT_INVALID, ex.getMessage())); + throw new ApplicationException(SchemaConstants.INVALID_INPUT); + } + } } diff --git a/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureEntityTypeStoreTest.java b/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureEntityTypeStoreTest.java index e5cb7138..90607572 100644 --- a/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureEntityTypeStoreTest.java +++ b/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureEntityTypeStoreTest.java @@ -109,10 +109,14 @@ public class AzureEntityTypeStoreTest { .findItem( eq(systemCosmosDBName), any(), - eq(sharedTenantId + ":" + entityTypeId), + eq(entityTypeId), eq(partitionKey), any()); + assertNotNull(store.getSystemEntity(entityTypeId)); + assertEquals(entityTypeId, store.getSystemEntity(entityTypeId).getEntityTypeId()); + + // This is temporary and will be removed once schema-core starts consuming *system* methods assertNotNull(store.get(entityTypeId)); assertEquals(entityTypeId, store.get(entityTypeId).getEntityTypeId()); } @@ -150,10 +154,21 @@ public class AzureEntityTypeStoreTest { .findItem( eq(systemCosmosDBName), any(), - eq(sharedTenantId + ":" + ""), + eq(""), eq(partitionKey), any()); + try { + store.getSystemEntity(""); + fail("Should not succeed"); + } catch (NotFoundException e) { + assertEquals("bad input parameter", e.getMessage()); + + } catch (Exception e) { + fail("Should not get different exception"); + } + + // This is temporary and will be removed once schema-core starts consuming *system* methods try { store.get(""); fail("Should not succeed"); @@ -177,6 +192,10 @@ public class AzureEntityTypeStoreTest { Mockito.when(headers.getPartitionId()).thenReturn(sharedTenantId); Mockito.when(mockEntityType.getEntityTypeId()).thenReturn(entityTypeId); doNothing().when(cosmosStore).createItem(eq(systemCosmosDBName), any(), eq(partitionKey), any()); + + assertNotNull(store.createSystemEntity(mockEntityType)); + + // This is temporary and will be removed once schema-core starts consuming *system* methods assertNotNull(store.create(mockEntityType)); } @@ -202,8 +221,19 @@ public class AzureEntityTypeStoreTest { throws NotFoundException, ApplicationException, BadRequestException, IOException { Mockito.when(headers.getPartitionId()).thenReturn(sharedTenantId); AppException exception = getMockAppException(409); - doThrow(exception).when(cosmosStore).createItem(eq(systemCosmosDBName), any(), eq("common:testEntityId"), any()); + doThrow(exception).when(cosmosStore).createItem(eq(systemCosmosDBName), any(), eq("testEntityId"), any()); + try { + store.createSystemEntity(mockEntityType); + fail("Should not succeed"); + } catch (BadRequestException e) { + assertEquals("EntityType already registered with Id: testEntityId", e.getMessage()); + + } catch (Exception e) { + fail("Should not get different exception"); + } + + // This is temporary and will be removed once schema-core starts consuming *system* methods try { store.create(mockEntityType); fail("Should not succeed"); @@ -236,8 +266,18 @@ public class AzureEntityTypeStoreTest { throws NotFoundException, ApplicationException, BadRequestException, CosmosException { Mockito.when(headers.getPartitionId()).thenReturn(sharedTenantId); AppException exception = getMockAppException(500); - doThrow(exception).when(cosmosStore).createItem(eq(systemCosmosDBName), any(), eq("common:testEntityId"), any()); + doThrow(exception).when(cosmosStore).createItem(eq(systemCosmosDBName), any(), eq("testEntityId"), any()); + + try { + store.createSystemEntity(mockEntityType); + fail("Should not succeed"); + } catch (ApplicationException e) { + assertEquals(SchemaConstants.INVALID_INPUT, e.getMessage()); + } catch (Exception e) { + fail("Should not get different exception"); + } + // This is temporary and will be removed once schema-core starts consuming *system* methods try { store.create(mockEntityType); fail("Should not succeed"); diff --git a/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSourceStoreTest.java b/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSourceStoreTest.java index 7580c6b3..8437f0d5 100644 --- a/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSourceStoreTest.java +++ b/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSourceStoreTest.java @@ -108,9 +108,11 @@ public class AzureSourceStoreTest { .findItem( eq(systemCosmosDBName), any(), - eq(sharedTenantId + ":" + sourceId), + eq(sourceId), eq(partitionKey), any()); + + // This is temporary and will be removed once schema-core starts consuming *system* methods assertNotNull(store.get(sourceId)); assertEquals(sourceId, store.get(sourceId).getSourceId()); } @@ -149,9 +151,11 @@ public class AzureSourceStoreTest { .findItem( eq(systemCosmosDBName), any(), - eq(sharedTenantId + ":" + ""), + eq(""), eq(sourceId), any()); + + // This is temporary and will be removed once schema-core starts consuming *system* methods try { store.get(sourceId); fail("Should not succeed"); @@ -173,7 +177,9 @@ public class AzureSourceStoreTest { @Test public void testCreateSource_PublicSchemas() throws ApplicationException, BadRequestException { Mockito.when(headers.getPartitionId()).thenReturn(sharedTenantId); - doNothing().when(cosmosStore).createItem(eq(systemCosmosDBName), any(), eq("common:testSourceId"), any()); + doNothing().when(cosmosStore).createItem(eq(systemCosmosDBName), any(), eq(sourceId), any()); + + // This is temporary and will be removed once schema-core starts consuming *system* methods assertNotNull(store.create(mockSource)); } @@ -198,7 +204,9 @@ public class AzureSourceStoreTest { throws NotFoundException, ApplicationException, BadRequestException, IOException { Mockito.when(headers.getPartitionId()).thenReturn(sharedTenantId); AppException exception = getMockAppException(409); - doThrow(exception).when(cosmosStore).createItem(eq(systemCosmosDBName), any(), eq("common:testSourceId"), any()); + doThrow(exception).when(cosmosStore).createItem(eq(systemCosmosDBName), any(), eq(sourceId), any()); + + // This is temporary and will be removed once schema-core starts consuming *system* methods try { store.create(mockSource); fail("Should not succeed"); @@ -231,8 +239,9 @@ public class AzureSourceStoreTest { throws NotFoundException, ApplicationException, BadRequestException, CosmosException { Mockito.when(headers.getPartitionId()).thenReturn(sharedTenantId); AppException exception = getMockAppException(500); - doThrow(exception).when(cosmosStore).createItem(eq(systemCosmosDBName), any(), eq("common:testSourceId"), any()); + doThrow(exception).when(cosmosStore).createItem(eq(systemCosmosDBName), any(), eq("testSourceId"), any()); + // This is temporary and will be removed once schema-core starts consuming *system* methods try { store.create(mockSource); fail("Should not succeed"); -- GitLab From 5e7ffbc3f9f52dbd921682c475c0dcf8b9928557 Mon Sep 17 00:00:00 2001 From: Aman Verma Date: Wed, 11 Aug 2021 13:01:15 +0530 Subject: [PATCH 05/24] updating the schemaStore class --- .../impl/schemastore/AzureSchemaStore.java | 29 +++++++++++++++- .../schemainfostore/AzureSourceStoreTest.java | 34 +++++++++++++++++++ .../schemastore/AzureSchemaStoreTest.java | 2 +- 3 files changed, 63 insertions(+), 2 deletions(-) diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemastore/AzureSchemaStore.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemastore/AzureSchemaStore.java index 3462747d..771e1bfd 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemastore/AzureSchemaStore.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemastore/AzureSchemaStore.java @@ -62,12 +62,12 @@ public class AzureSchemaStore implements ISchemaStore { */ @Override public String getSchema(String dataPartitionId, String filePath) throws ApplicationException, NotFoundException { - filePath = dataPartitionId + ":" + filePath + SchemaConstants.JSON_EXTENSION; try { String content = null; if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(dataPartitionId)) { content = blobStore.readFromStorageContainer(filePath, systemResourceConfig.getStorageContainerName()); } else { + filePath = dataPartitionId + ":" + filePath + SchemaConstants.JSON_EXTENSION; content = blobStore.readFromStorageContainer(dataPartitionId, filePath, config.containerName()); } if (content != null) @@ -80,6 +80,23 @@ public class AzureSchemaStore implements ISchemaStore { } } + @Override + public String getSystemSchema(String dataPartitionId, String filePath) throws NotFoundException, ApplicationException { + try { + String content = null; + + content = blobStore.readFromStorageContainer(filePath, systemResourceConfig.getStorageContainerName()); + + if (content != null) + return content; + else + throw new NotFoundException(SchemaConstants.SCHEMA_NOT_PRESENT); + } + catch (Exception ex) { + throw new NotFoundException(SchemaConstants.SCHEMA_NOT_PRESENT); + } + } + /** * Method to write schema to azure Storage given Tenant * @param filePath @@ -106,6 +123,11 @@ public class AzureSchemaStore implements ISchemaStore { } } + @Override + public String createSystemSchema(String filePath, String content) throws ApplicationException { + return null; + } + /** * Delete a schema from azure Storage * @param schemaId @@ -130,4 +152,9 @@ public class AzureSchemaStore implements ISchemaStore { throw new ApplicationException(SchemaConstants.INTERNAL_SERVER_ERROR); } } + + @Override + public boolean cleanSystemSchemaProject(String schemaId) throws ApplicationException { + return false; + } } diff --git a/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSourceStoreTest.java b/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSourceStoreTest.java index 8437f0d5..5dabcdc3 100644 --- a/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSourceStoreTest.java +++ b/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSourceStoreTest.java @@ -112,6 +112,9 @@ public class AzureSourceStoreTest { eq(partitionKey), any()); + assertNotNull(store.getSystemSource(sourceId)); + assertEquals(sourceId, store.getSystemSource(sourceId).getSourceId()); + // This is temporary and will be removed once schema-core starts consuming *system* methods assertNotNull(store.get(sourceId)); assertEquals(sourceId, store.get(sourceId).getSourceId()); @@ -155,6 +158,16 @@ public class AzureSourceStoreTest { eq(sourceId), any()); + try { + store.getSystemSource(sourceId); + fail("Should not succeed"); + } catch (NotFoundException e) { + assertEquals("bad input parameter", e.getMessage()); + + } catch (Exception e) { + fail("Should not get different exception"); + } + // This is temporary and will be removed once schema-core starts consuming *system* methods try { store.get(sourceId); @@ -179,6 +192,8 @@ public class AzureSourceStoreTest { Mockito.when(headers.getPartitionId()).thenReturn(sharedTenantId); doNothing().when(cosmosStore).createItem(eq(systemCosmosDBName), any(), eq(sourceId), any()); + assertNotNull(store.createSystemSource(mockSource)); + // This is temporary and will be removed once schema-core starts consuming *system* methods assertNotNull(store.create(mockSource)); } @@ -206,6 +221,16 @@ public class AzureSourceStoreTest { AppException exception = getMockAppException(409); doThrow(exception).when(cosmosStore).createItem(eq(systemCosmosDBName), any(), eq(sourceId), any()); + try { + store.createSystemSource(mockSource); + fail("Should not succeed"); + } catch (BadRequestException e) { + assertEquals("Source already registered with Id: testSourceId", e.getMessage()); + + } catch (Exception e) { + fail("Should not get different exception"); + } + // This is temporary and will be removed once schema-core starts consuming *system* methods try { store.create(mockSource); @@ -241,6 +266,15 @@ public class AzureSourceStoreTest { AppException exception = getMockAppException(500); doThrow(exception).when(cosmosStore).createItem(eq(systemCosmosDBName), any(), eq("testSourceId"), any()); + try { + store.createSystemSource(mockSource); + fail("Should not succeed"); + } catch (ApplicationException e) { + assertEquals(SchemaConstants.INVALID_INPUT, e.getMessage()); + } catch (Exception e) { + fail("Should not get different exception"); + } + // This is temporary and will be removed once schema-core starts consuming *system* methods try { store.create(mockSource); diff --git a/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemastore/AzureSchemaStoreTest.java b/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemastore/AzureSchemaStoreTest.java index f05614c3..f545c09a 100644 --- a/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemastore/AzureSchemaStoreTest.java +++ b/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemastore/AzureSchemaStoreTest.java @@ -67,7 +67,7 @@ public class AzureSchemaStoreTest { private static final String containerName = "opendes"; private static final String systemContainerName = "systemContainer"; private static final String filePath = dataPartitionId + ":" + FILE_PATH + SchemaConstants.JSON_EXTENSION; - private static final String filePathPublic = sharedTenantId + ":" + FILE_PATH + SchemaConstants.JSON_EXTENSION; + private static final String filePathPublic = FILE_PATH + SchemaConstants.JSON_EXTENSION; @Before public void init(){ -- GitLab From a22ade8123527c0a90a2969d9ffd82b10e37653c Mon Sep 17 00:00:00 2001 From: Aman Verma Date: Wed, 11 Aug 2021 13:46:37 +0530 Subject: [PATCH 06/24] fixing UTs --- provider/schema-azure/os-schema-azure.ipr | 107 +++++ provider/schema-azure/os-schema-azure.iws | 418 ++++++++++++++++++ .../impl/schemastore/AzureSchemaStore.java | 33 +- .../schemastore/AzureSchemaStoreTest.java | 27 ++ 4 files changed, 576 insertions(+), 9 deletions(-) create mode 100644 provider/schema-azure/os-schema-azure.ipr create mode 100644 provider/schema-azure/os-schema-azure.iws diff --git a/provider/schema-azure/os-schema-azure.ipr b/provider/schema-azure/os-schema-azure.ipr new file mode 100644 index 00000000..884e359a --- /dev/null +++ b/provider/schema-azure/os-schema-azure.ipr @@ -0,0 +1,107 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/provider/schema-azure/os-schema-azure.iws b/provider/schema-azure/os-schema-azure.iws new file mode 100644 index 00000000..03c854e9 --- /dev/null +++ b/provider/schema-azure/os-schema-azure.iws @@ -0,0 +1,418 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemastore/AzureSchemaStore.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemastore/AzureSchemaStore.java index 771e1bfd..10bdfc68 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemastore/AzureSchemaStore.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemastore/AzureSchemaStore.java @@ -65,7 +65,7 @@ public class AzureSchemaStore implements ISchemaStore { try { String content = null; if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(dataPartitionId)) { - content = blobStore.readFromStorageContainer(filePath, systemResourceConfig.getStorageContainerName()); + return this.getSystemSchema(filePath); } else { filePath = dataPartitionId + ":" + filePath + SchemaConstants.JSON_EXTENSION; content = blobStore.readFromStorageContainer(dataPartitionId, filePath, config.containerName()); @@ -81,10 +81,10 @@ public class AzureSchemaStore implements ISchemaStore { } @Override - public String getSystemSchema(String dataPartitionId, String filePath) throws NotFoundException, ApplicationException { + public String getSystemSchema(String filePath) throws NotFoundException, ApplicationException { + filePath = filePath + SchemaConstants.JSON_EXTENSION; try { String content = null; - content = blobStore.readFromStorageContainer(filePath, systemResourceConfig.getStorageContainerName()); if (content != null) @@ -108,12 +108,12 @@ public class AzureSchemaStore implements ISchemaStore { @Override public String createSchema(String filePath, String content) throws ApplicationException { String dataPartitionId = headers.getPartitionId(); - filePath = dataPartitionId + ":" + filePath + SchemaConstants.JSON_EXTENSION; try { if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { - blobStore.writeToStorageContainer(filePath, content, systemResourceConfig.getStorageContainerName()); + return this.createSystemSchema(filePath, content); } else { + filePath = dataPartitionId + ":" + filePath + SchemaConstants.JSON_EXTENSION; blobStore.writeToStorageContainer(dataPartitionId, filePath, content, config.containerName()); } log.info(SchemaConstants.SCHEMA_CREATED); @@ -125,7 +125,14 @@ public class AzureSchemaStore implements ISchemaStore { @Override public String createSystemSchema(String filePath, String content) throws ApplicationException { - return null; + filePath = filePath + SchemaConstants.JSON_EXTENSION; + try { + blobStore.writeToStorageContainer(filePath, content, systemResourceConfig.getStorageContainerName()); + log.info(SchemaConstants.SCHEMA_CREATED); + return filePath; + } catch (Exception ex) { + throw new ApplicationException(SchemaConstants.INTERNAL_SERVER_ERROR); + } } /** @@ -137,13 +144,13 @@ public class AzureSchemaStore implements ISchemaStore { @Override public boolean cleanSchemaProject(String schemaId) throws ApplicationException { String dataPartitionId = headers.getPartitionId(); - String filePath = dataPartitionId + ":" + schemaId + SchemaConstants.JSON_EXTENSION; try { if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { - return blobStore.deleteFromStorageContainer(filePath, systemResourceConfig.getStorageContainerName()); + return this.cleanSystemSchemaProject(schemaId); } else { + String filePath = dataPartitionId + ":" + schemaId + SchemaConstants.JSON_EXTENSION; return blobStore.deleteFromStorageContainer(dataPartitionId, filePath, config.containerName()); } } @@ -155,6 +162,14 @@ public class AzureSchemaStore implements ISchemaStore { @Override public boolean cleanSystemSchemaProject(String schemaId) throws ApplicationException { - return false; + String filePath = schemaId + SchemaConstants.JSON_EXTENSION; + try + { + return blobStore.deleteFromStorageContainer(filePath, systemResourceConfig.getStorageContainerName()); + } + catch (Exception e) + { + throw new ApplicationException(SchemaConstants.INTERNAL_SERVER_ERROR); + } } } diff --git a/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemastore/AzureSchemaStoreTest.java b/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemastore/AzureSchemaStoreTest.java index f545c09a..4745c893 100644 --- a/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemastore/AzureSchemaStoreTest.java +++ b/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemastore/AzureSchemaStoreTest.java @@ -88,6 +88,10 @@ public class AzureSchemaStoreTest { public void testGetSchema_PublicSchemas() throws ApplicationException, NotFoundException { Mockito.when(headers.getPartitionId()).thenReturn(sharedTenantId); doReturn(CONTENT).when(blobStore).readFromStorageContainer(filePathPublic, systemContainerName); + + Assert.assertEquals(CONTENT, schemaStore.getSystemSchema(FILE_PATH)); + + // This is temporary and will be removed once schema-core starts consuming *system* methods Assert.assertEquals(CONTENT, schemaStore.getSchema(sharedTenantId, FILE_PATH)); } @@ -105,6 +109,10 @@ public class AzureSchemaStoreTest { expectedException.expect(NotFoundException.class); expectedException.expectMessage(SchemaConstants.SCHEMA_NOT_PRESENT); doReturn(null).when(blobStore).readFromStorageContainer(filePathPublic, systemContainerName); + + schemaStore.getSystemSchema(FILE_PATH); + + // This is temporary and will be removed once schema-core starts consuming *system* methods schemaStore.getSchema(sharedTenantId, FILE_PATH); } @@ -124,6 +132,10 @@ public class AzureSchemaStoreTest { expectedException.expectMessage(SchemaConstants.SCHEMA_NOT_PRESENT); doThrow(AppException.class).when(blobStore).readFromStorageContainer(filePathPublic, systemContainerName); + + schemaStore.getSystemSchema(FILE_PATH); + + // This is temporary and will be removed once schema-core starts consuming *system* methods schemaStore.getSchema(sharedTenantId, FILE_PATH); } @@ -140,6 +152,9 @@ public class AzureSchemaStoreTest { Mockito.when(headers.getPartitionId()).thenReturn(sharedTenantId); doReturn(true).when(blobStore).deleteFromStorageContainer(filePathPublic, systemContainerName); + Assert.assertEquals(true, schemaStore.cleanSystemSchemaProject(FILE_PATH)); + + // This is temporary and will be removed once schema-core starts consuming *system* methods Boolean result = schemaStore.cleanSchemaProject(FILE_PATH); Assert.assertEquals(true, result); } @@ -160,6 +175,10 @@ public class AzureSchemaStoreTest { expectedException.expectMessage(SchemaConstants.INTERNAL_SERVER_ERROR); doThrow(AppException.class).when(blobStore).deleteFromStorageContainer(filePathPublic, systemContainerName); + + schemaStore.cleanSystemSchemaProject(FILE_PATH); + + // This is temporary and will be removed once schema-core starts consuming *system* methods schemaStore.cleanSchemaProject(FILE_PATH); } @@ -175,6 +194,10 @@ public class AzureSchemaStoreTest { Mockito.when(headers.getPartitionId()).thenReturn(sharedTenantId); doNothing().when(blobStore).writeToStorageContainer(filePathPublic, CONTENT, systemContainerName); + + Assert.assertEquals(filePathPublic, schemaStore.createSystemSchema(FILE_PATH, CONTENT)); + + // This is temporary and will be removed once schema-core starts consuming *system* methods Assert.assertEquals(filePathPublic, schemaStore.createSchema(FILE_PATH, CONTENT)); } @@ -194,6 +217,10 @@ public class AzureSchemaStoreTest { expectedException.expectMessage(SchemaConstants.INTERNAL_SERVER_ERROR); doThrow(AppException.class).when(blobStore).writeToStorageContainer(filePathPublic, CONTENT, systemContainerName); + + schemaStore.createSystemSchema(FILE_PATH, CONTENT); + + // This is temporary and will be removed once schema-core starts consuming *system* methods schemaStore.createSchema(FILE_PATH, CONTENT); } } -- GitLab From 97508ac68993c116c37d0df139f01457bb807d00 Mon Sep 17 00:00:00 2001 From: Aman Verma Date: Wed, 11 Aug 2021 15:41:18 +0530 Subject: [PATCH 07/24] implementing messaging related interfaces --- .../azure/impl/messagebus/MessageBusImpl.java | 41 ++++++++++-- .../impl/messagebus/MessageBusImplTest.java | 63 ++++++++++++++++++- 2 files changed, 98 insertions(+), 6 deletions(-) diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/messagebus/MessageBusImpl.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/messagebus/MessageBusImpl.java index 6877d09b..413b33df 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/messagebus/MessageBusImpl.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/messagebus/MessageBusImpl.java @@ -20,12 +20,15 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.UUID; +import java.util.stream.Collectors; import org.joda.time.DateTime; import org.opengroup.osdu.azure.eventgrid.EventGridTopicStore; import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.model.http.DpsHeaders; +import org.opengroup.osdu.core.common.model.tenant.TenantInfo; +import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory; import org.opengroup.osdu.schema.azure.di.EventGridConfig; import org.opengroup.osdu.schema.azure.impl.messagebus.model.SchemaPubSubInfo; import org.opengroup.osdu.schema.constants.SchemaConstants; @@ -49,6 +52,8 @@ public class MessageBusImpl implements IMessageBus { private AuditLogger auditLogger; @Autowired DpsHeaders headers; + @Autowired + private ITenantFactory tenantFactory; private final static String EVENT_DATA_VERSION = "1.0"; @@ -57,7 +62,7 @@ public class MessageBusImpl implements IMessageBus { if (eventGridConfig.isEventGridEnabled()) { logger.info("Generating event of type {}",eventType); try { - publishToEventGrid(schemaId, eventType); + publishToEventGrid(schemaId, eventType, headers.getPartitionIdWithFallbackToAccountId()); auditLogger.schemaNotificationSuccess(Collections.singletonList(schemaId)); }catch (AppException ex) { @@ -71,7 +76,33 @@ public class MessageBusImpl implements IMessageBus { } } - private void publishToEventGrid(String schemaId, String eventType) { + @Override + public void publishMessageForSystemSchema(String schemaId, String eventType) { + if (eventGridConfig.isEventGridEnabled()) { + logger.info("Generating event of type {}",eventType); + try { + // Publish the event for all the tenants. + List privateTenantList = tenantFactory.listTenantInfo().stream().map(TenantInfo::getName) + .collect(Collectors.toList()); + + for (String tenant : privateTenantList) { + publishToEventGrid(schemaId, eventType, tenant); + } + + auditLogger.schemaNotificationSuccess(Collections.singletonList(schemaId)); + }catch (AppException ex) { + + //We do not want to fail schema creation if notification delivery has failed, hence just logging the exception + auditLogger.schemaNotificationFailure(Collections.singletonList(schemaId)); + logger.warning(SchemaConstants.SCHEMA_NOTIFICATION_FAILED); + } + + }else { + logger.info(SchemaConstants.SCHEMA_NOTIFICATION_IS_DISABLED); + } + } + + private void publishToEventGrid(String schemaId, String eventType, String dataPartitionId) { String messageId = UUID.randomUUID().toString(); SchemaPubSubInfo[] schemaPubSubMsgs = new SchemaPubSubInfo [1]; @@ -79,8 +110,8 @@ public class MessageBusImpl implements IMessageBus { List eventsList = new ArrayList<>(); HashMap message = new HashMap<>(); message.put("data", schemaPubSubMsgs); - message.put(DpsHeaders.ACCOUNT_ID, headers.getPartitionIdWithFallbackToAccountId()); - message.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId()); + message.put(DpsHeaders.ACCOUNT_ID, dataPartitionId); + message.put(DpsHeaders.DATA_PARTITION_ID, dataPartitionId); message.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId()); //EventGridEvent supports array of messages to be triggered in a batch but at present we do not support @@ -95,7 +126,7 @@ public class MessageBusImpl implements IMessageBus { ); eventsList.add(eventGridEvent); logger.info("Schema event created: " + messageId); - eventGridTopicStore.publishToEventGridTopic(headers.getPartitionId(), eventGridConfig.getCustomTopicName(), eventsList); + eventGridTopicStore.publishToEventGridTopic(dataPartitionId, eventGridConfig.getCustomTopicName(), eventsList); logger.info("Schema event generated successfully"); } diff --git a/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/azure/impl/messagebus/MessageBusImplTest.java b/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/azure/impl/messagebus/MessageBusImplTest.java index 48d97c74..0224bb9d 100644 --- a/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/azure/impl/messagebus/MessageBusImplTest.java +++ b/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/azure/impl/messagebus/MessageBusImplTest.java @@ -31,8 +31,10 @@ import static org.mockito.Mockito.when; import static org.mockito.MockitoAnnotations.initMocks; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; +import com.google.common.collect.Lists; import org.junit.Before; import org.junit.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -43,6 +45,8 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.opengroup.osdu.azure.eventgrid.EventGridTopicStore; import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.common.model.http.DpsHeaders; +import org.opengroup.osdu.core.common.model.tenant.TenantInfo; +import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory; import org.opengroup.osdu.schema.azure.di.EventGridConfig; import org.opengroup.osdu.schema.azure.impl.messagebus.model.SchemaPubSubInfo; import org.opengroup.osdu.schema.logging.AuditLogger; @@ -56,6 +60,7 @@ public class MessageBusImplTest { private static final String DATA_PARTITION_WITH_FALLBACK_ACCOUNT_ID = "data-partition-account-id"; private static final String CORRELATION_ID = "correlation-id"; private static final String PARTITION_ID = "partition-id"; + private static final String OTHER_TENANT = "other-tenant-id"; @Mock private EventGridTopicStore eventGridTopicStore; @@ -71,6 +76,9 @@ public class MessageBusImplTest { @Mock private AuditLogger auditLogger; + + @Mock + private ITenantFactory tenantFactory; @InjectMocks private MessageBusImpl messageBusImpl; @@ -92,11 +100,21 @@ public class MessageBusImplTest { //Assert that eventGridTopicStore is not called even once verify(this.eventGridTopicStore, times(0)).publishToEventGridTopic(any(), any(), anyList()); } + + @Test + public void should_publishToEventGrid_WhenFlagIsFalse_PublicSchemas() { + //The schema-notification is turned off + when(this.eventGridConfig.isEventGridEnabled()).thenReturn(false); + //Call publish Message + messageBusImpl.publishMessageForSystemSchema("dummy", "dummy"); + //Assert that eventGridTopicStore is not called even once + verify(this.eventGridTopicStore, times(0)).publishToEventGridTopic(any(), any(), anyList()); + } @Test public void should_publishToEventGrid_WhenFlagIsTrue() { - //The schema-notification is turned off + //The schema-notification is turned on when(this.eventGridConfig.isEventGridEnabled()).thenReturn(true); //The schema-notification is turned off when(this.eventGridConfig.getCustomTopicName()).thenReturn("dummy-topic"); @@ -129,4 +147,47 @@ public class MessageBusImplTest { } + @Test + public void should_publishToEventGrid_WhenFlagIsTrue_PublicSchemas() { + + TenantInfo tenant1 = new TenantInfo(); + tenant1.setName(PARTITION_ID); + tenant1.setDataPartitionId(PARTITION_ID); + TenantInfo tenant2 = new TenantInfo(); + tenant2.setName(OTHER_TENANT); + tenant2.setDataPartitionId(OTHER_TENANT); + Collection tenants = Lists.newArrayList(tenant1, tenant2); + when(this.tenantFactory.listTenantInfo()).thenReturn(tenants); + + //The schema-notification is turned on + when(this.eventGridConfig.isEventGridEnabled()).thenReturn(true); + when(this.eventGridConfig.getCustomTopicName()).thenReturn("dummy-topic"); + doNothing().when(this.eventGridTopicStore).publishToEventGridTopic(anyString(), anyString(), anyList());; + ArgumentCaptor> captorList = ArgumentCaptor.forClass(ArrayList.class); + + + SchemaPubSubInfo[] schemaPubSubMsgs = new SchemaPubSubInfo [1]; + schemaPubSubMsgs[0]=new SchemaPubSubInfo("dummy","schema_create"); + + HashMap data = new HashMap<>(); + data.put("data", schemaPubSubMsgs); + data.put(DpsHeaders.ACCOUNT_ID, PARTITION_ID); + data.put(DpsHeaders.DATA_PARTITION_ID, PARTITION_ID); + data.put(DpsHeaders.CORRELATION_ID, CORRELATION_ID); + + //Call publish Message + messageBusImpl.publishMessageForSystemSchema("dummy", "schema_create"); + + //Assert that eventGridTopicStore is called once + verify(this.eventGridTopicStore, times(2)).publishToEventGridTopic(anyString(), anyString(), captorList.capture()); + ArrayList eventGridList = captorList.getValue(); + assertNotNull(eventGridList); + assertThat(eventGridList.size(), is(equalTo(1))); + + HashMap outputData = (HashMap)eventGridList.get(0).data(); + assertEquals(((SchemaPubSubInfo[])outputData.get("data"))[0].getKind(), "dummy"); + assertEquals(((SchemaPubSubInfo[])outputData.get("data"))[0].getOp(), "schema_create"); + + } + } -- GitLab From 791bf4e647f3b826bb3d400dff183590e69c80b0 Mon Sep 17 00:00:00 2001 From: Aman Verma Date: Wed, 11 Aug 2021 16:04:14 +0530 Subject: [PATCH 08/24] adding opening comments --- .../azure/impl/messagebus/MessageBusImpl.java | 5 +++++ .../schemainfostore/AzureAuthorityStore.java | 14 +++++++++++++ .../schemainfostore/AzureEntityTypeStore.java | 14 +++++++++++++ .../schemainfostore/AzureSourceStore.java | 14 +++++++++++++ .../impl/schemastore/AzureSchemaStore.java | 20 +++++++++++++++++++ 5 files changed, 67 insertions(+) diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/messagebus/MessageBusImpl.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/messagebus/MessageBusImpl.java index 413b33df..2de25002 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/messagebus/MessageBusImpl.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/messagebus/MessageBusImpl.java @@ -76,6 +76,11 @@ public class MessageBusImpl implements IMessageBus { } } + /** + * Method to publish schema create notification for system schemas. + * @param schemaId + * @param eventType + */ @Override public void publishMessageForSystemSchema(String schemaId, String eventType) { if (eventGridConfig.isEventGridEnabled()) { diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureAuthorityStore.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureAuthorityStore.java index d3332c20..dd60f70f 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureAuthorityStore.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureAuthorityStore.java @@ -86,6 +86,13 @@ public class AzureAuthorityStore implements IAuthorityStore { return authorityDoc.getAuthority(); } + /** + * Method to get system Authority + * @param authorityId + * @return + * @throws NotFoundException + * @throws ApplicationException + */ @Override public Authority getSystemAuthority(String authorityId) throws NotFoundException, ApplicationException { AuthorityDoc authorityDoc; @@ -121,6 +128,13 @@ public class AzureAuthorityStore implements IAuthorityStore { return authority; } + /** + * Method to register a System Authority + * @param authority + * @return + * @throws ApplicationException + * @throws BadRequestException + */ @Override public Authority createSystemAuthority(Authority authority) throws ApplicationException, BadRequestException { try { diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureEntityTypeStore.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureEntityTypeStore.java index 71c55d76..c22c2e25 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureEntityTypeStore.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureEntityTypeStore.java @@ -83,6 +83,13 @@ public class AzureEntityTypeStore implements IEntityTypeStore { return entityTypeDoc.getEntityType(); } + /** + * Method to get system Entity + * @param entityTypeId + * @return + * @throws NotFoundException + * @throws ApplicationException + */ @Override public EntityType getSystemEntity(String entityTypeId) throws NotFoundException, ApplicationException { EntityTypeDoc entityTypeDoc; @@ -118,6 +125,13 @@ public class AzureEntityTypeStore implements IEntityTypeStore { return entityType; } + /** + * Method to create a system Entity + * @param entityType + * @return + * @throws BadRequestException + * @throws ApplicationException + */ @Override public EntityType createSystemEntity(EntityType entityType) throws BadRequestException, ApplicationException { try { diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSourceStore.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSourceStore.java index bc72c99d..ff641515 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSourceStore.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSourceStore.java @@ -81,6 +81,13 @@ public class AzureSourceStore implements ISourceStore { return sourceDoc.getSource(); } + /** + * Method to get system Source + * @param sourceId + * @return + * @throws NotFoundException + * @throws ApplicationException + */ @Override public Source getSystemSource(String sourceId) throws NotFoundException, ApplicationException { SourceDoc sourceDoc; @@ -117,6 +124,13 @@ public class AzureSourceStore implements ISourceStore { return source; } + /** + * Method tp register a system Source + * @param source + * @return + * @throws BadRequestException + * @throws ApplicationException + */ @Override public Source createSystemSource(Source source) throws BadRequestException, ApplicationException { try { diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemastore/AzureSchemaStore.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemastore/AzureSchemaStore.java index 10bdfc68..c1fe3e0f 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemastore/AzureSchemaStore.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemastore/AzureSchemaStore.java @@ -80,6 +80,13 @@ public class AzureSchemaStore implements ISchemaStore { } } + /** + * Method to get a system schema + * @param filePath + * @return + * @throws NotFoundException + * @throws ApplicationException + */ @Override public String getSystemSchema(String filePath) throws NotFoundException, ApplicationException { filePath = filePath + SchemaConstants.JSON_EXTENSION; @@ -123,6 +130,13 @@ public class AzureSchemaStore implements ISchemaStore { } } + /** + * Method to create a system schema + * @param filePath + * @param content + * @return + * @throws ApplicationException + */ @Override public String createSystemSchema(String filePath, String content) throws ApplicationException { filePath = filePath + SchemaConstants.JSON_EXTENSION; @@ -160,6 +174,12 @@ public class AzureSchemaStore implements ISchemaStore { } } + /** + * Method to delete a system schema. + * @param schemaId + * @return + * @throws ApplicationException + */ @Override public boolean cleanSystemSchemaProject(String schemaId) throws ApplicationException { String filePath = schemaId + SchemaConstants.JSON_EXTENSION; -- GitLab From 0d4dc4875bbd7ba9bb8e271cdea425145f0664ad Mon Sep 17 00:00:00 2001 From: Aman Verma Date: Wed, 11 Aug 2021 16:38:16 +0530 Subject: [PATCH 09/24] reordering code --- .../schemainfostore/AzureAuthorityStore.java | 23 ++++++----- .../schemainfostore/AzureEntityTypeStore.java | 23 +++++------ .../schemainfostore/AzureSchemaInfoStore.java | 13 ++++++ .../schemainfostore/AzureSourceStore.java | 22 +++++----- .../impl/schemastore/AzureSchemaStore.java | 41 ++++++++++--------- 5 files changed, 68 insertions(+), 54 deletions(-) diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureAuthorityStore.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureAuthorityStore.java index dd60f70f..fc5a2ac8 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureAuthorityStore.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureAuthorityStore.java @@ -73,16 +73,16 @@ public class AzureAuthorityStore implements IAuthorityStore { @Override public Authority get(String authorityId) throws NotFoundException, ApplicationException { - String id = headers.getPartitionId() + ":" + authorityId; - AuthorityDoc authorityDoc; - if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { return this.getSystemAuthority(authorityId); - } else { - authorityDoc = cosmosStore.findItem(headers.getPartitionId(), cosmosDBName, authorityContainer, id, authorityId, AuthorityDoc.class) - .orElseThrow(() -> new NotFoundException("bad input parameter")); } + String id = headers.getPartitionId() + ":" + authorityId; + AuthorityDoc authorityDoc; + + authorityDoc = cosmosStore.findItem(headers.getPartitionId(), cosmosDBName, authorityContainer, id, authorityId, AuthorityDoc.class) + .orElseThrow(() -> new NotFoundException("bad input parameter")); + return authorityDoc.getAuthority(); } @@ -111,15 +111,16 @@ public class AzureAuthorityStore implements IAuthorityStore { */ @Override public Authority create(Authority authority) throws ApplicationException, BadRequestException { + + if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { + return this.createSystemAuthority(authority); + } + String id = headers.getPartitionId() + ":" + authority.getAuthorityId(); try { AuthorityDoc authorityDoc = new AuthorityDoc(id, authority); - if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { - return this.createSystemAuthority(authority); - } else { - cosmosStore.createItem(headers.getPartitionId(), cosmosDBName, authorityContainer, id, authorityDoc); - } + cosmosStore.createItem(headers.getPartitionId(), cosmosDBName, authorityContainer, id, authorityDoc); } catch (AppException ex) { handleAppException(ex, authority); } diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureEntityTypeStore.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureEntityTypeStore.java index c22c2e25..d3acacec 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureEntityTypeStore.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureEntityTypeStore.java @@ -70,16 +70,15 @@ public class AzureEntityTypeStore implements IEntityTypeStore { @Override public EntityType get(String entityTypeId) throws NotFoundException, ApplicationException { - String id = headers.getPartitionId() + ":" + entityTypeId; - EntityTypeDoc entityTypeDoc; - if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { return this.getSystemEntity(entityTypeId); - } else { - entityTypeDoc = cosmosStore.findItem(headers.getPartitionId(), cosmosDBName, entityTypeContainer, id, entityTypeId, EntityTypeDoc.class) - .orElseThrow(() -> new NotFoundException("bad input parameter")); } + String id = headers.getPartitionId() + ":" + entityTypeId; + EntityTypeDoc entityTypeDoc; + entityTypeDoc = cosmosStore.findItem(headers.getPartitionId(), cosmosDBName, entityTypeContainer, id, entityTypeId, EntityTypeDoc.class) + .orElseThrow(() -> new NotFoundException("bad input parameter")); + return entityTypeDoc.getEntityType(); } @@ -108,15 +107,15 @@ public class AzureEntityTypeStore implements IEntityTypeStore { */ @Override public EntityType create(EntityType entityType) throws BadRequestException, ApplicationException { - String id = headers.getPartitionId() + ":" + entityType.getEntityTypeId(); + if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { + this.createSystemEntity(entityType); + } + + String id = headers.getPartitionId() + ":" + entityType.getEntityTypeId(); try { EntityTypeDoc entityTypeDoc = new EntityTypeDoc(id, entityType); - if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { - this.createSystemEntity(entityType); - } else { - cosmosStore.createItem(headers.getPartitionId(), cosmosDBName, entityTypeContainer, id, entityTypeDoc); - } + cosmosStore.createItem(headers.getPartitionId(), cosmosDBName, entityTypeContainer, id, entityTypeDoc); } catch (AppException ex) { handleAppException(ex, entityType); } diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java index ecb9611d..a9e9d6e1 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java @@ -98,6 +98,10 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { @Override public SchemaInfo getSchemaInfo(String schemaId) throws ApplicationException, NotFoundException { +// if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { +// return this.getSystemSchemaInfo(schemaId); +// } + String id = headers.getPartitionId() + ":" + schemaId; SchemaIdentity schemaIdentity = schemaKindToSchemaIdentity(schemaId); @@ -109,6 +113,15 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { return getSchemaInfoObject(schemaInfoDoc.getFlattenedSchemaInfo(), headers.getPartitionId()); } + @Override + public SchemaInfo getSystemSchemaInfo(String schemaId) throws ApplicationException, NotFoundException { + SchemaIdentity schemaIdentity = schemaKindToSchemaIdentity(schemaId); + String partitioningKey = createSchemaInfoPartitionKey(schemaIdentity); + SchemaInfoDoc schemaInfoDoc = cosmosStore.findItem(systemResourceConfig.getCosmosDatabase(), schemaInfoContainer, schemaId, partitioningKey, SchemaInfoDoc.class) + .orElseThrow(() -> new NotFoundException(SchemaConstants.SCHEMA_NOT_PRESENT)); + return getSchemaInfoObject(schemaInfoDoc.getFlattenedSchemaInfo(), headers.getPartitionId()); + } + /** * Method to Create schema in azure store * diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSourceStore.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSourceStore.java index ff641515..1dd04efc 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSourceStore.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSourceStore.java @@ -68,16 +68,15 @@ public class AzureSourceStore implements ISourceStore { @Override public Source get(String sourceId) throws NotFoundException, ApplicationException { - String id = headers.getPartitionId().toString() + ":" + sourceId; - SourceDoc sourceDoc; - if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { return this.getSystemSource(sourceId); - } else { - sourceDoc = cosmosStore.findItem(headers.getPartitionId(), cosmosDBName, sourceContainer, id, sourceId, SourceDoc.class) - .orElseThrow(() -> new NotFoundException("bad input parameter")); } + String id = headers.getPartitionId().toString() + ":" + sourceId; + SourceDoc sourceDoc; + sourceDoc = cosmosStore.findItem(headers.getPartitionId(), cosmosDBName, sourceContainer, id, sourceId, SourceDoc.class) + .orElseThrow(() -> new NotFoundException("bad input parameter")); + return sourceDoc.getSource(); } @@ -107,15 +106,16 @@ public class AzureSourceStore implements ISourceStore { */ @Override public Source create(Source source) throws BadRequestException, ApplicationException { + + if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { + return this.createSystemSource(source); + } + String id = headers.getPartitionId() + ":" + source.getSourceId(); try { SourceDoc sourceDoc = new SourceDoc(id, source); - if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { - return this.createSystemSource(source); - } else { - cosmosStore.createItem(headers.getPartitionId(), cosmosDBName, sourceContainer, id, sourceDoc); - } + cosmosStore.createItem(headers.getPartitionId(), cosmosDBName, sourceContainer, id, sourceDoc); } catch (AppException ex) { handleAppException(ex, source); } diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemastore/AzureSchemaStore.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemastore/AzureSchemaStore.java index c1fe3e0f..03cc1fb6 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemastore/AzureSchemaStore.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemastore/AzureSchemaStore.java @@ -62,14 +62,16 @@ public class AzureSchemaStore implements ISchemaStore { */ @Override public String getSchema(String dataPartitionId, String filePath) throws ApplicationException, NotFoundException { + + if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(dataPartitionId)) { + return this.getSystemSchema(filePath); + } + + filePath = dataPartitionId + ":" + filePath + SchemaConstants.JSON_EXTENSION; try { String content = null; - if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(dataPartitionId)) { - return this.getSystemSchema(filePath); - } else { - filePath = dataPartitionId + ":" + filePath + SchemaConstants.JSON_EXTENSION; - content = blobStore.readFromStorageContainer(dataPartitionId, filePath, config.containerName()); - } + content = blobStore.readFromStorageContainer(dataPartitionId, filePath, config.containerName()); + if (content != null) return content; else @@ -114,15 +116,14 @@ public class AzureSchemaStore implements ISchemaStore { @Override public String createSchema(String filePath, String content) throws ApplicationException { - String dataPartitionId = headers.getPartitionId(); + if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { + return this.createSystemSchema(filePath, content); + } + String dataPartitionId = headers.getPartitionId(); + filePath = dataPartitionId + ":" + filePath + SchemaConstants.JSON_EXTENSION; try { - if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { - return this.createSystemSchema(filePath, content); - } else { - filePath = dataPartitionId + ":" + filePath + SchemaConstants.JSON_EXTENSION; - blobStore.writeToStorageContainer(dataPartitionId, filePath, content, config.containerName()); - } + blobStore.writeToStorageContainer(dataPartitionId, filePath, content, config.containerName()); log.info(SchemaConstants.SCHEMA_CREATED); return filePath; } catch (Exception ex) { @@ -157,16 +158,16 @@ public class AzureSchemaStore implements ISchemaStore { */ @Override public boolean cleanSchemaProject(String schemaId) throws ApplicationException { - String dataPartitionId = headers.getPartitionId(); + if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { + return this.cleanSystemSchemaProject(schemaId); + } + + String dataPartitionId = headers.getPartitionId(); + String filePath = dataPartitionId + ":" + schemaId + SchemaConstants.JSON_EXTENSION; try { - if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { - return this.cleanSystemSchemaProject(schemaId); - } else { - String filePath = dataPartitionId + ":" + schemaId + SchemaConstants.JSON_EXTENSION; - return blobStore.deleteFromStorageContainer(dataPartitionId, filePath, config.containerName()); - } + return blobStore.deleteFromStorageContainer(dataPartitionId, filePath, config.containerName()); } catch (Exception e) { -- GitLab From e78ff04506cab4d1791c1b1fd35be6df637ab9ef Mon Sep 17 00:00:00 2001 From: Aman Verma Date: Wed, 11 Aug 2021 16:42:08 +0530 Subject: [PATCH 10/24] adding block comments --- .../azure/impl/schemainfostore/AzureAuthorityStore.java | 4 ++-- .../azure/impl/schemainfostore/AzureEntityTypeStore.java | 4 ++-- .../schema/azure/impl/schemainfostore/AzureSourceStore.java | 4 ++-- .../osdu/schema/azure/impl/schemastore/AzureSchemaStore.java | 5 +++-- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureAuthorityStore.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureAuthorityStore.java index fc5a2ac8..ad5f2632 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureAuthorityStore.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureAuthorityStore.java @@ -72,7 +72,7 @@ public class AzureAuthorityStore implements IAuthorityStore { */ @Override public Authority get(String authorityId) throws NotFoundException, ApplicationException { - + // This if block will be removed once schema-core starts consuming *System* methods. if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { return this.getSystemAuthority(authorityId); } @@ -111,7 +111,7 @@ public class AzureAuthorityStore implements IAuthorityStore { */ @Override public Authority create(Authority authority) throws ApplicationException, BadRequestException { - + // This if block will be removed once schema-core starts consuming *System* methods. if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { return this.createSystemAuthority(authority); } diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureEntityTypeStore.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureEntityTypeStore.java index d3acacec..e4590b65 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureEntityTypeStore.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureEntityTypeStore.java @@ -69,7 +69,7 @@ public class AzureEntityTypeStore implements IEntityTypeStore { */ @Override public EntityType get(String entityTypeId) throws NotFoundException, ApplicationException { - + // This if block will be removed once schema-core starts consuming *System* methods. if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { return this.getSystemEntity(entityTypeId); } @@ -107,7 +107,7 @@ public class AzureEntityTypeStore implements IEntityTypeStore { */ @Override public EntityType create(EntityType entityType) throws BadRequestException, ApplicationException { - + // This if block will be removed once schema-core starts consuming *System* methods. if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { this.createSystemEntity(entityType); } diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSourceStore.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSourceStore.java index 1dd04efc..5009d9f1 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSourceStore.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSourceStore.java @@ -67,7 +67,7 @@ public class AzureSourceStore implements ISourceStore { */ @Override public Source get(String sourceId) throws NotFoundException, ApplicationException { - + // This if block will be removed once schema-core starts consuming *System* methods. if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { return this.getSystemSource(sourceId); } @@ -106,7 +106,7 @@ public class AzureSourceStore implements ISourceStore { */ @Override public Source create(Source source) throws BadRequestException, ApplicationException { - + // This if block will be removed once schema-core starts consuming *System* methods. if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { return this.createSystemSource(source); } diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemastore/AzureSchemaStore.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemastore/AzureSchemaStore.java index 03cc1fb6..b6c16d69 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemastore/AzureSchemaStore.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemastore/AzureSchemaStore.java @@ -62,7 +62,7 @@ public class AzureSchemaStore implements ISchemaStore { */ @Override public String getSchema(String dataPartitionId, String filePath) throws ApplicationException, NotFoundException { - + // This if block will be removed once schema-core starts consuming *System* methods. if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(dataPartitionId)) { return this.getSystemSchema(filePath); } @@ -116,6 +116,7 @@ public class AzureSchemaStore implements ISchemaStore { @Override public String createSchema(String filePath, String content) throws ApplicationException { + // This if block will be removed once schema-core starts consuming *System* methods. if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { return this.createSystemSchema(filePath, content); } @@ -158,7 +159,7 @@ public class AzureSchemaStore implements ISchemaStore { */ @Override public boolean cleanSchemaProject(String schemaId) throws ApplicationException { - + // This if block will be removed once schema-core starts consuming *System* methods. if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { return this.cleanSystemSchemaProject(schemaId); } -- GitLab From bb0518b81c1b701f94b916e0549382ff76cc660a Mon Sep 17 00:00:00 2001 From: Aman Verma Date: Wed, 11 Aug 2021 18:04:11 +0530 Subject: [PATCH 11/24] adding new methods for system schemas --- .../schemainfostore/AzureSchemaInfoStore.java | 83 +++++++++++++++---- .../AzureSchemaInfoStoreTest.java | 18 +++- 2 files changed, 79 insertions(+), 22 deletions(-) diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java index a9e9d6e1..7e9c9892 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java @@ -97,10 +97,10 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { */ @Override public SchemaInfo getSchemaInfo(String schemaId) throws ApplicationException, NotFoundException { - -// if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { -// return this.getSystemSchemaInfo(schemaId); -// } + // This if block will be removed once schema-core starts consuming *System* methods. + if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { + return this.getSystemSchemaInfo(schemaId); + } String id = headers.getPartitionId() + ":" + schemaId; @@ -119,7 +119,7 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { String partitioningKey = createSchemaInfoPartitionKey(schemaIdentity); SchemaInfoDoc schemaInfoDoc = cosmosStore.findItem(systemResourceConfig.getCosmosDatabase(), schemaInfoContainer, schemaId, partitioningKey, SchemaInfoDoc.class) .orElseThrow(() -> new NotFoundException(SchemaConstants.SCHEMA_NOT_PRESENT)); - return getSchemaInfoObject(schemaInfoDoc.getFlattenedSchemaInfo(), headers.getPartitionId()); + return getSchemaInfoObject(schemaInfoDoc.getFlattenedSchemaInfo(), null); } /** @@ -132,12 +132,40 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { */ @Override public SchemaInfo createSchemaInfo(SchemaRequest schema) throws ApplicationException, BadRequestException { + // This if block will be removed once schema-core starts consuming *System* methods. + if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { + return this.createSystemSchemaInfo(schema); + } + String id = headers.getPartitionId() + ":" + schema.getSchemaInfo().getSchemaIdentity().getId(); - FlattenedSchemaInfo flattenedSchemaInfo = populateSchemaInfo(schema); + FlattenedSchemaInfo flattenedSchemaInfo = populateSchemaInfo(schema, headers.getPartitionId()); String partitionKey = createSchemaInfoPartitionKey(schema.getSchemaInfo().getSchemaIdentity()); SchemaInfoDoc schemaInfoDoc = new SchemaInfoDoc(id, partitionKey, flattenedSchemaInfo); try { - crateItemInCosmos(headers.getPartitionId(), cosmosDBName, schemaInfoContainer, partitionKey, schemaInfoDoc); + cosmosStore.createItem(headers.getPartitionId(), cosmosDBName, schemaInfoContainer, partitionKey, schemaInfoDoc); + } catch (AppException ex) { + if (ex.getError().getCode() == 409) { + log.warning(SchemaConstants.SCHEMA_ID_EXISTS); + throw new BadRequestException(SchemaConstants.SCHEMA_ID_EXISTS); + } else { + log.error(MessageFormat.format(SchemaConstants.OBJECT_INVALID, ex.getMessage())); + throw new ApplicationException(SchemaConstants.SCHEMA_CREATION_FAILED_INVALID_OBJECT); + } + } + + log.info(SchemaConstants.SCHEMA_INFO_CREATED); + return getSchemaInfoObject(flattenedSchemaInfo, headers.getPartitionId()); + } + + @Override + public SchemaInfo createSystemSchemaInfo(SchemaRequest schema) throws ApplicationException, BadRequestException { + String id = schema.getSchemaInfo().getSchemaIdentity().getId(); + FlattenedSchemaInfo flattenedSchemaInfo = populateSchemaInfo(schema, null); + String partitionKey = createSchemaInfoPartitionKey(schema.getSchemaInfo().getSchemaIdentity()); + SchemaInfoDoc schemaInfoDoc = new SchemaInfoDoc(id, partitionKey, flattenedSchemaInfo); + + try { + cosmosStore.createItem(systemResourceConfig.getCosmosDatabase(), schemaInfoContainer, partitionKey, schemaInfoDoc); } catch (AppException ex) { if (ex.getError().getCode() == 409) { log.warning(SchemaConstants.SCHEMA_ID_EXISTS); @@ -163,7 +191,7 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { @Override public SchemaInfo updateSchemaInfo(SchemaRequest schema) throws ApplicationException, BadRequestException { String id = headers.getPartitionId() + ":" + schema.getSchemaInfo().getSchemaIdentity().getId(); - FlattenedSchemaInfo flattenedSchemaInfo = populateSchemaInfo(schema); + FlattenedSchemaInfo flattenedSchemaInfo = populateSchemaInfo(schema, headers.getPartitionId()); String partitionKey = createSchemaInfoPartitionKey(schema.getSchemaInfo().getSchemaIdentity()); SchemaInfoDoc schemaInfoDoc = new SchemaInfoDoc(id, partitionKey, flattenedSchemaInfo); try { @@ -244,16 +272,21 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { /** * Creates schemaInfo object and populates required properties. * - * @param schema + * @param schemaRequest + * @param dataPartitionId * @return */ - private FlattenedSchemaInfo populateSchemaInfo(SchemaRequest schemaRequest) + private FlattenedSchemaInfo populateSchemaInfo(SchemaRequest schemaRequest, String dataPartitionId) throws BadRequestException { SchemaInfo schemaInfo = schemaRequest.getSchemaInfo(); // check for super-seeding schemas String supersededById = ""; if (schemaInfo.getSupersededBy() != null) { - String id = headers.getPartitionId() + ":" + schemaInfo.getSupersededBy().getId(); + String id = schemaInfo.getSupersededBy().getId(); + + if (dataPartitionId != null && !dataPartitionId.isEmpty()) { + id = dataPartitionId + ":" + id; + } if (schemaInfo.getSupersededBy().getId() == null) throw new BadRequestException(SchemaConstants.INVALID_SUPERSEDEDBY_ID); @@ -261,7 +294,7 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { SchemaIdentity schemaIdentity = schemaKindToSchemaIdentity(schemaInfo.getSupersededBy().getId()); String partitionKey = createSchemaInfoPartitionKey(schemaIdentity); - if ( !findItemInCosmosStore(headers.getPartitionId(), cosmosDBName, schemaInfoContainer, id, partitionKey, FlattenedSchemaInfo.class).isPresent()) { + if ( !findItemInCosmosStore(dataPartitionId, cosmosDBName, schemaInfoContainer, id, partitionKey, FlattenedSchemaInfo.class).isPresent()) { log.error(SchemaConstants.INVALID_SUPERSEDEDBY_ID); throw new BadRequestException(SchemaConstants.INVALID_SUPERSEDEDBY_ID); } @@ -288,7 +321,11 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { private SchemaInfo getSchemaInfoObject(FlattenedSchemaInfo flattenedSchemaInfo, String dataPartitionId) { SchemaIdentity superSededBy = null; if (!flattenedSchemaInfo.getSupersededBy().isEmpty()) { - String id = dataPartitionId + ":" + flattenedSchemaInfo.getSupersededBy(); + String id = flattenedSchemaInfo.getSupersededBy(); + if ( dataPartitionId != null && !dataPartitionId.isEmpty()) { + id = dataPartitionId + ":" + id; + } + SchemaIdentity schemaIdentity = schemaKindToSchemaIdentity(flattenedSchemaInfo.getSupersededBy()); String partitionKey = createSchemaInfoPartitionKey(schemaIdentity); SchemaInfoDoc doc = findItemInCosmosStore(dataPartitionId, cosmosDBName, schemaInfoContainer, id, partitionKey, SchemaInfoDoc.class).get(); @@ -493,7 +530,9 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { String id, String partitionKey, Class clazz) { - if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(dataPartitionId)) { + if (dataPartitionId == null || + dataPartitionId.isEmpty() || + systemResourceConfig.getSharedTenant().equalsIgnoreCase(dataPartitionId)) { return cosmosStore.findItem(systemResourceConfig.getCosmosDatabase(), containerName, id, partitionKey, clazz); } else { return cosmosStore.findItem(dataPartitionId, dataBaseName, containerName, id, partitionKey, clazz); @@ -506,7 +545,9 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { String containerName, String partitionKey, T item) { - if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(dataPartitionId)) { + if (dataPartitionId == null || + dataPartitionId.isEmpty() || + systemResourceConfig.getSharedTenant().equalsIgnoreCase(dataPartitionId)) { cosmosStore.createItem(systemResourceConfig.getCosmosDatabase(), containerName, partitionKey, item); } else { cosmosStore.createItem(dataPartitionId, dataBaseName, containerName, partitionKey, item); @@ -519,7 +560,9 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { String containerName, String partitionKey, T item) { - if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(dataPartitionId)) { + if (dataPartitionId == null || + dataPartitionId.isEmpty() || + systemResourceConfig.getSharedTenant().equalsIgnoreCase(dataPartitionId)) { cosmosStore.upsertItem(systemResourceConfig.getCosmosDatabase(), containerName, partitionKey, item); } else { cosmosStore.upsertItem(dataPartitionId, dataBaseName, containerName, partitionKey, item); @@ -532,7 +575,9 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { String containerName, String id, String partitionKey) { - if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(dataPartitionId)) { + if (dataPartitionId == null || + dataPartitionId.isEmpty() || + systemResourceConfig.getSharedTenant().equalsIgnoreCase(dataPartitionId)) { cosmosStore.deleteItem(systemResourceConfig.getCosmosDatabase(), containerName, id, partitionKey); } else { cosmosStore.deleteItem(dataPartitionId, dataBaseName, containerName, id, partitionKey); @@ -546,7 +591,9 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { SqlQuerySpec query, CosmosQueryRequestOptions options, Class clazz) { - if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(dataPartitionId)) { + if (dataPartitionId == null || + dataPartitionId.isEmpty() || + systemResourceConfig.getSharedTenant().equalsIgnoreCase(dataPartitionId)) { return cosmosStore.queryItems(systemResourceConfig.getCosmosDatabase(), collection, query, options, clazz); } else { return cosmosStore.queryItems(dataPartitionId, cosmosDBName, collection, query, options, clazz); diff --git a/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSchemaInfoStoreTest.java b/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSchemaInfoStoreTest.java index 080fd22d..be1aca3b 100644 --- a/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSchemaInfoStoreTest.java +++ b/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSchemaInfoStoreTest.java @@ -171,14 +171,20 @@ public class AzureSchemaInfoStoreTest { .findItem( eq(systemCosmosDBName), any(), - eq(commonTenantId + ":" + schemaId), + eq(schemaId), eq(partitionKey), any()); doReturn(getFlattenedSchemaInfo()).when(schemaInfoDoc).getFlattenedSchemaInfo(); + + SchemaInfo schemaInfo1 = schemaInfoStore.getSystemSchemaInfo(schemaId); + assertNotNull(schemaInfo1); + + // This is temporary and will be removed once schema-core starts consuming *system* methods SchemaInfo schemaInfo = schemaInfoStore.getSchemaInfo(schemaId); assertNotNull(schemaInfo); - verify(this.cosmosStore, times(1)).findItem(any(), any(), eq("common:os:wks:well:1.1.1"), eq("os:wks:well:1"), eq(SchemaInfoDoc.class)); + + verify(this.cosmosStore, times(2)).findItem(any(), any(), eq("os:wks:well:1.1.1"), eq("os:wks:well:1"), eq(SchemaInfoDoc.class)); verify(this.cosmosStore, times(0)).findItem(anyString(), anyString(), anyString(), anyString(), anyString(), any()); } @@ -226,13 +232,17 @@ public class AzureSchemaInfoStoreTest { .findItem( eq(systemCosmosDBName), any(), - eq(commonTenantId + ":" + schemaId), + eq(schemaId), anyString(), any()); doReturn(getFlattenedSchemaInfo()).when(schemaInfoDoc).getFlattenedSchemaInfo(); + assertNotNull(schemaInfoStore.createSystemSchemaInfo(getMockSchemaObject_Published())); + + // This is temporary and will be removed once schema-core starts consuming *system* methods assertNotNull(schemaInfoStore.createSchemaInfo(getMockSchemaObject_Published())); - verify(this.cosmosStore, times(1)).createItem(any(), any(), eq("os:wks:well:1"), any()); + + verify(this.cosmosStore, times(2)).createItem(any(), any(), eq("os:wks:well:1"), any()); verify(this.cosmosStore, times(0)).createItem(anyString(), anyString(), anyString(), anyString(), any()); } -- GitLab From a47bdcfa1f9b13b07235d82df87fb5f4d6aae1d8 Mon Sep 17 00:00:00 2001 From: Aman Verma Date: Wed, 11 Aug 2021 18:34:39 +0530 Subject: [PATCH 12/24] implementing another question --- .../schemainfostore/AzureSchemaInfoStore.java | 39 ++++++++++++++++++- .../AzureSchemaInfoStoreTest.java | 15 ++++++- 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java index 7e9c9892..a403ed94 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java @@ -113,6 +113,13 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { return getSchemaInfoObject(schemaInfoDoc.getFlattenedSchemaInfo(), headers.getPartitionId()); } + /** + * Method to get system schema info. + * @param schemaId + * @return + * @throws ApplicationException + * @throws NotFoundException + */ @Override public SchemaInfo getSystemSchemaInfo(String schemaId) throws ApplicationException, NotFoundException { SchemaIdentity schemaIdentity = schemaKindToSchemaIdentity(schemaId); @@ -157,6 +164,13 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { return getSchemaInfoObject(flattenedSchemaInfo, headers.getPartitionId()); } + /** + * Method to create system schema info. + * @param schema + * @return + * @throws ApplicationException + * @throws BadRequestException + */ @Override public SchemaInfo createSystemSchemaInfo(SchemaRequest schema) throws ApplicationException, BadRequestException { String id = schema.getSchemaInfo().getSchemaIdentity().getId(); @@ -190,12 +204,35 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { */ @Override public SchemaInfo updateSchemaInfo(SchemaRequest schema) throws ApplicationException, BadRequestException { + // This if block will be removed once schema-core starts consuming *System* methods. + if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { + return this.createSystemSchemaInfo(schema); + } + String id = headers.getPartitionId() + ":" + schema.getSchemaInfo().getSchemaIdentity().getId(); FlattenedSchemaInfo flattenedSchemaInfo = populateSchemaInfo(schema, headers.getPartitionId()); String partitionKey = createSchemaInfoPartitionKey(schema.getSchemaInfo().getSchemaIdentity()); SchemaInfoDoc schemaInfoDoc = new SchemaInfoDoc(id, partitionKey, flattenedSchemaInfo); try { - upsertItemInCosmos(headers.getPartitionId(), cosmosDBName, schemaInfoContainer, partitionKey,schemaInfoDoc); + cosmosStore.upsertItem(headers.getPartitionId(), cosmosDBName, schemaInfoContainer, partitionKey,schemaInfoDoc); + } catch (Exception ex) { + log.error(MessageFormat.format(SchemaConstants.OBJECT_INVALID, ex.getMessage())); + throw new ApplicationException(SchemaConstants.SCHEMA_CREATION_FAILED_INVALID_OBJECT); + } + + log.info(SchemaConstants.SCHEMA_INFO_UPDATED); + return getSchemaInfoObject(flattenedSchemaInfo, headers.getPartitionId()); + } + + @Override + public SchemaInfo updateSystemSchemaInfo(SchemaRequest schema) throws ApplicationException, BadRequestException { + String id = schema.getSchemaInfo().getSchemaIdentity().getId(); + FlattenedSchemaInfo flattenedSchemaInfo = populateSchemaInfo(schema, headers.getPartitionId()); + String partitionKey = createSchemaInfoPartitionKey(schema.getSchemaInfo().getSchemaIdentity()); + SchemaInfoDoc schemaInfoDoc = new SchemaInfoDoc(id, partitionKey, flattenedSchemaInfo); + + try { + cosmosStore.upsertItem(systemResourceConfig.getCosmosDatabase(), schemaInfoContainer, partitionKey,schemaInfoDoc); } catch (Exception ex) { log.error(MessageFormat.format(SchemaConstants.OBJECT_INVALID, ex.getMessage())); throw new ApplicationException(SchemaConstants.SCHEMA_CREATION_FAILED_INVALID_OBJECT); diff --git a/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSchemaInfoStoreTest.java b/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSchemaInfoStoreTest.java index be1aca3b..18fef1b6 100644 --- a/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSchemaInfoStoreTest.java +++ b/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSchemaInfoStoreTest.java @@ -431,12 +431,23 @@ public class AzureSchemaInfoStoreTest { .findItem( eq(systemCosmosDBName), any(), - eq(commonTenantId + ":" + supersedingSchemaId), + eq(supersedingSchemaId), eq(partitionKey), any()); + doNothing().when(cosmosStore) + .upsertItem( + eq(systemCosmosDBName), + any(), + any(), + any()); doReturn(getFlattenedSchemaInfo()).when(schemaInfoDoc).getFlattenedSchemaInfo(); - assertNotNull(schemaInfoStore.updateSchemaInfo(getMockSchemaObject_Published())); + + assertNotNull(schemaInfoStore.updateSystemSchemaInfo(getMockSchemaObject_Published())); + + // This is temporary and will be removed once schema-core starts consuming *system* methods + //assertNotNull(schemaInfoStore.updateSchemaInfo(getMockSchemaObject_Published())); + verify(this.cosmosStore, times(1)).upsertItem(any(), any(), any(), any()); verify(this.cosmosStore, times(0)).upsertItem(any(), any(), any(), any(), any()); } -- GitLab From 18c9d0d17af27680e2cbfc59290deb21eb4a2947 Mon Sep 17 00:00:00 2001 From: Aman Verma Date: Thu, 12 Aug 2021 11:36:28 +0530 Subject: [PATCH 13/24] implementing another method --- .../schemainfostore/AzureSchemaInfoStore.java | 30 +++++++++++++++++-- .../AzureSchemaInfoStoreTest.java | 7 ++--- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java index a403ed94..54c2835e 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java @@ -224,6 +224,13 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { return getSchemaInfoObject(flattenedSchemaInfo, headers.getPartitionId()); } + /** + * method to update system schema info. + * @param schema + * @return + * @throws ApplicationException + * @throws BadRequestException + */ @Override public SchemaInfo updateSystemSchemaInfo(SchemaRequest schema) throws ApplicationException, BadRequestException { String id = schema.getSchemaInfo().getSchemaIdentity().getId(); @@ -251,22 +258,41 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { */ @Override public boolean cleanSchema(String schemaId) throws ApplicationException { + // This if block will be removed once schema-core starts consuming *System* methods. + if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { + return this.cleanSystemSchema(schemaId); + } + String id = headers.getPartitionId() + ":" + schemaId; SchemaIdentity schemaIdentity = schemaKindToSchemaIdentity(schemaId); String partitionKey = createSchemaInfoPartitionKey(schemaIdentity); // Check whether SchemaInfo already exists - Boolean exists = findItemInCosmosStore(headers.getPartitionId(), cosmosDBName, schemaInfoContainer, id, partitionKey, SchemaInfoDoc.class).isPresent(); + Boolean exists = cosmosStore.findItem(headers.getPartitionId(), cosmosDBName, schemaInfoContainer, id, partitionKey, SchemaInfoDoc.class).isPresent(); if (!exists) { return false; } // Delete the item. - deleteItemInCosmos(headers.getPartitionId(), cosmosDBName, schemaInfoContainer, id, headers.getPartitionId()); + cosmosStore.deleteItem(headers.getPartitionId(), cosmosDBName, schemaInfoContainer, id, headers.getPartitionId()); return true; } + @Override + public boolean cleanSystemSchema(String schemaId) throws ApplicationException { + SchemaIdentity schemaIdentity = schemaKindToSchemaIdentity(schemaId); + String partitionKey = createSchemaInfoPartitionKey(schemaIdentity); + // Check whether SchemaInfo already exists + Boolean exists = cosmosStore.findItem(systemResourceConfig.getCosmosDatabase(), schemaInfoContainer, schemaId, partitionKey, SchemaInfoDoc.class).isPresent(); + if (!exists) { + return false; + } + + cosmosStore.deleteItem(systemResourceConfig.getCosmosDatabase(), schemaInfoContainer, schemaId, partitionKey); + return false; + } + @Override public String getLatestMinorVerSchema(SchemaInfo schemaInfo) throws ApplicationException { diff --git a/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSchemaInfoStoreTest.java b/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSchemaInfoStoreTest.java index 18fef1b6..e2201f49 100644 --- a/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSchemaInfoStoreTest.java +++ b/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSchemaInfoStoreTest.java @@ -445,9 +445,6 @@ public class AzureSchemaInfoStoreTest { assertNotNull(schemaInfoStore.updateSystemSchemaInfo(getMockSchemaObject_Published())); - // This is temporary and will be removed once schema-core starts consuming *system* methods - //assertNotNull(schemaInfoStore.updateSchemaInfo(getMockSchemaObject_Published())); - verify(this.cosmosStore, times(1)).upsertItem(any(), any(), any(), any()); verify(this.cosmosStore, times(0)).upsertItem(any(), any(), any(), any(), any()); } @@ -626,9 +623,11 @@ public class AzureSchemaInfoStoreTest { .findItem( eq(systemCosmosDBName), any(), - eq(commonTenantId + ":" + schemaId), + eq(schemaId), eq(partitionKey), any()); + + // This is temporary and will be removed once schema-core starts consuming *system* methods assertEquals(true, schemaInfoStore.cleanSchema(schemaId)); verify(cosmosStore, times(1)).deleteItem(any(), any(), any(), any()); verify(cosmosStore, times(0)).deleteItem(any(), any(), any(), any(), any()); -- GitLab From fbc6ad30b392028805bb7a0e526204b3f0be886f Mon Sep 17 00:00:00 2001 From: Aman Verma Date: Thu, 12 Aug 2021 15:41:03 +0530 Subject: [PATCH 14/24] implementing another method --- .../schemainfostore/AzureSchemaInfoStore.java | 89 ++++++++++++++++--- .../AzureSchemaInfoStoreTest.java | 19 ++++ 2 files changed, 98 insertions(+), 10 deletions(-) diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java index 54c2835e..3d9920be 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java @@ -279,18 +279,26 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { return true; } + /** + * method to clean up system schema info + * @param schemaId + * @return + * @throws ApplicationException + */ @Override public boolean cleanSystemSchema(String schemaId) throws ApplicationException { SchemaIdentity schemaIdentity = schemaKindToSchemaIdentity(schemaId); String partitionKey = createSchemaInfoPartitionKey(schemaIdentity); // Check whether SchemaInfo already exists Boolean exists = cosmosStore.findItem(systemResourceConfig.getCosmosDatabase(), schemaInfoContainer, schemaId, partitionKey, SchemaInfoDoc.class).isPresent(); + System.out.println(exists); if (!exists) { return false; } + // Delete the item. cosmosStore.deleteItem(systemResourceConfig.getCosmosDatabase(), schemaInfoContainer, schemaId, partitionKey); - return false; + return true; } @Override @@ -413,8 +421,63 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { .schemaVersionPatch(flattenedSchemaInfo.getPatchVersion()).build(); } + /** + * Method to get schema info list for a given tenant. + * @param queryParams + * @param tenantId + * @return + * @throws ApplicationException + */ @Override public List getSchemaInfoList(QueryParams queryParams, String tenantId) throws ApplicationException { + // This if block will be removed once schema-core starts consuming *System* methods. + if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(tenantId)) { + return this.getSystemSchemaInfoList(queryParams); + } + + SqlQuerySpec query = this.prepareSqlQuery(queryParams); + CosmosQueryRequestOptions options = this.prepareCosmosQueryRequestOptions(queryParams); + + List schemaInfoList = queryItemsInCosmos(tenantId, cosmosDBName,schemaInfoContainer, query, options, SchemaInfoDoc.class); + List schemaList = new LinkedList<>(); + for (SchemaInfoDoc info: schemaInfoList) + { + schemaList.add(getSchemaInfoObject(info.getFlattenedSchemaInfo(), tenantId)); + } + + if (queryParams.getLatestVersion() != null && queryParams.getLatestVersion()) { + return getLatestVersionSchemaList(schemaList); + } + + return schemaList; + } + + /** + * Method to get schema info list for system schemas. + * @param queryParams + * @return + * @throws ApplicationException + */ + @Override + public List getSystemSchemaInfoList(QueryParams queryParams) throws ApplicationException { + SqlQuerySpec query = this.prepareSqlQuery(queryParams); + CosmosQueryRequestOptions options = this.prepareCosmosQueryRequestOptions(queryParams); + + List schemaInfoList = cosmosStore.queryItems(systemResourceConfig.getCosmosDatabase(),schemaInfoContainer, query, options, SchemaInfoDoc.class); + List schemaList = new LinkedList<>(); + for (SchemaInfoDoc info: schemaInfoList) + { + schemaList.add(getSchemaInfoObject(info.getFlattenedSchemaInfo(), null)); + } + + if (queryParams.getLatestVersion() != null && queryParams.getLatestVersion()) { + return getLatestVersionSchemaList(schemaList); + } + + return schemaList; + } + + private SqlQuerySpec prepareSqlQuery(QueryParams queryParams) { String queryText = "SELECT * FROM c WHERE 1=1 "; HashMap parameterMap = new HashMap<>(); @@ -469,19 +532,25 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { query.getParameters().add(new SqlParameter(param, parameterMap.get(param))); } - List schemaInfoList = queryItemsInCosmos(tenantId, cosmosDBName,schemaInfoContainer, query, options, SchemaInfoDoc.class); + return query; + } - List schemaList = new LinkedList<>(); - for (SchemaInfoDoc info: schemaInfoList) - { - schemaList.add(getSchemaInfoObject(info.getFlattenedSchemaInfo(), tenantId)); - } + private CosmosQueryRequestOptions prepareCosmosQueryRequestOptions(QueryParams queryParams) { + CosmosQueryRequestOptions options = new CosmosQueryRequestOptions(); - if (queryParams.getLatestVersion() != null && queryParams.getLatestVersion()) { - return getLatestVersionSchemaList(schemaList); + SchemaIdentity schemaIdentity = SchemaIdentity.builder() + .authority(queryParams.getAuthority()) + .source(queryParams.getSource()) + .entityType(queryParams.getEntityType()) + .schemaVersionMajor(queryParams.getSchemaVersionMajor()).build(); + String partitionKeyStr = createSchemaInfoPartitionKey(schemaIdentity); + + if(false == StringUtils.isBlank(partitionKeyStr)) { + PartitionKey partitionKey = new PartitionKey(partitionKeyStr); + options = options.setPartitionKey(partitionKey); } - return schemaList; + return options; } @Override diff --git a/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSchemaInfoStoreTest.java b/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSchemaInfoStoreTest.java index e2201f49..ea08fec7 100644 --- a/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSchemaInfoStoreTest.java +++ b/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSchemaInfoStoreTest.java @@ -589,6 +589,25 @@ public class AzureSchemaInfoStoreTest { .limit(100).offset(0).build(), dataPartitionId).size()); } + @Test + public void testGetSchemaInfoList_withqueryparam_PublicSchemas() + throws NotFoundException, ApplicationException, BadRequestException { + Mockito.when(headers.getPartitionId()).thenReturn(commonTenantId); + List schemaInfoDocsList = new LinkedList<>(); + schemaInfoDocsList.add(getMockSchemaInfoDoc()); + doReturn(schemaInfoDocsList).when(cosmosStore).queryItems(eq(commonTenantId), any(), any(), any(), any(), any()); + + assertEquals(1, + schemaInfoStore.getSystemSchemaInfoList(QueryParams.builder().authority("test").source("test").entityType("test") + .schemaVersionMajor(1l).schemaVersionMinor(1l).scope("test").status("test").latestVersion(false) + .limit(100).offset(0).build()).size()); + +// assertEquals(1, +// schemaInfoStore.getSchemaInfoList(QueryParams.builder().authority("test").source("test").entityType("test") +// .schemaVersionMajor(1l).schemaVersionMinor(1l).scope("test").status("test").latestVersion(false) +// .limit(100).offset(0).build(), commonTenantId).size()); + } + @Test public void testGetSchemaInfoList_latestVersionTrue_NoSchemaMatchScenario() throws NotFoundException, ApplicationException, BadRequestException { -- GitLab From ea17d5d952d9b7e650fd1fe9d3e40fe6c84cc7d9 Mon Sep 17 00:00:00 2001 From: Aman Verma Date: Thu, 12 Aug 2021 15:52:00 +0530 Subject: [PATCH 15/24] fixing the UT --- .../AzureSchemaInfoStoreTest.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSchemaInfoStoreTest.java b/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSchemaInfoStoreTest.java index ea08fec7..92456b5a 100644 --- a/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSchemaInfoStoreTest.java +++ b/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSchemaInfoStoreTest.java @@ -595,17 +595,18 @@ public class AzureSchemaInfoStoreTest { Mockito.when(headers.getPartitionId()).thenReturn(commonTenantId); List schemaInfoDocsList = new LinkedList<>(); schemaInfoDocsList.add(getMockSchemaInfoDoc()); - doReturn(schemaInfoDocsList).when(cosmosStore).queryItems(eq(commonTenantId), any(), any(), any(), any(), any()); + doReturn(schemaInfoDocsList).when(cosmosStore).queryItems(eq(systemCosmosDBName), any(), any(), any(), any()); assertEquals(1, schemaInfoStore.getSystemSchemaInfoList(QueryParams.builder().authority("test").source("test").entityType("test") .schemaVersionMajor(1l).schemaVersionMinor(1l).scope("test").status("test").latestVersion(false) .limit(100).offset(0).build()).size()); -// assertEquals(1, -// schemaInfoStore.getSchemaInfoList(QueryParams.builder().authority("test").source("test").entityType("test") -// .schemaVersionMajor(1l).schemaVersionMinor(1l).scope("test").status("test").latestVersion(false) -// .limit(100).offset(0).build(), commonTenantId).size()); + // This is temporary and will be removed once schema-core starts consuming *system* methods + assertEquals(1, + schemaInfoStore.getSchemaInfoList(QueryParams.builder().authority("test").source("test").entityType("test") + .schemaVersionMajor(1l).schemaVersionMinor(1l).scope("test").status("test").latestVersion(false) + .limit(100).offset(0).build(), commonTenantId).size()); } @Test @@ -646,9 +647,12 @@ public class AzureSchemaInfoStoreTest { eq(partitionKey), any()); + assertEquals(true, schemaInfoStore.cleanSystemSchema(schemaId)); + // This is temporary and will be removed once schema-core starts consuming *system* methods assertEquals(true, schemaInfoStore.cleanSchema(schemaId)); - verify(cosmosStore, times(1)).deleteItem(any(), any(), any(), any()); + + verify(cosmosStore, times(2)).deleteItem(any(), any(), any(), any()); verify(cosmosStore, times(0)).deleteItem(any(), any(), any(), any(), any()); } -- GitLab From f6e709c50e38db6534db8b61e2bd41e07eeb919b Mon Sep 17 00:00:00 2001 From: Aman Verma Date: Thu, 12 Aug 2021 16:17:18 +0530 Subject: [PATCH 16/24] implementing the isUniqueSystemSchema method --- .../schemainfostore/AzureSchemaInfoStore.java | 202 ++++++++++-------- .../AzureSchemaInfoStoreTest.java | 9 +- 2 files changed, 122 insertions(+), 89 deletions(-) diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java index 3d9920be..f2289f5c 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java @@ -340,6 +340,120 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { return new String(); } + /** + * Method to get schema info list for a given tenant. + * @param queryParams + * @param tenantId + * @return + * @throws ApplicationException + */ + @Override + public List getSchemaInfoList(QueryParams queryParams, String tenantId) throws ApplicationException { + // This if block will be removed once schema-core starts consuming *System* methods. + if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(tenantId)) { + return this.getSystemSchemaInfoList(queryParams); + } + + SqlQuerySpec query = this.prepareSqlQuery(queryParams); + CosmosQueryRequestOptions options = this.prepareCosmosQueryRequestOptions(queryParams); + + List schemaInfoList = queryItemsInCosmos(tenantId, cosmosDBName,schemaInfoContainer, query, options, SchemaInfoDoc.class); + List schemaList = new LinkedList<>(); + for (SchemaInfoDoc info: schemaInfoList) + { + schemaList.add(getSchemaInfoObject(info.getFlattenedSchemaInfo(), tenantId)); + } + + if (queryParams.getLatestVersion() != null && queryParams.getLatestVersion()) { + return getLatestVersionSchemaList(schemaList); + } + + return schemaList; + } + + /** + * Method to get schema info list for system schemas. + * @param queryParams + * @return + * @throws ApplicationException + */ + @Override + public List getSystemSchemaInfoList(QueryParams queryParams) throws ApplicationException { + SqlQuerySpec query = this.prepareSqlQuery(queryParams); + CosmosQueryRequestOptions options = this.prepareCosmosQueryRequestOptions(queryParams); + + List schemaInfoList = cosmosStore.queryItems(systemResourceConfig.getCosmosDatabase(),schemaInfoContainer, query, options, SchemaInfoDoc.class); + List schemaList = new LinkedList<>(); + for (SchemaInfoDoc info: schemaInfoList) + { + schemaList.add(getSchemaInfoObject(info.getFlattenedSchemaInfo(), null)); + } + + if (queryParams.getLatestVersion() != null && queryParams.getLatestVersion()) { + return getLatestVersionSchemaList(schemaList); + } + + return schemaList; + } + + @Override + public boolean isUnique(String schemaId, String tenantId) throws ApplicationException { + // This if block will be removed once schema-core starts consuming *System* methods. + if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(tenantId)) { + return this.isUniqueSystemSchema(schemaId); + } + + // Check for uniqueness in the system schemas + String partitionKey = createSchemaInfoPartitionKey(schemaKindToSchemaIdentity(schemaId)); + Boolean exists = cosmosStore.findItem(systemResourceConfig.getCosmosDatabase(),schemaInfoContainer, schemaId, partitionKey, SchemaInfoDoc.class).isPresent(); + if (exists) { + return false; + } + + // Check for uniqueness in current tenant. + String id = tenantId + ":" + schemaId; + try { + exists = cosmosStore.findItem(tenantId, cosmosDBName, schemaInfoContainer, id, partitionKey, SchemaInfoDoc.class).isPresent(); + if (exists) { + return false; + } + } catch (AppException ex) { + log.warning(String.format("Error occurred while performing uniqueness check in tenant '%s'", tenantId), ex); + } + + return true; + } + + @Override + public boolean isUniqueSystemSchema(String schemaId) throws ApplicationException { + // Check whether the target schema is already part of system schemas + String partitionKey = createSchemaInfoPartitionKey(schemaKindToSchemaIdentity(schemaId)); + Boolean exists = cosmosStore.findItem(systemResourceConfig.getCosmosDatabase(),schemaInfoContainer, schemaId, partitionKey, SchemaInfoDoc.class).isPresent(); + if (exists) { + return false; + } + + // Check in other tenants + Set tenantList = new HashSet<>(); + List privateTenantList = tenantFactory.listTenantInfo().stream().map(TenantInfo::getName) + .collect(Collectors.toList()); + tenantList.addAll(privateTenantList); + + for (String tenant : tenantList) { + String id = tenant + ":" + schemaId; + + try { + exists = findItemInCosmosStore(tenant, cosmosDBName, schemaInfoContainer, id, partitionKey, SchemaInfoDoc.class).isPresent(); + if (exists) { + return false; + } + } catch (AppException ex) { + log.warning(String.format("Error occurred while performing uniqueness check in tenant '%s'", tenant), ex); + } + } + return true; + } + /** * Creates schemaInfo object and populates required properties. * @@ -421,62 +535,6 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { .schemaVersionPatch(flattenedSchemaInfo.getPatchVersion()).build(); } - /** - * Method to get schema info list for a given tenant. - * @param queryParams - * @param tenantId - * @return - * @throws ApplicationException - */ - @Override - public List getSchemaInfoList(QueryParams queryParams, String tenantId) throws ApplicationException { - // This if block will be removed once schema-core starts consuming *System* methods. - if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(tenantId)) { - return this.getSystemSchemaInfoList(queryParams); - } - - SqlQuerySpec query = this.prepareSqlQuery(queryParams); - CosmosQueryRequestOptions options = this.prepareCosmosQueryRequestOptions(queryParams); - - List schemaInfoList = queryItemsInCosmos(tenantId, cosmosDBName,schemaInfoContainer, query, options, SchemaInfoDoc.class); - List schemaList = new LinkedList<>(); - for (SchemaInfoDoc info: schemaInfoList) - { - schemaList.add(getSchemaInfoObject(info.getFlattenedSchemaInfo(), tenantId)); - } - - if (queryParams.getLatestVersion() != null && queryParams.getLatestVersion()) { - return getLatestVersionSchemaList(schemaList); - } - - return schemaList; - } - - /** - * Method to get schema info list for system schemas. - * @param queryParams - * @return - * @throws ApplicationException - */ - @Override - public List getSystemSchemaInfoList(QueryParams queryParams) throws ApplicationException { - SqlQuerySpec query = this.prepareSqlQuery(queryParams); - CosmosQueryRequestOptions options = this.prepareCosmosQueryRequestOptions(queryParams); - - List schemaInfoList = cosmosStore.queryItems(systemResourceConfig.getCosmosDatabase(),schemaInfoContainer, query, options, SchemaInfoDoc.class); - List schemaList = new LinkedList<>(); - for (SchemaInfoDoc info: schemaInfoList) - { - schemaList.add(getSchemaInfoObject(info.getFlattenedSchemaInfo(), null)); - } - - if (queryParams.getLatestVersion() != null && queryParams.getLatestVersion()) { - return getLatestVersionSchemaList(schemaList); - } - - return schemaList; - } - private SqlQuerySpec prepareSqlQuery(QueryParams queryParams) { String queryText = "SELECT * FROM c WHERE 1=1 "; HashMap parameterMap = new HashMap<>(); @@ -553,36 +611,6 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { return options; } - @Override - public boolean isUnique(String schemaId, String tenantId) throws ApplicationException { - Set tenantList = new HashSet<>(); - tenantList.add(sharedTenant); - tenantList.add(tenantId); - - /* TODO : Below code enables uniqueness check across tenants and is redundant now. This will be handled/updated as part - of data partition changes. - */ - if (tenantId.equalsIgnoreCase(sharedTenant)) { - List privateTenantList = tenantFactory.listTenantInfo().stream().map(TenantInfo::getName) - .collect(Collectors.toList()); - tenantList.addAll(privateTenantList); - } - - for (String tenant : tenantList) { - String id = tenant + ":" + schemaId; - String partitionKey = createSchemaInfoPartitionKey(schemaKindToSchemaIdentity(schemaId)); - try { - Boolean exists = findItemInCosmosStore(tenant, cosmosDBName, schemaInfoContainer, id, partitionKey, SchemaInfoDoc.class).isPresent(); - if (exists) { - return false; - } - } catch (AppException ex) { - log.warning(String.format("Error occurred while performing uniqueness check in tenant '%s'", tenant), ex); - } - } - return true; - } - private List getLatestVersionSchemaList(List filteredSchemaList) { List latestSchemaList = new LinkedList<>(); SchemaInfo previousSchemaInfo = null; @@ -641,7 +669,7 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { } private String createSchemaInfoPartitionKey(SchemaIdentity schemaIdentity) { - + if(StringUtils.isBlank(schemaIdentity.getAuthority()) || StringUtils.isBlank(schemaIdentity.getSource()) || StringUtils.isBlank(schemaIdentity.getEntityType()) diff --git a/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSchemaInfoStoreTest.java b/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSchemaInfoStoreTest.java index 92456b5a..33830f0d 100644 --- a/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSchemaInfoStoreTest.java +++ b/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSchemaInfoStoreTest.java @@ -374,11 +374,16 @@ public class AzureSchemaInfoStoreTest { .findItem( eq(systemCosmosDBName), any(), - eq(commonTenantId + ":" + schemaId), + eq(schemaId), eq(partitionKey), any()); + + assertFalse(schemaInfoStore.isUniqueSystemSchema(schemaId)); + + // This is temporary and will be removed once schema-core starts consuming *system* methods assertFalse(schemaInfoStore.isUnique(schemaId, commonTenantId)); - verify(this.cosmosStore, times(1)).findItem(any(), any(), anyString(), anyString(), any()); + + verify(this.cosmosStore, times(2)).findItem(any(), any(), anyString(), anyString(), any()); verify(this.cosmosStore, times(0)).findItem(anyString(), anyString(), anyString(), anyString(), anyString(), any()); } -- GitLab From 9ecc6ea9d436da49f01e959a1751aa331fe34ec5 Mon Sep 17 00:00:00 2001 From: Aman Verma Date: Thu, 12 Aug 2021 17:13:49 +0530 Subject: [PATCH 17/24] deleting now unused methods --- .../schemainfostore/AzureSchemaInfoStore.java | 51 ++----------------- 1 file changed, 3 insertions(+), 48 deletions(-) diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java index f2289f5c..ed30d716 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java @@ -107,7 +107,7 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { SchemaIdentity schemaIdentity = schemaKindToSchemaIdentity(schemaId); String partitioningKey = createSchemaInfoPartitionKey(schemaIdentity); - SchemaInfoDoc schemaInfoDoc = findItemInCosmosStore(headers.getPartitionId(), cosmosDBName, schemaInfoContainer, id, partitioningKey, SchemaInfoDoc.class) + SchemaInfoDoc schemaInfoDoc = cosmosStore.findItem(headers.getPartitionId(), cosmosDBName, schemaInfoContainer, id, partitioningKey, SchemaInfoDoc.class) .orElseThrow(() -> new NotFoundException(SchemaConstants.SCHEMA_NOT_PRESENT)); return getSchemaInfoObject(schemaInfoDoc.getFlattenedSchemaInfo(), headers.getPartitionId()); @@ -357,7 +357,7 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { SqlQuerySpec query = this.prepareSqlQuery(queryParams); CosmosQueryRequestOptions options = this.prepareCosmosQueryRequestOptions(queryParams); - List schemaInfoList = queryItemsInCosmos(tenantId, cosmosDBName,schemaInfoContainer, query, options, SchemaInfoDoc.class); + List schemaInfoList = cosmosStore.queryItems(tenantId, cosmosDBName,schemaInfoContainer, query, options, SchemaInfoDoc.class); List schemaList = new LinkedList<>(); for (SchemaInfoDoc info: schemaInfoList) { @@ -443,7 +443,7 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { String id = tenant + ":" + schemaId; try { - exists = findItemInCosmosStore(tenant, cosmosDBName, schemaInfoContainer, id, partitionKey, SchemaInfoDoc.class).isPresent(); + exists = cosmosStore.findItem(tenant, cosmosDBName, schemaInfoContainer, id, partitionKey, SchemaInfoDoc.class).isPresent(); if (exists) { return false; } @@ -699,51 +699,6 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { } } - private void crateItemInCosmos( - String dataPartitionId, - String dataBaseName, - String containerName, - String partitionKey, - T item) { - if (dataPartitionId == null || - dataPartitionId.isEmpty() || - systemResourceConfig.getSharedTenant().equalsIgnoreCase(dataPartitionId)) { - cosmosStore.createItem(systemResourceConfig.getCosmosDatabase(), containerName, partitionKey, item); - } else { - cosmosStore.createItem(dataPartitionId, dataBaseName, containerName, partitionKey, item); - } - } - - private void upsertItemInCosmos( - String dataPartitionId, - String dataBaseName, - String containerName, - String partitionKey, - T item) { - if (dataPartitionId == null || - dataPartitionId.isEmpty() || - systemResourceConfig.getSharedTenant().equalsIgnoreCase(dataPartitionId)) { - cosmosStore.upsertItem(systemResourceConfig.getCosmosDatabase(), containerName, partitionKey, item); - } else { - cosmosStore.upsertItem(dataPartitionId, dataBaseName, containerName, partitionKey, item); - } - } - - private void deleteItemInCosmos( - String dataPartitionId, - String dataBaseName, - String containerName, - String id, - String partitionKey) { - if (dataPartitionId == null || - dataPartitionId.isEmpty() || - systemResourceConfig.getSharedTenant().equalsIgnoreCase(dataPartitionId)) { - cosmosStore.deleteItem(systemResourceConfig.getCosmosDatabase(), containerName, id, partitionKey); - } else { - cosmosStore.deleteItem(dataPartitionId, dataBaseName, containerName, id, partitionKey); - } - } - private List queryItemsInCosmos( String dataPartitionId, String cosmosDBName, -- GitLab From ec5f6609a182a0a2149811bd45222f0349c57aff Mon Sep 17 00:00:00 2001 From: Aman Verma Date: Thu, 12 Aug 2021 17:17:55 +0530 Subject: [PATCH 18/24] deleting IDE generated files --- provider/schema-azure/os-schema-azure.ipr | 107 ------ provider/schema-azure/os-schema-azure.iws | 418 ---------------------- 2 files changed, 525 deletions(-) delete mode 100644 provider/schema-azure/os-schema-azure.ipr delete mode 100644 provider/schema-azure/os-schema-azure.iws diff --git a/provider/schema-azure/os-schema-azure.ipr b/provider/schema-azure/os-schema-azure.ipr deleted file mode 100644 index 884e359a..00000000 --- a/provider/schema-azure/os-schema-azure.ipr +++ /dev/null @@ -1,107 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/provider/schema-azure/os-schema-azure.iws b/provider/schema-azure/os-schema-azure.iws deleted file mode 100644 index 03c854e9..00000000 --- a/provider/schema-azure/os-schema-azure.iws +++ /dev/null @@ -1,418 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -- GitLab From efedf13dc10ea67863924b3ef34306ada5558180 Mon Sep 17 00:00:00 2001 From: Aman Verma Date: Thu, 12 Aug 2021 20:25:31 +0530 Subject: [PATCH 19/24] fixing method call --- .../schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java index ed30d716..34f230a6 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java @@ -206,7 +206,7 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { public SchemaInfo updateSchemaInfo(SchemaRequest schema) throws ApplicationException, BadRequestException { // This if block will be removed once schema-core starts consuming *System* methods. if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { - return this.createSystemSchemaInfo(schema); + return this.updateSystemSchemaInfo(schema); } String id = headers.getPartitionId() + ":" + schema.getSchemaInfo().getSchemaIdentity().getId(); -- GitLab From e95492bd4a41f185ac1d97b3b2374cee236a7a1c Mon Sep 17 00:00:00 2001 From: Aman Verma Date: Mon, 16 Aug 2021 11:41:49 +0530 Subject: [PATCH 20/24] fixing logic --- .../impl/schemainfostore/AzureSchemaInfoStore.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java index 34f230a6..1f62f4c2 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java @@ -447,7 +447,7 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { if (exists) { return false; } - } catch (AppException ex) { + } catch (Exception ex) { log.warning(String.format("Error occurred while performing uniqueness check in tenant '%s'", tenant), ex); } } @@ -503,7 +503,7 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { .build(); } - private SchemaInfo getSchemaInfoObject(FlattenedSchemaInfo flattenedSchemaInfo, String dataPartitionId) { + private SchemaInfo getSchemaInfoObject(FlattenedSchemaInfo flattenedSchemaInfo, String dataPartitionId) throws ApplicationException { SchemaIdentity superSededBy = null; if (!flattenedSchemaInfo.getSupersededBy().isEmpty()) { String id = flattenedSchemaInfo.getSupersededBy(); @@ -513,8 +513,13 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { SchemaIdentity schemaIdentity = schemaKindToSchemaIdentity(flattenedSchemaInfo.getSupersededBy()); String partitionKey = createSchemaInfoPartitionKey(schemaIdentity); - SchemaInfoDoc doc = findItemInCosmosStore(dataPartitionId, cosmosDBName, schemaInfoContainer, id, partitionKey, SchemaInfoDoc.class).get(); - superSededBy = getSchemaIdentity(doc.getFlattenedSchemaInfo()); + try { + SchemaInfoDoc doc = findItemInCosmosStore(dataPartitionId, cosmosDBName, schemaInfoContainer, id, partitionKey, SchemaInfoDoc.class).get(); + superSededBy = getSchemaIdentity(doc.getFlattenedSchemaInfo()); + } catch (AppException ex) { + log.error(SchemaConstants.INVALID_SUPERSEDEDBY_ID); + throw new ApplicationException(SchemaConstants.INVALID_SUPERSEDEDBY_ID); + } } return SchemaInfo.builder().createdBy(flattenedSchemaInfo.getCreatedBy()) -- GitLab From 8b6135f337a98c85f03f364bbe9d7f421cdf5325 Mon Sep 17 00:00:00 2001 From: Aman Verma Date: Mon, 16 Aug 2021 17:10:44 +0530 Subject: [PATCH 21/24] updating UTs --- .../impl/schemainfostore/AzureSchemaInfoStore.java | 13 ++++++------- .../schemainfostore/AzureSchemaInfoStoreTest.java | 6 ++++++ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java index 1f62f4c2..92cfc815 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java @@ -191,7 +191,7 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { } log.info(SchemaConstants.SCHEMA_INFO_CREATED); - return getSchemaInfoObject(flattenedSchemaInfo, headers.getPartitionId()); + return getSchemaInfoObject(flattenedSchemaInfo, null); } /** @@ -234,7 +234,7 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { @Override public SchemaInfo updateSystemSchemaInfo(SchemaRequest schema) throws ApplicationException, BadRequestException { String id = schema.getSchemaInfo().getSchemaIdentity().getId(); - FlattenedSchemaInfo flattenedSchemaInfo = populateSchemaInfo(schema, headers.getPartitionId()); + FlattenedSchemaInfo flattenedSchemaInfo = populateSchemaInfo(schema, null); String partitionKey = createSchemaInfoPartitionKey(schema.getSchemaInfo().getSchemaIdentity()); SchemaInfoDoc schemaInfoDoc = new SchemaInfoDoc(id, partitionKey, flattenedSchemaInfo); @@ -246,7 +246,7 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { } log.info(SchemaConstants.SCHEMA_INFO_UPDATED); - return getSchemaInfoObject(flattenedSchemaInfo, headers.getPartitionId()); + return getSchemaInfoObject(flattenedSchemaInfo, null); } /** @@ -291,7 +291,6 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { String partitionKey = createSchemaInfoPartitionKey(schemaIdentity); // Check whether SchemaInfo already exists Boolean exists = cosmosStore.findItem(systemResourceConfig.getCosmosDatabase(), schemaInfoContainer, schemaId, partitionKey, SchemaInfoDoc.class).isPresent(); - System.out.println(exists); if (!exists) { return false; } @@ -443,11 +442,11 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { String id = tenant + ":" + schemaId; try { - exists = cosmosStore.findItem(tenant, cosmosDBName, schemaInfoContainer, id, partitionKey, SchemaInfoDoc.class).isPresent(); - if (exists) { + Boolean existsInTenant = findItemInCosmosStore(tenant, cosmosDBName, schemaInfoContainer, id, partitionKey, SchemaInfoDoc.class).isPresent(); + if (existsInTenant) { return false; } - } catch (Exception ex) { + } catch (AppException ex) { log.warning(String.format("Error occurred while performing uniqueness check in tenant '%s'", tenant), ex); } } diff --git a/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSchemaInfoStoreTest.java b/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSchemaInfoStoreTest.java index 33830f0d..bd5abb35 100644 --- a/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSchemaInfoStoreTest.java +++ b/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/provider/azure/impl/schemainfostore/AzureSchemaInfoStoreTest.java @@ -244,6 +244,7 @@ public class AzureSchemaInfoStoreTest { verify(this.cosmosStore, times(2)).createItem(any(), any(), eq("os:wks:well:1"), any()); verify(this.cosmosStore, times(0)).createItem(anyString(), anyString(), anyString(), anyString(), any()); + verify(this.cosmosStore, times(0)).findItem(any(), anyString(), anyString(), anyString(), anyString(), any()); } @Test @@ -452,6 +453,7 @@ public class AzureSchemaInfoStoreTest { verify(this.cosmosStore, times(1)).upsertItem(any(), any(), any(), any()); verify(this.cosmosStore, times(0)).upsertItem(any(), any(), any(), any(), any()); + verify(this.cosmosStore, times(0)).findItem(any(), any(), any(), any(), any(), any()); } @Test @@ -612,6 +614,9 @@ public class AzureSchemaInfoStoreTest { schemaInfoStore.getSchemaInfoList(QueryParams.builder().authority("test").source("test").entityType("test") .schemaVersionMajor(1l).schemaVersionMinor(1l).scope("test").status("test").latestVersion(false) .limit(100).offset(0).build(), commonTenantId).size()); + + verify(this.cosmosStore, times(2)).queryItems(eq(systemCosmosDBName), any(),any(), any(), eq(SchemaInfoDoc.class)); + verify(this.cosmosStore, times(0)).queryItems(any(), any(), any(),any(), any(), eq(SchemaInfoDoc.class)); } @Test @@ -659,6 +664,7 @@ public class AzureSchemaInfoStoreTest { verify(cosmosStore, times(2)).deleteItem(any(), any(), any(), any()); verify(cosmosStore, times(0)).deleteItem(any(), any(), any(), any(), any()); + verify(cosmosStore, times(0)).findItem(any(), any(), any(), any(), any(), any()); } @Test -- GitLab From 4b80fcc117dc920b3f47d4c5793affd0ef9d3096 Mon Sep 17 00:00:00 2001 From: Aman Verma Date: Mon, 16 Aug 2021 17:35:24 +0530 Subject: [PATCH 22/24] updating the MessageBusImpl class --- .../schema/azure/impl/messagebus/MessageBusImpl.java | 10 ++++++++++ .../impl/schemainfostore/AzureEntityTypeStore.java | 2 +- .../azure/impl/messagebus/MessageBusImplTest.java | 12 ++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/messagebus/MessageBusImpl.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/messagebus/MessageBusImpl.java index 2de25002..6aceb8b8 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/messagebus/MessageBusImpl.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/messagebus/MessageBusImpl.java @@ -30,6 +30,7 @@ import org.opengroup.osdu.core.common.model.http.DpsHeaders; import org.opengroup.osdu.core.common.model.tenant.TenantInfo; import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory; import org.opengroup.osdu.schema.azure.di.EventGridConfig; +import org.opengroup.osdu.schema.azure.di.SystemResourceConfig; import org.opengroup.osdu.schema.azure.impl.messagebus.model.SchemaPubSubInfo; import org.opengroup.osdu.schema.constants.SchemaConstants; import org.opengroup.osdu.schema.logging.AuditLogger; @@ -55,10 +56,19 @@ public class MessageBusImpl implements IMessageBus { @Autowired private ITenantFactory tenantFactory; + @Autowired + SystemResourceConfig systemResourceConfig; + private final static String EVENT_DATA_VERSION = "1.0"; @Override public void publishMessage(String schemaId, String eventType) { + // This if block will be removed once schema-core starts consuming *System* methods. + if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { + this.publishMessageForSystemSchema(schemaId, eventType); + return; + } + if (eventGridConfig.isEventGridEnabled()) { logger.info("Generating event of type {}",eventType); try { diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureEntityTypeStore.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureEntityTypeStore.java index e4590b65..7b3c8f8d 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureEntityTypeStore.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureEntityTypeStore.java @@ -109,7 +109,7 @@ public class AzureEntityTypeStore implements IEntityTypeStore { public EntityType create(EntityType entityType) throws BadRequestException, ApplicationException { // This if block will be removed once schema-core starts consuming *System* methods. if (systemResourceConfig.getSharedTenant().equalsIgnoreCase(headers.getPartitionId())) { - this.createSystemEntity(entityType); + return this.createSystemEntity(entityType); } String id = headers.getPartitionId() + ":" + entityType.getEntityTypeId(); diff --git a/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/azure/impl/messagebus/MessageBusImplTest.java b/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/azure/impl/messagebus/MessageBusImplTest.java index 0224bb9d..b72eb4a0 100644 --- a/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/azure/impl/messagebus/MessageBusImplTest.java +++ b/provider/schema-azure/src/test/java/org/opengroup/osdu/schema/azure/impl/messagebus/MessageBusImplTest.java @@ -41,6 +41,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.opengroup.osdu.azure.eventgrid.EventGridTopicStore; import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; @@ -48,6 +49,7 @@ import org.opengroup.osdu.core.common.model.http.DpsHeaders; import org.opengroup.osdu.core.common.model.tenant.TenantInfo; import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory; import org.opengroup.osdu.schema.azure.di.EventGridConfig; +import org.opengroup.osdu.schema.azure.di.SystemResourceConfig; import org.opengroup.osdu.schema.azure.impl.messagebus.model.SchemaPubSubInfo; import org.opengroup.osdu.schema.logging.AuditLogger; @@ -61,6 +63,8 @@ public class MessageBusImplTest { private static final String CORRELATION_ID = "correlation-id"; private static final String PARTITION_ID = "partition-id"; private static final String OTHER_TENANT = "other-tenant-id"; + private static final String systemCosmosDBName = "osdu-system-db"; + private static final String sharedTenantId = "common"; @Mock private EventGridTopicStore eventGridTopicStore; @@ -79,6 +83,9 @@ public class MessageBusImplTest { @Mock private ITenantFactory tenantFactory; + + @Mock + SystemResourceConfig systemResourceConfig; @InjectMocks private MessageBusImpl messageBusImpl; @@ -89,6 +96,8 @@ public class MessageBusImplTest { doReturn(DATA_PARTITION_WITH_FALLBACK_ACCOUNT_ID).when(dpsHeaders).getPartitionIdWithFallbackToAccountId(); doReturn(PARTITION_ID).when(dpsHeaders).getPartitionId(); doReturn(CORRELATION_ID).when(dpsHeaders).getCorrelationId(); + Mockito.when(systemResourceConfig.getCosmosDatabase()).thenReturn(systemCosmosDBName); + Mockito.when(systemResourceConfig.getSharedTenant()).thenReturn(sharedTenantId); } @Test @@ -103,10 +112,12 @@ public class MessageBusImplTest { @Test public void should_publishToEventGrid_WhenFlagIsFalse_PublicSchemas() { + Mockito.when(dpsHeaders.getPartitionId()).thenReturn(sharedTenantId); //The schema-notification is turned off when(this.eventGridConfig.isEventGridEnabled()).thenReturn(false); //Call publish Message messageBusImpl.publishMessageForSystemSchema("dummy", "dummy"); + messageBusImpl.publishMessage("dummy", "dummy"); //Assert that eventGridTopicStore is not called even once verify(this.eventGridTopicStore, times(0)).publishToEventGridTopic(any(), any(), anyList()); } @@ -150,6 +161,7 @@ public class MessageBusImplTest { @Test public void should_publishToEventGrid_WhenFlagIsTrue_PublicSchemas() { + Mockito.when(dpsHeaders.getPartitionId()).thenReturn(sharedTenantId); TenantInfo tenant1 = new TenantInfo(); tenant1.setName(PARTITION_ID); tenant1.setDataPartitionId(PARTITION_ID); -- GitLab From b0b747b93a098a272ba9dc27212e696011e3269a Mon Sep 17 00:00:00 2001 From: Aman Verma Date: Fri, 27 Aug 2021 15:42:25 +0530 Subject: [PATCH 23/24] addressing MR comments --- .../azure/impl/messagebus/MessageBusImpl.java | 2 +- .../schemainfostore/AzureSchemaInfoStore.java | 27 +++++++++---------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/messagebus/MessageBusImpl.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/messagebus/MessageBusImpl.java index 6aceb8b8..2a06c3d7 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/messagebus/MessageBusImpl.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/messagebus/MessageBusImpl.java @@ -72,7 +72,7 @@ public class MessageBusImpl implements IMessageBus { if (eventGridConfig.isEventGridEnabled()) { logger.info("Generating event of type {}",eventType); try { - publishToEventGrid(schemaId, eventType, headers.getPartitionIdWithFallbackToAccountId()); + publishToEventGrid(schemaId, eventType, headers.getPartitionId()); auditLogger.schemaNotificationSuccess(Collections.singletonList(schemaId)); }catch (AppException ex) { diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java index 92cfc815..b63d5d9b 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java @@ -151,13 +151,7 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { try { cosmosStore.createItem(headers.getPartitionId(), cosmosDBName, schemaInfoContainer, partitionKey, schemaInfoDoc); } catch (AppException ex) { - if (ex.getError().getCode() == 409) { - log.warning(SchemaConstants.SCHEMA_ID_EXISTS); - throw new BadRequestException(SchemaConstants.SCHEMA_ID_EXISTS); - } else { - log.error(MessageFormat.format(SchemaConstants.OBJECT_INVALID, ex.getMessage())); - throw new ApplicationException(SchemaConstants.SCHEMA_CREATION_FAILED_INVALID_OBJECT); - } + this.handleSchemaCreateException(ex); } log.info(SchemaConstants.SCHEMA_INFO_CREATED); @@ -181,13 +175,7 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { try { cosmosStore.createItem(systemResourceConfig.getCosmosDatabase(), schemaInfoContainer, partitionKey, schemaInfoDoc); } catch (AppException ex) { - if (ex.getError().getCode() == 409) { - log.warning(SchemaConstants.SCHEMA_ID_EXISTS); - throw new BadRequestException(SchemaConstants.SCHEMA_ID_EXISTS); - } else { - log.error(MessageFormat.format(SchemaConstants.OBJECT_INVALID, ex.getMessage())); - throw new ApplicationException(SchemaConstants.SCHEMA_CREATION_FAILED_INVALID_OBJECT); - } + this.handleSchemaCreateException(ex); } log.info(SchemaConstants.SCHEMA_INFO_CREATED); @@ -718,5 +706,16 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { return cosmosStore.queryItems(dataPartitionId, cosmosDBName, collection, query, options, clazz); } } + + + private void handleSchemaCreateException(AppException ex) throws BadRequestException, ApplicationException { + if (ex.getError().getCode() == 409) { + log.warning(SchemaConstants.SCHEMA_ID_EXISTS); + throw new BadRequestException(SchemaConstants.SCHEMA_ID_EXISTS); + } else { + log.error(MessageFormat.format(SchemaConstants.OBJECT_INVALID, ex.getMessage())); + throw new ApplicationException(SchemaConstants.SCHEMA_CREATION_FAILED_INVALID_OBJECT); + } + } } -- GitLab From 42987df8901bd8c6c7a26dab4b5ae68a0ea24c2b Mon Sep 17 00:00:00 2001 From: Aman Verma Date: Wed, 8 Sep 2021 13:15:51 +0530 Subject: [PATCH 24/24] adding check to igore the common tenant --- .../azure/impl/schemainfostore/AzureSchemaInfoStore.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java index b63d5d9b..df32d937 100644 --- a/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java +++ b/provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java @@ -429,6 +429,11 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore { for (String tenant : tenantList) { String id = tenant + ":" + schemaId; + // don't check in the erstwhile common tenant + if (tenant.equalsIgnoreCase(sharedTenant)) { + continue; + } + try { Boolean existsInTenant = findItemInCosmosStore(tenant, cosmosDBName, schemaInfoContainer, id, partitionKey, SchemaInfoDoc.class).isPresent(); if (existsInTenant) { -- GitLab