Skip to content
GitLab
Commit 53ef302f authored by devesh bajpai's avatar devesh bajpai
Browse files

fetch latest schema

parent d97c217c
Pipeline #26703 passed with stages
in 19 minutes and 42 seconds
Hide whitespace changes
Inline Side-by-side
testing/wks-test-core/src/test/resources/input_payloads/mappingFile.json
View file @ 53ef302f
{
"wksSchemaKind": "<tenant_name>:wks:wellbore:1.0.0",
"attributeMappings": [
"mappings": [
{ "identifier": "",
"operations": [
{
"type": "copy",
"sourceProperty": "data.rawAttribute1",
"targetProperty": "data.wksAttribute1.x"
}
]
},
{
"rawAttributeName": "acl",
"wksAttributeName": "acl"
"identifier":"",
"operations":[
{
"type": "copy",
"sourceProperty": "acl",
"targetProperty": "acl"
}
]
},
{
"rawAttributeName": "ancestry",
"wksAttributeName": "ancestry"
"identifier":"",
"operations":[
{
"type": "copy",
"sourceProperty": "ancestry",
"targetProperty": "ancestry"
}
]
},
{
"rawAttributeName": "id",
"wksAttributeName": "id"
"identifier":"",
"operations":[
{
"type": "copy",
"sourceProperty": "id",
"targetProperty": "id"
}
]
},
{
"rawAttributeName": "kind",
"wksAttributeName": "kind"
"identifier":"",
"operations":[
{
"type": "copy",
"sourceProperty": "kind",
"targetProperty": "kind"
}
]
},
{
"rawAttributeName": "legal",
"wksAttributeName": "legal"
"identifier":"",
"operations":[
{
"type": "copy",
"sourceProperty": "legal",
"targetProperty": "legal"
}
]
},
{
"rawAttributeName": "meta",
"wksAttributeName": "meta"
"identifier":"",
"operations":[
{
"type": "copy",
"sourceProperty": "meta",
"targetProperty": "meta"
}
]
},
{
"rawAttributeName": "version",
"wksAttributeName": "version"
"identifier":"",
"operations":[
{
"type": "copy",
"sourceProperty": "version",
"targetProperty": "version"
}
]
},
{
"rawAttributeName": "data.Spud date",
"wksAttributeName": "data.spudDate"
"identifier":"",
"operations":[
{
"type": "copy",
"sourceProperty": "data.Spud date",
"targetProperty": "data.spudDate"
}
]
},
{
"rawAttributeName": "data.UWI",
"wksAttributeName": "data.uwi"
"identifier":"",
"operations":[
{
"type": "copy",
"sourceProperty": "data.UWI",
"targetProperty": "data.uwi"
}
]
},
{
"rawAttributeName": "data.dlLatLongWGS84.latitude",
"wksAttributeName": "data.dlWGS84.latitude"
"identifier":"",
"operations":[
{
"type": "copy",
"sourceProperty": "data.dlLatLongWGS84.latitude",
"targetProperty": "data.dlWGS84.latitude"
}
]
},
{
"rawAttributeName": "data.dlLatLongWGS84.longitude",
"wksAttributeName": "data.dlWGS84.longitude"
"identifier":"",
"operations":[
{
"type": "copy",
"sourceProperty": "data.dlLatLongWGS84.longitude",
"targetProperty": "data.dlWGS84.longitude"
}
]
}
]
}
wks-core/src/main/java/org/opengroup/osdu/wks/model/SchemaIdentity.java
View file @ 53ef302f
package org.opengroup.osdu.wks.model;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import lombok.*;
@Getter
@Setter
@ToString
@Data
@AllArgsConstructor
@NoArgsConstructor
@Builder
public class SchemaIdentity {
private String authority;
private String source;
......
wks-core/src/main/java/org/opengroup/osdu/wks/service/ISchemaService.java
View file @ 53ef302f
package org.opengroup.osdu.wks.service;
import java.util.List;
import java.util.Optional;
import org.opengroup.osdu.wks.exceptions.ApplicationException;
public interface ISchemaService {
List<String> getSchemas(String authority, String source, String entityType, String majorVersion) throws ApplicationException;
Optional<String> getLatestSchemaKind(String authority, String source, String entityType, String majorVersion) throws ApplicationException;
}
wks-core/src/main/java/org/opengroup/osdu/wks/service/SchemaService.java
View file @ 53ef302f
......@@ -22,10 +22,12 @@ import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpMethod;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
@Component
public class SchemaService implements ISchemaService{
......@@ -51,7 +53,7 @@ public class SchemaService implements ISchemaService{
@Autowired
private KindUtil kindUtil;
public List<String> getSchemas(String authority, String source, String entityType, String majorVersion) throws ApplicationException {
public Optional<String> getLatestSchemaKind(String authority, String source, String entityType, String majorVersion) throws ApplicationException {
RequestParameters requestParameters = new RequestParameters();
TenantInfo tenantInfo = tenantFactory.getTenantInfo(requestIdentity.getDataPartitionId());
requestParameters.setUrl(getRequestURL(authority, source, entityType, majorVersion));
......@@ -65,7 +67,13 @@ public class SchemaService implements ISchemaService{
RestResponse response = restClient.processRequest(requestParameters);
verifyResponseStatus(response);
return extractResponseBody(response.getMessage());
List<String> schemakinds = extractResponseBody(response.getMessage());
if(!CollectionUtils.isEmpty(schemakinds)){
return Optional.of(schemakinds.get(0));
}
else{
return Optional.empty();
}
}
private void verifyResponseStatus(RestResponse response) throws ApplicationException {
......
wks-core/src/main/java/org/opengroup/osdu/wks/service/WKSServiceImpl.java
View file @ 53ef302f
......@@ -70,8 +70,6 @@ public class WKSServiceImpl implements WKSService {
if(rawRecordsDetailsForTransformation.length > 0) {
Map<String, List<MappingDefinition>> mappingsMap = retrieveMappingsMap(rawRecordsDetailsForTransformation);
// // Updating mappings with latest minor and patch versions
// updateMappingsWithLatestMinorAndPatchVersions(mappingsMap);
transformRecords(rawRecordsDetailsForTransformation, mappingsMap, dataPartitionId, correlationId);
}
} finally {
......@@ -156,20 +154,6 @@ public class WKSServiceImpl implements WKSService {
LOGGER.info(Constants.TRANFORMATION_PROCESS_COMPLETED);
}
// private void updateMappingsWithLatestMinorAndPatchVersions(Map<String, List<MappingDefinition>> mappingsMap) throws ApplicationException {
//
// for (Map.Entry<String, List<MappingDefinition>> entry : mappingsMap.entrySet()) {
// List<MappingDefinition> mappingsModelList = new ArrayList<>();
// for (MappingDefinition mappingsModel : entry.getValue()) {
// List<String> schemaKindsWithLatestMinorAndPatchVersions = schemaService.getSchemas(mappingsModel.getWksSchemaKind());
// mappingsModel.setWksSchemaKind(schemaKindsWithLatestMinorAndPatchVersions.get(0));
// mappingsModelList.add(mappingsModel);
// }
// entry.setValue(mappingsModelList);
// }
// LOGGER.info("Schemas for WKS fetched with latest minor and patch versions");
// }
private void initializeMessageContext(String dataPartitionId, String correlationId) {
requestIdentity.setDataPartitionId(dataPartitionId);
requestIdentity.setCorrelationId(correlationId);
......
wks-core/src/main/java/org/opengroup/osdu/wks/util/RelationshipBlockHandler.java
View file @ 53ef302f
......@@ -11,15 +11,9 @@ import java.util.stream.Collectors;
import org.opengroup.osdu.wks.constants.Constants;
import org.opengroup.osdu.wks.exceptions.ApplicationException;
import org.opengroup.osdu.wks.model.Kind;
import org.opengroup.osdu.wks.exceptions.NotFoundException;
import org.opengroup.osdu.wks.model.*;
import org.opengroup.osdu.wks.model.mapping.MappingInfo;
import org.opengroup.osdu.wks.model.Relationship;
import org.opengroup.osdu.wks.model.RelationshipConstants;
import org.opengroup.osdu.wks.model.RelationshipsBlock;
import org.opengroup.osdu.wks.model.SearchByAncestryQuery;
import org.opengroup.osdu.wks.model.SearchQuery;
import org.opengroup.osdu.wks.model.SearchRequestPayload;
import org.opengroup.osdu.wks.model.UpdatedRelationshipBlock;
import org.opengroup.osdu.wks.service.MappingService;
import org.opengroup.osdu.wks.service.SearchService;
import org.opengroup.osdu.wks.service.StorageService;
......@@ -45,22 +39,27 @@ public class RelationshipBlockHandler {
private String targetSchemaKindValue;
private MappingService mappingService;
private KindUtil kindUtil;
private TargetSchemaUtil targetSchemaUtil;
@Autowired
public RelationshipBlockHandler(
StorageService storageService,
SearchService searchService,
MappingService mappingService,
KindUtil kindUtil) {
KindUtil kindUtil,
TargetSchemaUtil targetSchemaUtil) {
this.storageService = storageService;
this.searchService = searchService;
this.mappingService = mappingService;
this.kindUtil = kindUtil;
this.targetSchemaUtil = targetSchemaUtil;
}
public UpdatedRelationshipBlock updateRelationshipBlockWithWksIds(Tree rawRelationshipsBlockTree, String targetSchemaKind) {
public UpdatedRelationshipBlock updateRelationshipBlockWithWksIds(
Tree rawRelationshipsBlockTree, String targetSchemaKind) {
targetSchemaKindValue = targetSchemaKind;
RelationshipsBlock rawRelationshipsBlock = new RelationshipsBlock(rawRelationshipsBlockTree);
List<Relationship> rawRelationships = rawRelationshipsBlock.getRelationships();
......@@ -69,8 +68,8 @@ public class RelationshipBlockHandler {
List<String> ids = relationship.getIds();
rawIds.addAll(ids);
}
List<Relationship> wksRelationships = fetchWksRelationships(rawRelationships);
List<Relationship> wksRelationships = fetchWksRelationships(rawRelationships);
Iterator<Relationship> relationshipIterator = rawRelationships.iterator();
Iterator<Relationship> wksRelationshipIterator = wksRelationships.iterator();
Set<String> pendingEntitiesToBeAdded = new HashSet<>();
......@@ -92,122 +91,132 @@ public class RelationshipBlockHandler {
return buildUpdatedRelationshipBlock(rawRelationshipsBlock, pendingEntitiesToBeAdded, pendingIdsToBeAdded);
}
private UpdatedRelationshipBlock buildUpdatedRelationshipBlock(RelationshipsBlock rawRelationshipsBlock,
Set<String> pendingEntitiesToBeAdded, List<String> pendingIdsToBeAdded) {
UpdatedRelationshipBlock updatedRelationshipBlock = new UpdatedRelationshipBlock();
private UpdatedRelationshipBlock buildUpdatedRelationshipBlock(
RelationshipsBlock rawRelationshipsBlock,
Set<String> pendingEntitiesToBeAdded,
List<String> pendingIdsToBeAdded) {
UpdatedRelationshipBlock updatedRelationshipBlock = new UpdatedRelationshipBlock();
updatedRelationshipBlock.setRelationshipTree(rawRelationshipsBlock.getRelationshipBlockAsTree());
updatedRelationshipBlock.setPendingIds(pendingIdsToBeAdded);
updatedRelationshipBlock.setPendingEnitites(pendingEntitiesToBeAdded);
return updatedRelationshipBlock;
}
private List<Relationship> fetchWksRelationships(List<Relationship> rawRelationships) {
List<Relationship> wksRelationships = new ArrayList<>();
for(Relationship rawRelationship : rawRelationships) {
Relationship wksRelationship = null;
try {
wksRelationship = getRelationshipBlockWithWksIds(rawRelationship);
} catch (ApplicationException e) {
LOGGER.info(e.getMessage(), e);
}
if(wksRelationship != null) {
wksRelationships.add(wksRelationship);
}
}
private List<Relationship> fetchWksRelationships(List<Relationship> rawRelationships) {
List<Relationship> wksRelationships = new ArrayList<>();
for (Relationship rawRelationship : rawRelationships) {
Relationship wksRelationship = null;
try {
wksRelationship = getRelationshipBlockWithWksIds(rawRelationship);
} catch (ApplicationException | NotFoundException e) {
LOGGER.info(e.getMessage(), e);
}
if (wksRelationship != null) {
wksRelationships.add(wksRelationship);
}
}
return wksRelationships;
}
private Relationship getRelationshipBlockWithWksIds(Relationship rawRelationship)
throws ApplicationException, NotFoundException {
List<String> rawIds = rawRelationship.getIds();
List<String> wksIds = getWksIds(rawIds);
rawRelationship.setIds(wksIds);
return rawRelationship;
}
private String getRelatedRecordIdUsingMapping(String relatedRecordId)
throws NotFoundException, ApplicationException {
Optional<String> rawRecordOption = storageService.getRecord(relatedRecordId);
if (rawRecordOption.isPresent()) {
Optional<String> relatedRecordKindOption = getRelatedRecordKind(rawRecordOption.get());
if (relatedRecordKindOption.isPresent()) {
Optional<String> relatedRecordTargetKind = getTargetKind(relatedRecordKindOption.get());
if (relatedRecordTargetKind.isPresent()) {
return WksIdGenerator.createRecordId(relatedRecordId, relatedRecordTargetKind.get());
}
}
}
return relatedRecordId;
}
private Optional<String> getRelatedRecordKind(String rawRecord) {
try {
String relatedRecordKind = new Tree(rawRecord).get("kind").asString();
return Optional.of(relatedRecordKind);
} catch (Exception e) {
LOGGER.warn("Error retrieving related record kind so keeping related record id");
}
return Optional.ofNullable(null);
}
private Optional<String> getTargetKind(String kind) throws NotFoundException, ApplicationException {
Optional<MappingInfo> mappingInfoOption = getLatestMappingInfoWithRelatedAuthority(kind);
if (mappingInfoOption.isPresent()) {
MappingInfo mappingInfo = mappingInfoOption.get();
SchemaIdentity targetSchemaIdentity = SchemaIdentity
.builder()
.authority(mappingInfo.getTargetSchemaAuthority())
.source(mappingInfo.getTargetSchemaSource())
.entityType(mappingInfo.getTargetEntityType())
.schemaVersionMajor(mappingInfo.getTargetSchemaMajorVersion())
.build();
return targetSchemaUtil.getTargetSchemaKind(targetSchemaIdentity);
} else {
return Optional.empty();
}
}
/**
* This method takes kind of related raw record. Using kind it finds all target
* mapping infos and then find related mapping infos using authority of
* container raw record. After this it returns Optional Mapping Info which has
* highest major version. Here we assume that mapping info version is not
* available only mapping major version is available.
*
* @param kind
* @return Optional<MappingInfo>
*/
private Optional<MappingInfo> getLatestMappingInfoWithRelatedAuthority(String kind) {
List<MappingInfo> mappingInfos = mappingService.getMappingInfos(kindUtil.retrieveAuthorityName(kind),
kindUtil.retrieveEntityName(kind),
kindUtil.retrieveSourceName(kind),
kindUtil.retrieveMajorVersion(kind));
if (mappingInfos.isEmpty()) {
return Optional.ofNullable(null);
}
String authority = kindUtil.retrieveAuthorityName(targetSchemaKindValue);
mappingInfos = mappingInfos
.stream()
.filter(mappingInfo -> mappingInfo.getTargetSchemaAuthority().equalsIgnoreCase(authority))
.collect(Collectors.toList());
return mappingInfos.stream().max(Comparator.comparing(MappingInfo::getMappingSchemaMajorVersion));
}
private List<String> getWksIds(List<String> relatedRawIds) throws NotFoundException, ApplicationException {
List<String> wksIds = new ArrayList<>();
for (String relatedRawId : relatedRawIds) {
Optional<String> wksIdOptional = getWksId(relatedRawId);
if (wksIdOptional.isPresent()) {
wksIds.add(wksIdOptional.get());
} else {
String relatedId = getRelatedRecordIdUsingMapping(relatedRawId);
wksIds.add(relatedId);
}
}
return wksRelationships;
}
private Relationship getRelationshipBlockWithWksIds(Relationship rawRelationship) throws ApplicationException {
List<String> rawIds = rawRelationship.getIds();
List<String> wksIds = getWksIds(rawIds);
rawRelationship.setIds(wksIds);
return rawRelationship;
}
private String getRelatedRecordIdUsingMapping(String relatedRecordId) {
Optional<String> rawRecordOption = storageService.getRecord(relatedRecordId);
if (rawRecordOption.isPresent()) {
Optional<String> relatedRecordKindOption = getRelatedRecordKind(rawRecordOption.get());
if (relatedRecordKindOption.isPresent()) {
Optional<String> relatedRecordTargetKind = getTargetKind(relatedRecordKindOption.get());
if (relatedRecordTargetKind.isPresent()) {
return WksIdGenerator.createRecordId(relatedRecordId, relatedRecordTargetKind.get());
}
}
}
return relatedRecordId;
}
private Optional<String> getRelatedRecordKind(String rawRecord) {
try {
String relatedRecordKind = new Tree(rawRecord).get("kind").asString();
return Optional.of(relatedRecordKind);
} catch (Exception e) {
LOGGER.warn("Error retrieving related record kind so keeping related record id");
}
return Optional.ofNullable(null);
}
private Optional<String> getTargetKind(String kind) {
Optional<MappingInfo> mappingInfoOption = getLatestMappingInfoWithRelatedAuthority(kind);
if(mappingInfoOption.isPresent()) {
MappingInfo mappingInfo = mappingInfoOption.get();
return Optional.of(kindUtil.prepareKind(mappingInfo.getTargetSchemaAuthority(),
mappingInfo.getTargetSchemaSource(), mappingInfo.getTargetEntityType(),
mappingInfo.getTargetSchemaMajorVersion(), "0", "0"));
}
else {
return Optional.ofNullable(null);
}
}
/**
* This method takes kind of related raw record. Using kind it finds all target
* mapping infos and then find related mapping infos using authority of
* container raw record. After this it returns Optional Mapping Info which has
* highest major version. Here we assume that mapping info version is not
* available only mapping major version is available.
*
* @param kind
* @return Optional<MappingInfo>
*/
private Optional<MappingInfo> getLatestMappingInfoWithRelatedAuthority(String kind) {
List<MappingInfo> mappingInfos = mappingService.getMappingInfos(kindUtil.retrieveAuthorityName(kind),
kindUtil.retrieveEntityName(kind), kindUtil.retrieveSourceName(kind),
kindUtil.retrieveMajorVersion(kind));
if (mappingInfos.isEmpty()) {
return Optional.ofNullable(null);
}
String authority = kindUtil.retrieveAuthorityName(targetSchemaKindValue);
mappingInfos = mappingInfos.stream()
.filter(mappingInfo -> mappingInfo.getTargetSchemaAuthority().equalsIgnoreCase(authority))
.collect(Collectors.toList());
return mappingInfos.stream().max(Comparator.comparing(MappingInfo::getMappingSchemaMajorVersion));
}
private List<String> getWksIds(List<String> relatedRawIds) {
List<String> wksIds = new ArrayList<>();
for(String relatedRawId : relatedRawIds) {
Optional<String> wksIdOptional = getWksId(relatedRawId);
if(wksIdOptional.isPresent()) {
wksIds.add(wksIdOptional.get());
}
else {
String relatedId = getRelatedRecordIdUsingMapping(relatedRawId);
wksIds.add(relatedId);
}
}
return wksIds;
}
private Optional<String> getWksId(String rawId) {
String searchResults = null;
// TODO Look out for performance improvement through caching
String searchResults = null;
// TODO Look out for performance improvement through caching
Optional<String> optionalRawRecord = storageService.getRecord(rawId);
if (!optionalRawRecord.isPresent()) {
return Optional.ofNullable(null);
......@@ -217,8 +226,8 @@ public class RelationshipBlockHandler {
try {
// TODO Look out for performance improvement through caching (FYI: If restrictions imposed by storage
// on authority name are lifted search results will be unique every time and caching won't be required
searchResults = searchService.query(searchRequestPayload.getJsonString());
} catch(ApplicationException e) {
searchResults = searchService.query(searchRequestPayload.getJsonString());
} catch (ApplicationException e) {
LOGGER.info("WKS not found from search service", e);
}
......@@ -245,26 +254,32 @@ public class RelationshipBlockHandler {
List<WksInfo> wksGeneratedByWKSService = getWKSRecordsCreatedByWKSService(rawId, wksInfos, rawKind);
LOGGER.info("Total {} Records created by WKS service", wksGeneratedByWKSService.size());
if(wksGeneratedByWKSService.isEmpty()) {
if (wksGeneratedByWKSService.isEmpty()) {
return Optional.ofNullable(null);
}
String targetSchemaAuthority = targetSchemaKindValue.split(Constants.COLON_SEPARATOR)[0];
List<WksInfo> wksFromGivenAuthority = getWKSRecordsFromGivenAuthority(targetSchemaAuthority, wksGeneratedByWKSService);