Skip to content
GitLab
Menu
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
Menu
Open sidebar
Open Subsurface Data Universe Software
Platform
Data Flow
Data Enrichment
wks
Commits
b9bb9637
Commit
b9bb9637
authored
Feb 06, 2021
by
harshit aggarwal
Browse files
updating UTs
parent
63501491
Pipeline
#25139
passed with stages
in 16 minutes and 56 seconds
Changes
3
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
wks-core/src/main/java/org/opengroup/osdu/wks/service/StorageServiceImpl.java
View file @
b9bb9637
package
org.opengroup.osdu.wks.service
;
package
org.opengroup.osdu.wks.service
;
import
java.lang.reflect.Type
;
import
java.lang.reflect.Type
;
import
java.util.ArrayList
;
import
java.util.Arrays
;
import
java.util.Arrays
;
import
java.util.List
;
import
java.util.List
;
import
java.util.Optional
;
import
java.util.Optional
;
import
com.fasterxml.jackson.core.JsonProcessingException
;
import
com.fasterxml.jackson.databind.ObjectMapper
;
import
com.google.common.base.Strings
;
import
com.google.common.base.Strings
;
import
com.google.common.reflect.TypeToken
;
import
com.google.common.reflect.TypeToken
;
import
com.google.gson.Gson
;
import
com.google.gson.Gson
;
import
io.datatree.Tree
;
import
com.google.gson.JsonSyntaxException
;
import
org.apache.http.HttpStatus
;
import
org.apache.http.HttpStatus
;
import
org.opengroup.osdu.core.common.model.http.AppException
;
import
org.opengroup.osdu.core.common.model.http.AppException
;
import
org.opengroup.osdu.core.common.model.indexer.Records
;
import
org.opengroup.osdu.core.common.model.indexer.Records
;
...
@@ -25,8 +22,6 @@ import org.opengroup.osdu.wks.exceptions.ApplicationException;
...
@@ -25,8 +22,6 @@ import org.opengroup.osdu.wks.exceptions.ApplicationException;
import
org.opengroup.osdu.wks.exceptions.AuthorizationException
;
import
org.opengroup.osdu.wks.exceptions.AuthorizationException
;
import
org.opengroup.osdu.wks.model.RequestParameters
;
import
org.opengroup.osdu.wks.model.RequestParameters
;
import
org.opengroup.osdu.wks.model.RestResponse
;
import
org.opengroup.osdu.wks.model.RestResponse
;
import
org.opengroup.osdu.wks.model.SchemaInfo
;
import
org.opengroup.osdu.wks.model.SchemaInfoResponse
;
import
org.opengroup.osdu.wks.provider.interfaces.UserCredential
;
import
org.opengroup.osdu.wks.provider.interfaces.UserCredential
;
import
org.opengroup.osdu.wks.util.RestClient
;
import
org.opengroup.osdu.wks.util.RestClient
;
import
org.slf4j.Logger
;
import
org.slf4j.Logger
;
...
@@ -116,7 +111,7 @@ public class StorageServiceImpl implements StorageService {
...
@@ -116,7 +111,7 @@ public class StorageServiceImpl implements StorageService {
verifyResponseStatus
(
response
,
HttpStatus
.
SC_OK
);
verifyResponseStatus
(
response
,
HttpStatus
.
SC_OK
);
return
extractResponseBody
(
response
);
return
extractResponseBody
FromQueryRecordsApi
(
response
);
}
}
...
@@ -169,7 +164,7 @@ private String getAuthKey(TenantInfo tenantInfo) {
...
@@ -169,7 +164,7 @@ private String getAuthKey(TenantInfo tenantInfo) {
}
}
}
}
private
Records
extractResponseBody
(
RestResponse
response
)
throws
Application
Exception
{
private
Records
extractResponseBody
FromQueryRecordsApi
(
RestResponse
response
)
throws
JsonSyntax
Exception
{
String
bulkStorageData
=
response
.
getMessage
();
String
bulkStorageData
=
response
.
getMessage
();
...
...
wks-core/src/main/java/org/opengroup/osdu/wks/service/WKSServiceImpl.java
View file @
b9bb9637
...
@@ -6,7 +6,6 @@ import java.util.Arrays;
...
@@ -6,7 +6,6 @@ import java.util.Arrays;
import
java.util.HashMap
;
import
java.util.HashMap
;
import
java.util.List
;
import
java.util.List
;
import
java.util.Map
;
import
java.util.Map
;
import
java.util.Optional
;
import
com.google.gson.Gson
;
import
com.google.gson.Gson
;
import
org.apache.commons.lang3.ArrayUtils
;
import
org.apache.commons.lang3.ArrayUtils
;
...
@@ -211,12 +210,15 @@ public class WKSServiceImpl implements WKSService {
...
@@ -211,12 +210,15 @@ public class WKSServiceImpl implements WKSService {
ids
.
add
(
rawRecordDetail
.
getId
());
ids
.
add
(
rawRecordDetail
.
getId
());
}
}
Records
records
=
storageService
.
getRecordBatch
(
ids
);
try
{
Records
records
=
storageService
.
getRecordBatch
(
ids
);
for
(
Records
.
Entity
rawRecord
:
records
.
getRecords
())
{
for
(
Records
.
Entity
rawRecord
:
records
.
getRecords
())
{
rawRecords
.
put
(
rawRecord
.
getId
(),
convertJsonStringToTree
(
this
.
gson
.
toJson
(
rawRecord
)));
rawRecords
.
put
(
rawRecord
.
getId
(),
convertJsonStringToTree
(
this
.
gson
.
toJson
(
rawRecord
)));
}
}
catch
(
Exception
e
)
{
LOGGER
.
warn
(
"Failed to fetch a batch of raw records with ids {} from storage"
,
ids
,
e
);
}
}
}
}
return
rawRecords
;
return
rawRecords
;
...
@@ -251,24 +253,6 @@ public class WKSServiceImpl implements WKSService {
...
@@ -251,24 +253,6 @@ public class WKSServiceImpl implements WKSService {
LOGGER
.
info
(
message
);
LOGGER
.
info
(
message
);
}
}
// private Tree retrieveRawRecordTree(RawRecordDetails rawRecord)
// throws ApplicationException, BadRequestException {
//
// Optional<String> optionalRawRecord = storageService.getRecord(rawRecord.getId());
// if (!optionalRawRecord.isPresent()) {
// LOGGER.warn(Constants.RAW_RECORD_NOT_PRESENT);
// throw new BadRequestException(Constants.RAW_RECORD_NOT_PRESENT);
// }
//
// String rawRecordJsonStr = optionalRawRecord.get();
// Tree rawRecordTree = convertJsonStringToTree(rawRecordJsonStr);
// if (rawRecordTree.isEmpty()) {
// LOGGER.warn(Constants.RAW_RECORD_NOT_PRESENT);
// throw new ApplicationException(Constants.RAW_RECORD_NOT_PRESENT);
// }
// return rawRecordTree;
// }
private
boolean
isEmptyMappings
(
List
<
MappingsModel
>
mappings
)
{
private
boolean
isEmptyMappings
(
List
<
MappingsModel
>
mappings
)
{
return
mappings
==
null
||
mappings
.
isEmpty
();
return
mappings
==
null
||
mappings
.
isEmpty
();
}
}
...
...
wks-core/src/test/java/org/opengroup/osdu/wks/service/WKSServiceImplTest.java
View file @
b9bb9637
...
@@ -6,7 +6,9 @@ import static org.mockito.Mockito.times;
...
@@ -6,7 +6,9 @@ import static org.mockito.Mockito.times;
import
static
org
.
mockito
.
Mockito
.
verify
;
import
static
org
.
mockito
.
Mockito
.
verify
;
import
static
org
.
mockito
.
Mockito
.
when
;
import
static
org
.
mockito
.
Mockito
.
when
;
import
java.util.ArrayList
;
import
java.util.Collections
;
import
java.util.Collections
;
import
java.util.HashMap
;
import
java.util.List
;
import
java.util.List
;
import
java.util.Optional
;
import
java.util.Optional
;
...
@@ -19,6 +21,10 @@ import org.mockito.InjectMocks;
...
@@ -19,6 +21,10 @@ import org.mockito.InjectMocks;
import
org.mockito.Mock
;
import
org.mockito.Mock
;
import
org.mockito.Mockito
;
import
org.mockito.Mockito
;
import
org.mockito.MockitoAnnotations
;
import
org.mockito.MockitoAnnotations
;
import
org.opengroup.osdu.core.common.model.entitlements.Acl
;
import
org.opengroup.osdu.core.common.model.indexer.Records
;
import
org.opengroup.osdu.core.common.model.legal.Legal
;
import
org.opengroup.osdu.core.common.model.storage.RecordAncestry
;
import
org.opengroup.osdu.wks.config.RequestIdentity
;
import
org.opengroup.osdu.wks.config.RequestIdentity
;
import
org.opengroup.osdu.wks.constants.Constants
;
import
org.opengroup.osdu.wks.constants.Constants
;
import
org.opengroup.osdu.wks.exceptions.ApplicationException
;
import
org.opengroup.osdu.wks.exceptions.ApplicationException
;
...
@@ -36,6 +42,9 @@ import io.datatree.Tree;
...
@@ -36,6 +42,9 @@ import io.datatree.Tree;
@RunWith
(
SpringJUnit4ClassRunner
.
class
)
@RunWith
(
SpringJUnit4ClassRunner
.
class
)
public
class
WKSServiceImplTest
{
public
class
WKSServiceImplTest
{
private
static
final
String
ID
=
"datapartition1:source:entity-123"
;
private
static
final
long
VERSION
=
1000
;
private
static
final
String
MOCK_SCHEMA_KIND
=
"datapartition1:source:entity:1.0.0"
;
private
static
final
String
MOCK_SCHEMA_KIND
=
"datapartition1:source:entity:1.0.0"
;
private
static
final
String
JSON_SAMPLE_STR
=
"{\n"
+
" \"message\": {\n"
+
" \"data\": [\n"
+
" {\n"
private
static
final
String
JSON_SAMPLE_STR
=
"{\n"
+
" \"message\": {\n"
+
" \"data\": [\n"
+
" {\n"
+
" \"id\": \"datapartition1:source:enitity87e2f18b2eaa40d9876330161fda628e\",\n"
+
" \"id\": \"datapartition1:source:enitity87e2f18b2eaa40d9876330161fda628e\",\n"
...
@@ -140,15 +149,19 @@ public class WKSServiceImplTest {
...
@@ -140,15 +149,19 @@ public class WKSServiceImplTest {
TransformationResult
transformationResult
=
new
TransformationResult
();
TransformationResult
transformationResult
=
new
TransformationResult
();
transformationResult
.
setTransformedTree
(
new
Tree
(
wks
));
transformationResult
.
setTransformedTree
(
new
Tree
(
wks
));
Records
records
=
new
Records
();
records
.
setRecords
(
Collections
.
singletonList
(
new
Records
.
Entity
(
ID
,
VERSION
,
MOCK_SCHEMA_KIND
,
new
Acl
(),
new
Legal
(),
new
RecordAncestry
(),
new
HashMap
<>(),
new
ArrayList
<>())));
when
(
mappingService
.
getMapping
(
mockData1
.
getKind
())).
thenReturn
(
mappings
);
when
(
mappingService
.
getMapping
(
mockData1
.
getKind
())).
thenReturn
(
mappings
);
when
(
storageService
.
getRecord
(
anyString
())).
thenReturn
(
Optional
.
of
(
JSON_SAMPLE_STR
)
);
when
(
storageService
.
getRecord
Batch
(
any
())).
thenReturn
(
records
);
when
(
wksTransformator
.
transform
(
any
())).
thenReturn
(
transformationResult
);
when
(
wksTransformator
.
transform
(
any
())).
thenReturn
(
transformationResult
);
when
(
schemaService
.
getSchemas
(
MOCK_SCHEMA_KIND
)).
thenReturn
(
Collections
.
singletonList
(
MOCK_SCHEMA_KIND
));
when
(
schemaService
.
getSchemas
(
MOCK_SCHEMA_KIND
)).
thenReturn
(
Collections
.
singletonList
(
MOCK_SCHEMA_KIND
));
wksService
.
transform
(
mockDataset
,
dataPartitionId
,
correlationId
);
wksService
.
transform
(
mockDataset
,
dataPartitionId
,
correlationId
);
verify
(
mappingService
,
times
(
1
)).
getMapping
(
anyString
());
verify
(
mappingService
,
times
(
1
)).
getMapping
(
anyString
());
verify
(
storageService
,
times
(
1
)).
getRecord
(
anyString
());
verify
(
storageService
,
times
(
1
)).
getRecord
Batch
(
any
());
verify
(
storageService
,
times
(
1
)).
putRecords
(
any
(),
anyString
(),
anyString
());
verify
(
storageService
,
times
(
1
)).
putRecords
(
any
(),
anyString
(),
anyString
());
}
}
...
@@ -169,15 +182,18 @@ public class WKSServiceImplTest {
...
@@ -169,15 +182,18 @@ public class WKSServiceImplTest {
TransformationResult
transformationResult
=
new
TransformationResult
();
TransformationResult
transformationResult
=
new
TransformationResult
();
transformationResult
.
setTransformedTree
(
new
Tree
(
wks
));
transformationResult
.
setTransformedTree
(
new
Tree
(
wks
));
Records
records
=
new
Records
();
records
.
setRecords
(
Collections
.
singletonList
(
new
Records
.
Entity
(
ID
,
VERSION
,
MOCK_SCHEMA_KIND
,
new
Acl
(),
new
Legal
(),
new
RecordAncestry
(),
new
HashMap
<>(),
new
ArrayList
<>())));
when
(
schemaService
.
getSchemas
(
MOCK_SCHEMA_KIND
)).
thenReturn
(
Collections
.
singletonList
(
MOCK_SCHEMA_KIND
));
when
(
schemaService
.
getSchemas
(
MOCK_SCHEMA_KIND
)).
thenReturn
(
Collections
.
singletonList
(
MOCK_SCHEMA_KIND
));
when
(
mappingService
.
getMapping
(
mockData1
.
getKind
())).
thenReturn
(
mappings
);
when
(
mappingService
.
getMapping
(
mockData1
.
getKind
())).
thenReturn
(
mappings
);
when
(
storageService
.
getRecord
(
anyString
())).
thenReturn
(
Optional
.
of
(
JSON_SAMPLE_STR
)
);
when
(
storageService
.
getRecord
Batch
(
any
())).
thenReturn
(
records
);
when
(
wksTransformator
.
transform
(
any
())).
thenReturn
(
transformationResult
);
when
(
wksTransformator
.
transform
(
any
())).
thenReturn
(
transformationResult
);
wksService
.
transform
(
mockDataset
,
dataPartitionId
,
correlationId
);
wksService
.
transform
(
mockDataset
,
dataPartitionId
,
correlationId
);
verify
(
mappingService
,
times
(
1
)).
getMapping
(
anyString
());
verify
(
mappingService
,
times
(
1
)).
getMapping
(
anyString
());
verify
(
storageService
,
times
(
1
)).
getRecord
(
anyString
());
verify
(
storageService
,
times
(
1
)).
getRecord
Batch
(
any
());
verify
(
storageService
,
times
(
1
)).
putRecords
(
any
(),
anyString
(),
anyString
());
verify
(
storageService
,
times
(
1
)).
putRecords
(
any
(),
anyString
(),
anyString
());
}
}
...
@@ -219,14 +235,18 @@ public class WKSServiceImplTest {
...
@@ -219,14 +235,18 @@ public class WKSServiceImplTest {
mockDataset
[
0
]
=
mockData
;
mockDataset
[
0
]
=
mockData
;
MappingsModel
mappings
=
createMappings
();
MappingsModel
mappings
=
createMappings
();
String
id
=
"datapartition1:source:enitity-uniqueId"
;
Records
records
=
new
Records
();
records
.
setRecords
(
Collections
.
singletonList
(
new
Records
.
Entity
(
id
,
VERSION
,
MOCK_SCHEMA_KIND
,
new
Acl
(),
new
Legal
(),
new
RecordAncestry
(),
new
HashMap
<>(),
new
ArrayList
<>())));
when
(
schemaService
.
getSchemas
(
MOCK_SCHEMA_KIND
)).
thenReturn
(
Collections
.
singletonList
(
MOCK_SCHEMA_KIND
));
when
(
schemaService
.
getSchemas
(
MOCK_SCHEMA_KIND
)).
thenReturn
(
Collections
.
singletonList
(
MOCK_SCHEMA_KIND
));
when
(
mappingService
.
getMapping
(
mockData
.
getKind
())).
thenReturn
(
Collections
.
singletonList
(
mappings
));
when
(
mappingService
.
getMapping
(
mockData
.
getKind
())).
thenReturn
(
Collections
.
singletonList
(
mappings
));
when
(
storageService
.
getRecord
(
anyString
())).
thenReturn
(
Optional
.
of
(
JSON_SAMPLE_STR
)
);
when
(
storageService
.
getRecord
Batch
(
any
())).
thenReturn
(
records
);
when
(
wksTransformator
.
transform
(
any
(
TransformRequest
.
class
))).
thenReturn
(
createTransformationResult
(
wks
));
when
(
wksTransformator
.
transform
(
any
(
TransformRequest
.
class
))).
thenReturn
(
createTransformationResult
(
wks
));
wksService
.
transform
(
getMockDataCorrect
(
mockDataset
),
dataPartitionId
,
correlationId
);
wksService
.
transform
(
getMockDataCorrect
(
mockDataset
),
dataPartitionId
,
correlationId
);
verify
(
mappingService
,
times
(
1
)).
getMapping
(
anyString
());
verify
(
mappingService
,
times
(
1
)).
getMapping
(
anyString
());
verify
(
storageService
,
times
(
1
)).
getRecord
(
anyString
());
verify
(
storageService
,
times
(
1
)).
getRecord
Batch
(
any
());
verify
(
storageService
,
times
(
1
)).
putRecords
(
any
(),
anyString
(),
anyString
());
verify
(
storageService
,
times
(
1
)).
putRecords
(
any
(),
anyString
(),
anyString
());
}
}
...
@@ -237,14 +257,17 @@ public class WKSServiceImplTest {
...
@@ -237,14 +257,17 @@ public class WKSServiceImplTest {
mockDataset
[
0
]
=
mockData
;
mockDataset
[
0
]
=
mockData
;
MappingsModel
mappings
=
createMappings
();
MappingsModel
mappings
=
createMappings
();
Records
records
=
new
Records
();
records
.
setRecords
(
Collections
.
singletonList
(
new
Records
.
Entity
(
"some-id"
,
VERSION
,
MOCK_SCHEMA_KIND
,
new
Acl
(),
new
Legal
(),
new
RecordAncestry
(),
new
HashMap
<>(),
new
ArrayList
<>())));
when
(
schemaService
.
getSchemas
(
MOCK_SCHEMA_KIND
)).
thenReturn
(
Collections
.
singletonList
(
MOCK_SCHEMA_KIND
));
when
(
schemaService
.
getSchemas
(
MOCK_SCHEMA_KIND
)).
thenReturn
(
Collections
.
singletonList
(
MOCK_SCHEMA_KIND
));
when
(
mappingService
.
getMapping
(
Mockito
.
any
())).
thenReturn
(
Collections
.
singletonList
(
mappings
));
when
(
mappingService
.
getMapping
(
Mockito
.
any
())).
thenReturn
(
Collections
.
singletonList
(
mappings
));
when
(
storageService
.
getRecord
(
anyString
())).
thenReturn
(
Optional
.
of
(
"Invalid Json"
)
);
when
(
storageService
.
getRecord
Batch
(
any
())).
thenReturn
(
records
);
when
(
wksTransformator
.
transform
(
any
(
TransformRequest
.
class
))).
thenReturn
(
createTransformationResult
(
JSON_SAMPLE_STR
));
when
(
wksTransformator
.
transform
(
any
(
TransformRequest
.
class
))).
thenReturn
(
createTransformationResult
(
JSON_SAMPLE_STR
));
wksService
.
transform
(
getMockDataCorrect
(
mockDataset
),
dataPartitionId
,
correlationId
);
wksService
.
transform
(
getMockDataCorrect
(
mockDataset
),
dataPartitionId
,
correlationId
);
verify
(
mappingService
,
times
(
1
)).
getMapping
(
anyString
());
verify
(
mappingService
,
times
(
1
)).
getMapping
(
anyString
());
verify
(
storageService
,
times
(
1
)).
getRecord
(
anyString
());
verify
(
storageService
,
times
(
1
)).
getRecord
Batch
(
any
());
verify
(
storageService
,
times
(
0
)).
putRecords
(
any
(),
anyString
(),
anyString
());
verify
(
storageService
,
times
(
0
)).
putRecords
(
any
(),
anyString
(),
anyString
());
}
}
...
@@ -256,14 +279,18 @@ public class WKSServiceImplTest {
...
@@ -256,14 +279,18 @@ public class WKSServiceImplTest {
mockDataset
[
0
]
=
mockData
;
mockDataset
[
0
]
=
mockData
;
MappingsModel
mappings
=
createMappings
();
MappingsModel
mappings
=
createMappings
();
Records
records
=
new
Records
();
String
id
=
"datapartition1:source:enitity-uniqueId"
;
records
.
setRecords
(
Collections
.
singletonList
(
new
Records
.
Entity
(
id
,
VERSION
,
MOCK_SCHEMA_KIND
,
new
Acl
(),
new
Legal
(),
new
RecordAncestry
(),
new
HashMap
<>(),
new
ArrayList
<>())));
when
(
schemaService
.
getSchemas
(
MOCK_SCHEMA_KIND
)).
thenReturn
(
Collections
.
singletonList
(
MOCK_SCHEMA_KIND
));
when
(
schemaService
.
getSchemas
(
MOCK_SCHEMA_KIND
)).
thenReturn
(
Collections
.
singletonList
(
MOCK_SCHEMA_KIND
));
when
(
mappingService
.
getMapping
(
mockData
.
getKind
())).
thenReturn
(
Collections
.
singletonList
(
mappings
));
when
(
mappingService
.
getMapping
(
mockData
.
getKind
())).
thenReturn
(
Collections
.
singletonList
(
mappings
));
when
(
storageService
.
getRecord
(
anyString
())).
thenReturn
(
Optional
.
of
(
JSON_SAMPLE_STR
)
);
when
(
storageService
.
getRecord
Batch
(
any
())).
thenReturn
(
records
);
when
(
wksTransformator
.
transform
(
any
(
TransformRequest
.
class
))).
thenReturn
(
createTransformationResult
(
wks
));
when
(
wksTransformator
.
transform
(
any
(
TransformRequest
.
class
))).
thenReturn
(
createTransformationResult
(
wks
));
wksService
.
transform
(
getMockDataCorrectOpUpdate
(
mockDataset
),
dataPartitionId
,
correlationId
);
wksService
.
transform
(
getMockDataCorrectOpUpdate
(
mockDataset
),
dataPartitionId
,
correlationId
);
verify
(
mappingService
,
times
(
1
)).
getMapping
(
anyString
());
verify
(
mappingService
,
times
(
1
)).
getMapping
(
anyString
());
verify
(
storageService
,
times
(
1
)).
getRecord
(
anyString
());
verify
(
storageService
,
times
(
1
)).
getRecord
Batch
(
any
());
verify
(
storageService
,
times
(
1
)).
putRecords
(
any
(),
anyString
(),
anyString
());
verify
(
storageService
,
times
(
1
)).
putRecords
(
any
(),
anyString
(),
anyString
());
}
}
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment