Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
Menu
Open sidebar
Open Subsurface Data Universe Software
Platform
System
Indexer
Commits
3e2d0a6b
Commit
3e2d0a6b
authored
Dec 22, 2020
by
Sviatoslav Nekhaienko
Browse files
enable retrieval from the schema service for all cloud providers
parent
1a56229b
Pipeline
#19962
failed with stages
in 21 minutes and 1 second
Changes
5
Pipelines
4
Hide whitespace changes
Inline
Side-by-side
indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
View file @
3e2d0a6b
...
...
@@ -22,6 +22,7 @@ import org.elasticsearch.client.RestHighLevelClient;
import
org.opengroup.osdu.core.common.model.storage.SchemaItem
;
import
org.opengroup.osdu.core.common.model.http.AppException
;
import
org.opengroup.osdu.core.common.logging.JaxRsDpsLog
;
import
org.opengroup.osdu.indexer.service.impl.SchemaProviderImpl
;
import
org.opengroup.osdu.indexer.util.ElasticClientHandler
;
import
org.opengroup.osdu.core.common.search.ElasticIndexNameResolver
;
import
org.opengroup.osdu.core.common.search.IndicesService
;
...
...
@@ -54,7 +55,7 @@ public class IndexSchemaServiceImpl implements IndexSchemaService {
@Inject
private
JaxRsDpsLog
log
;
@Inject
private
S
torageService
storageService
;
private
S
chemaProviderImpl
schemaProvider
;
@Inject
private
ElasticClientHandler
elasticClientHandler
;
@Inject
...
...
@@ -162,7 +163,7 @@ public class IndexSchemaServiceImpl implements IndexSchemaService {
}
protected
String
getSchema
(
String
kind
)
throws
URISyntaxException
,
UnsupportedEncodingException
{
return
this
.
s
torageService
.
getStorage
Schema
(
kind
);
return
this
.
s
chemaProvider
.
get
Schema
(
kind
);
}
public
void
syncIndexMappingWithStorageSchema
(
String
kind
)
throws
ElasticsearchException
,
IOException
,
AppException
{
...
...
indexer-core/src/main/java/org/opengroup/osdu/indexer/service/impl/Schema
Service
Impl.java
→
indexer-core/src/main/java/org/opengroup/osdu/indexer/service/impl/Schema
Provider
Impl.java
View file @
3e2d0a6b
...
...
@@ -15,7 +15,6 @@
package
org.opengroup.osdu.indexer.service.impl
;
import
com.google.api.client.http.HttpMethods
;
import
lombok.extern.java.Log
;
import
org.apache.http.HttpStatus
;
import
org.opengroup.osdu.core.common.http.FetchServiceHttpRequest
;
import
org.opengroup.osdu.core.common.http.IUrlFetchService
;
...
...
@@ -26,7 +25,6 @@ import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
import
org.opengroup.osdu.indexer.schema.converter.interfaces.SchemaToStorageFormat
;
import
org.opengroup.osdu.indexer.service.SchemaService
;
import
org.opengroup.osdu.indexer.service.StorageService
;
import
org.springframework.beans.factory.annotation.Value
;
import
org.springframework.context.annotation.Lazy
;
import
org.springframework.stereotype.Component
;
...
...
@@ -41,7 +39,7 @@ import java.nio.charset.StandardCharsets;
*/
@Component
@Lazy
public
class
Schema
Service
Impl
implements
SchemaService
{
public
class
Schema
Provider
Impl
implements
SchemaService
{
@Inject
private
JaxRsDpsLog
log
;
...
...
indexer-core/src/test/java/org/opengroup/osdu/indexer/service/impl/Schema
Service
ImplTest.java
→
indexer-core/src/test/java/org/opengroup/osdu/indexer/service/impl/Schema
Provider
ImplTest.java
View file @
3e2d0a6b
...
...
@@ -27,7 +27,7 @@ import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
import
org.opengroup.osdu.indexer.config.IndexerConfigurationProperties
;
import
org.opengroup.osdu.indexer.schema.converter.SchemaToStorageFormatImpl
;
import
org.opengroup.osdu.indexer.service.StorageService
;
import
org.opengroup.osdu.indexer.service.impl.Schema
Service
Impl
;
import
org.opengroup.osdu.indexer.service.impl.Schema
Provider
Impl
;
import
org.springframework.http.converter.json.Jackson2ObjectMapperBuilder
;
import
org.springframework.test.context.junit4.SpringRunner
;
...
...
@@ -39,7 +39,7 @@ import static org.mockito.Mockito.mock;
import
static
org
.
mockito
.
Mockito
.
when
;
@RunWith
(
SpringRunner
.
class
)
public
class
Schema
Service
ImplTest
{
public
class
Schema
Provider
ImplTest
{
private
ObjectMapper
objectMapper
=
Jackson2ObjectMapperBuilder
.
json
().
build
();
private
JaxRsDpsLog
jaxRsDpsLog
=
Mockito
.
mock
(
JaxRsDpsLog
.
class
);
...
...
@@ -60,7 +60,7 @@ public class SchemaServiceImplTest {
private
IndexerConfigurationProperties
configurationProperties
;
@InjectMocks
private
Schema
Service
Impl
sut
;
private
Schema
Provider
Impl
sut
;
@Test
public
void
test_empty_schema
()
throws
UnsupportedEncodingException
,
URISyntaxException
{
...
...
provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/Schema
Service
Test.java
→
provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/Schema
Provider
Test.java
View file @
3e2d0a6b
...
...
@@ -27,12 +27,11 @@ import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
import
org.opengroup.osdu.indexer.config.IndexerConfigurationProperties
;
import
org.opengroup.osdu.indexer.schema.converter.SchemaToStorageFormatImpl
;
import
org.opengroup.osdu.indexer.service.StorageService
;
import
org.opengroup.osdu.indexer.service.impl.Schema
Service
Impl
;
import
org.opengroup.osdu.indexer.service.impl.Schema
Provider
Impl
;
import
org.springframework.http.HttpStatus
;
import
org.springframework.http.converter.json.Jackson2ObjectMapperBuilder
;
import
org.springframework.test.context.junit4.SpringRunner
;
import
javax.inject.Inject
;
import
java.util.HashMap
;
import
java.util.Map
;
...
...
@@ -42,7 +41,7 @@ import static org.mockito.Mockito.*;
import
static
org
.
powermock
.
api
.
mockito
.
PowerMockito
.
when
;
@RunWith
(
SpringRunner
.
class
)
public
class
Schema
Service
Test
{
public
class
Schema
Provider
Test
{
private
ObjectMapper
objectMapper
=
Jackson2ObjectMapperBuilder
.
json
().
build
();
private
JaxRsDpsLog
log
=
Mockito
.
mock
(
JaxRsDpsLog
.
class
);
...
...
@@ -59,7 +58,7 @@ public class SchemaServiceTest {
private
IndexerConfigurationProperties
configurationProperties
;
@InjectMocks
private
Schema
Service
Impl
sut
;
private
Schema
Provider
Impl
sut
;
@Before
public
void
setup
()
{
...
...
@@ -124,7 +123,7 @@ public class SchemaServiceTest {
public
void
should_call_Storage_then_Schema
()
throws
Exception
{
String
kind
=
"tenant:test:test:1.0.0"
;
Schema
Service
Impl
schemaService
=
Mockito
.
mock
(
Schema
Service
Impl
.
class
);
Schema
Provider
Impl
schemaService
=
Mockito
.
mock
(
Schema
Provider
Impl
.
class
);
when
(
schemaService
.
getSchema
(
any
())).
thenCallRealMethod
();
InOrder
inOrder
=
inOrder
(
schemaService
);
...
...
@@ -143,7 +142,7 @@ public class SchemaServiceTest {
public
void
should_call_only_Storage_if_it_returns_result
()
throws
Exception
{
String
kind
=
"tenant:test:test:1.0.0"
;
Schema
Service
Impl
schemaService
=
Mockito
.
mock
(
Schema
Service
Impl
.
class
);
Schema
Provider
Impl
schemaService
=
Mockito
.
mock
(
Schema
Provider
Impl
.
class
);
when
(
schemaService
.
getSchema
(
any
())).
thenCallRealMethod
();
String
someSchema
=
"some schema"
;
when
(
schemaService
.
getFromStorageService
(
any
())).
thenReturn
(
someSchema
);
...
...
provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java
View file @
3e2d0a6b
...
...
@@ -29,6 +29,7 @@ import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache;
import
org.opengroup.osdu.core.common.model.indexer.IndexSchema
;
import
org.opengroup.osdu.core.common.model.http.RequestStatus
;
import
org.opengroup.osdu.core.common.search.IndicesService
;
import
org.opengroup.osdu.indexer.service.impl.SchemaProviderImpl
;
import
org.opengroup.osdu.indexer.util.ElasticClientHandler
;
import
org.opengroup.osdu.core.common.search.ElasticIndexNameResolver
;
import
org.powermock.core.classloader.annotations.PrepareForTest
;
...
...
@@ -56,7 +57,7 @@ public class IndexerSchemaServiceTest {
@Mock
private
JaxRsDpsLog
log
;
@Mock
private
S
torageService
storageService
;
private
S
chemaProviderImpl
schemaProvider
;
@Mock
private
ElasticClientHandler
elasticClientHandler
;
@Mock
...
...
@@ -79,7 +80,7 @@ public class IndexerSchemaServiceTest {
@Test
public
void
should_returnNull_givenEmptySchema_getIndexerInputSchemaSchemaTest
()
throws
Exception
{
when
(
s
torageService
.
getStorage
Schema
(
any
())).
thenReturn
(
emptySchema
);
when
(
s
chemaProvider
.
get
Schema
(
any
())).
thenReturn
(
emptySchema
);
IndexSchema
indexSchema
=
this
.
sut
.
getIndexerInputSchema
(
kind
,
false
);
...
...
@@ -88,7 +89,7 @@ public class IndexerSchemaServiceTest {
@Test
public
void
should_returnValidResponse_givenValidSchema_getIndexerInputSchemaTest
()
throws
Exception
{
when
(
s
torageService
.
getStorage
Schema
(
any
())).
thenReturn
(
someSchema
);
when
(
s
chemaProvider
.
get
Schema
(
any
())).
thenReturn
(
someSchema
);
IndexSchema
indexSchema
=
this
.
sut
.
getIndexerInputSchema
(
kind
,
false
);
...
...
@@ -97,7 +98,7 @@ public class IndexerSchemaServiceTest {
@Test
public
void
should_returnValidResponse_givenValidSchemaWithCacheHit_getIndexerInputSchemaTest
()
throws
Exception
{
when
(
s
torageService
.
getStorage
Schema
(
any
())).
thenReturn
(
someSchema
);
when
(
s
chemaProvider
.
get
Schema
(
any
())).
thenReturn
(
someSchema
);
when
(
this
.
schemaCache
.
get
(
kind
+
"_flattened"
)).
thenReturn
(
someSchema
);
IndexSchema
indexSchema
=
this
.
sut
.
getIndexerInputSchema
(
kind
,
false
);
...
...
@@ -109,7 +110,7 @@ public class IndexerSchemaServiceTest {
public
void
should_throw500_givenInvalidSchemaCacheHit_getIndexerInputSchemaTest
()
{
try
{
String
invalidSchema
=
"{}}"
;
when
(
s
torageService
.
getStorage
Schema
(
any
())).
thenReturn
(
invalidSchema
);
when
(
s
chemaProvider
.
get
Schema
(
any
())).
thenReturn
(
invalidSchema
);
this
.
sut
.
getIndexerInputSchema
(
kind
,
false
);
fail
(
"Should throw exception"
);
...
...
@@ -164,7 +165,7 @@ public class IndexerSchemaServiceTest {
when
(
this
.
elasticIndexNameResolver
.
getIndexNameFromKind
(
kind
)).
thenReturn
(
kind
.
replace
(
":"
,
"-"
));
when
(
this
.
schemaCache
.
get
(
kind
)).
thenReturn
(
null
);
when
(
this
.
indicesService
.
isIndexExist
(
any
(),
any
())).
thenReturn
(
false
);
when
(
this
.
s
torageService
.
getStorage
Schema
(
kind
)).
thenReturn
(
storageSchema
);
when
(
this
.
s
chemaProvider
.
get
Schema
(
kind
)).
thenReturn
(
storageSchema
);
this
.
sut
.
processSchemaMessages
(
schemaMessages
);
...
...
@@ -195,7 +196,7 @@ public class IndexerSchemaServiceTest {
when
(
this
.
elasticIndexNameResolver
.
getIndexNameFromKind
(
kind
)).
thenReturn
(
kind
.
replace
(
":"
,
"-"
));
when
(
this
.
schemaCache
.
get
(
kind
)).
thenReturn
(
null
);
when
(
this
.
indicesService
.
isIndexExist
(
any
(),
any
())).
thenReturn
(
true
);
when
(
this
.
s
torageService
.
getStorage
Schema
(
kind
)).
thenReturn
(
storageSchema
);
when
(
this
.
s
chemaProvider
.
get
Schema
(
kind
)).
thenReturn
(
storageSchema
);
this
.
sut
.
processSchemaMessages
(
schemaMessages
);
...
...
@@ -223,7 +224,7 @@ public class IndexerSchemaServiceTest {
when
(
this
.
elasticIndexNameResolver
.
getIndexNameFromKind
(
kind
)).
thenReturn
(
kind
.
replace
(
":"
,
"-"
));
when
(
this
.
schemaCache
.
get
(
kind
)).
thenReturn
(
null
);
when
(
this
.
indicesService
.
isIndexExist
(
any
(),
any
())).
thenReturn
(
true
);
when
(
this
.
s
torageService
.
getStorage
Schema
(
kind
)).
thenReturn
(
storageSchema
);
when
(
this
.
s
chemaProvider
.
get
Schema
(
kind
)).
thenReturn
(
storageSchema
);
when
(
this
.
mappingService
.
createMapping
(
any
(),
any
(),
any
(),
anyBoolean
())).
thenThrow
(
new
AppException
(
HttpStatus
.
SC_BAD_REQUEST
,
reason
,
""
));
try
{
...
...
@@ -256,7 +257,7 @@ public class IndexerSchemaServiceTest {
when
(
this
.
elasticIndexNameResolver
.
getIndexNameFromKind
(
kind
)).
thenReturn
(
kind
.
replace
(
":"
,
"-"
));
when
(
this
.
schemaCache
.
get
(
kind
)).
thenReturn
(
null
);
when
(
this
.
indicesService
.
isIndexExist
(
any
(),
any
())).
thenReturn
(
true
);
when
(
this
.
s
torageService
.
getStorage
Schema
(
kind
)).
thenReturn
(
storageSchema
);
when
(
this
.
s
chemaProvider
.
get
Schema
(
kind
)).
thenReturn
(
storageSchema
);
when
(
this
.
mappingService
.
createMapping
(
any
(),
any
(),
any
(),
anyBoolean
())).
thenThrow
(
new
AppException
(
HttpStatus
.
SC_FORBIDDEN
,
reason
,
"blah"
));
try
{
...
...
@@ -282,7 +283,7 @@ public class IndexerSchemaServiceTest {
when
(
this
.
elasticIndexNameResolver
.
getIndexNameFromKind
(
kind
)).
thenReturn
(
kind
.
replace
(
":"
,
"-"
));
when
(
this
.
schemaCache
.
get
(
kind
)).
thenReturn
(
null
);
when
(
this
.
indicesService
.
isIndexExist
(
any
(),
any
())).
thenReturn
(
true
);
when
(
this
.
s
torageService
.
getStorage
Schema
(
kind
)).
thenReturn
(
storageSchema
);
when
(
this
.
s
chemaProvider
.
get
Schema
(
kind
)).
thenReturn
(
storageSchema
);
this
.
sut
.
processSchemaMessages
(
schemaMessages
);
...
...
@@ -336,7 +337,7 @@ public class IndexerSchemaServiceTest {
when
(
this
.
schemaCache
.
get
(
kind
)).
thenReturn
(
null
);
when
(
this
.
indicesService
.
isIndexExist
(
any
(),
any
())).
thenReturn
(
true
);
when
(
this
.
indicesService
.
deleteIndex
(
any
(),
any
())).
thenReturn
(
true
);
when
(
this
.
s
torageService
.
getStorage
Schema
(
kind
)).
thenReturn
(
storageSchema
);
when
(
this
.
s
chemaProvider
.
get
Schema
(
kind
)).
thenReturn
(
storageSchema
);
this
.
sut
.
syncIndexMappingWithStorageSchema
(
kind
);
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment