Skip to content
Snippets Groups Projects
Commit 1446a607 authored by Alan Braz's avatar Alan Braz
Browse files

restore azure and core from master

parent 4d53a194
No related branches found
No related tags found
1 merge request!6Trusted ibm
Hide whitespace changes
Inline Side-by-side
Showing
with 29 additions and 493 deletions
indexer-core/src/main/java/org/opengroup/osdu/indexer/api/RecordIndexerApi.java
+ 1
− 3
View file @ 1446a607
......@@ -21,7 +21,6 @@ import com.google.gson.JsonParseException;
import io.swagger.annotations.ApiOperation;
import lombok.extern.java.Log;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
import org.opengroup.osdu.indexer.SwaggerDoc;
......@@ -51,8 +50,7 @@ public class RecordIndexerApi {
private IndexerService indexerService;
@Inject
private ReindexService reIndexService;
@Inject
private JaxRsDpsLog log;
// THIS IS AN INTERNAL USE API ONLY
// THAT MEANS WE DON'T DOCUMENT IT IN SWAGGER, ACCESS IS LIMITED TO CLOUD TASK QUEUE CALLS ONLY
@PostMapping(path = "/index-worker", consumes = "application/json")
......
provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/AttributesCache.java
+ 1
− 1
View file @ 1446a607
......@@ -52,4 +52,4 @@ public class AttributesCache implements IAttributesCache<String,Set> {
this.cache.clearAll();
}
}
}
\ No newline at end of file
provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/ElasticCredentialsCache.java deleted 100644 → 0
+ 0
− 39
View file @ 4d53a194
package org.opengroup.osdu.indexer.azure.cache;
import org.opengroup.osdu.core.cache.VmCache;
import org.opengroup.osdu.is.core.model.ClusterSettings;
import org.opengroup.osdu.is.core.provider.interfaces.cache.IElasticCredentialsCache;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@Component
public class ElasticCredentialsCache implements IElasticCredentialsCache<String, ClusterSettings> {
private VmCache<String, ClusterSettings> cache;
public ElasticCredentialsCache(@Value("${ELASTIC_CACHE_EXPIRATION}") final String ELASTIC_CACHE_EXPIRATION,
@Value("${MAX_CACHE_VALUE_SIZE}") final String MAX_CACHE_VALUE_SIZE) {
cache = new VmCache<>(Integer.parseInt(ELASTIC_CACHE_EXPIRATION) * 60,
Integer.parseInt(MAX_CACHE_VALUE_SIZE));
}
@Override
public void put(String s, ClusterSettings o) {
this.cache.put(s,o);
}
@Override
public ClusterSettings get(String s) {
return this.cache.get(s);
}
@Override
public void delete(String s) {
this.cache.delete(s);
}
@Override
public void clearAll() {
this.cache.clearAll();
}
}
provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/di/DpsLogFactory.java deleted 100644 → 0
+ 0
− 35
View file @ 4d53a194
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.indexer.azure.di;
import org.opengroup.osdu.core.logging.DpsLog;
import org.opengroup.osdu.core.logging.Log;
import org.springframework.beans.factory.FactoryBean;
import org.springframework.stereotype.Component;
import org.springframework.web.context.annotation.RequestScope;
@Component
@RequestScope
public class DpsLogFactory implements FactoryBean<DpsLog> {
@Override
public DpsLog getObject() throws Exception {
return new Log();
}
@Override
public Class<?> getObjectType() {
return DpsLog.class;
}
}
provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/di/TenantInfoFactory.java deleted 100644 → 0
+ 0
− 43
View file @ 4d53a194
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.indexer.azure.di;
import lombok.extern.java.Log;
import org.opengroup.osdu.core.api.DpsHeaders;
import org.opengroup.osdu.core.multitenancy.ITenantFactory;
import org.opengroup.osdu.core.multitenancy.TenantInfo;
import org.springframework.beans.factory.config.AbstractFactoryBean;
import org.springframework.stereotype.Component;
import org.springframework.web.context.annotation.RequestScope;
import javax.inject.Inject;
@Log
@Component
@RequestScope
public class TenantInfoFactory extends AbstractFactoryBean<TenantInfo> {
@Inject
private ITenantFactory tenantFactory;
@Inject
private DpsHeaders headers;
@Override
protected TenantInfo createInstance() throws Exception {
String id = this.headers.getPartitionIdWithFallbackToAccountId();
return this.tenantFactory.getTenantInfo(id);
}
@Override
public Class<?> getObjectType() {
return TenantInfo.class;
}
}
provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/persistence/ElasticRepositoryCosmosDB.java deleted 100644 → 0
+ 0
− 69
View file @ 4d53a194
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.indexer.azure.persistence;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.multitenancy.TenantInfo;
import org.opengroup.osdu.indexer.azure.model.ElasticSettingSchema;
import org.opengroup.osdu.is.core.model.ClusterSettings;
import org.opengroup.osdu.is.core.provider.interfaces.persistence.ElasticRepository;
import org.opengroup.osdu.is.core.util.AppException;
import org.opengroup.osdu.is.core.util.Preconditions;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import javax.inject.Inject;
@Component
public class ElasticRepositoryCosmosDB implements ElasticRepository {
@Inject
private CosmosDBElasticSettings cosmosDB;
@Value("${ELASTIC_DATASTORE_KIND}")
private String ELASTIC_DATASTORE_KIND;
@Value("${ELASTIC_DATASTORE_ID}")
private String ELASTIC_DATASTORE_ID;
@Inject
private ISchemaRepository schemaRepository;
@Override
public ClusterSettings getElasticClusterSettings(TenantInfo tenantInfo) {
if(tenantInfo == null)
throw new AppException(HttpStatus.SC_NOT_FOUND, "TenantInfo is null", "");
String settingId = tenantInfo.getName().concat("-").concat(ELASTIC_DATASTORE_ID);
ElasticSettingSchema schema = this.schemaRepository.get(settingId);
if (schema == null) {
throw new AppException(HttpStatus.SC_NOT_FOUND, "Elastic setting not found", "The requested cluster setting was not found in CosmosDB.", String.format("Elastic setting with key: '%s' does not exist in CosmostDB.", ELASTIC_DATASTORE_KIND));
}
String host = schema.getHost();
String portString = schema.getPort();
String usernameAndPassword = schema.getUsernameAndPassword();
Preconditions.checkNotNullOrEmpty(host, "host cannot be null");
Preconditions.checkNotNullOrEmpty(portString, "port cannot be null");
Preconditions.checkNotNullOrEmpty(usernameAndPassword, "configuration cannot be null");
int port = Integer.parseInt(portString);
return new ClusterSettings(host, port, usernameAndPassword, schema.isHttps(), schema.isHttps());
}
}
provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/HeadersInfoAzureImpl.java deleted 100644 → 0
+ 0
− 95
View file @ 4d53a194
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.indexer.azure.util;
import lombok.extern.java.Log;
import org.opengroup.osdu.core.api.DpsHeaders;
import org.opengroup.osdu.is.core.model.SlbHeaders;
import org.opengroup.osdu.is.core.provider.interfaces.util.IHeadersInfo;
import org.opengroup.osdu.is.core.util.Preconditions;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.web.context.annotation.RequestScope;
import javax.inject.Inject;
import java.util.HashSet;
import java.util.Map;
import java.util.stream.Collectors;
@Log
@Component
@RequestScope
public class HeadersInfoAzureImpl implements IHeadersInfo {
@Inject
private DpsHeaders headersMap;
@Value("${indexer.queue.key}")
private String queueKey;
//ToDo this should be moved to Azure client-lib
public static final String INDEXER_QUEUE_KEY = "x-functions-key";
private static final HashSet<String> FORBIDDEN_FROM_LOGGING = new HashSet<>();
static {
FORBIDDEN_FROM_LOGGING.add(DpsHeaders.AUTHORIZATION);
FORBIDDEN_FROM_LOGGING.add(DpsHeaders.ON_BEHALF_OF);
// FORBIDDEN_FROM_LOGGING.add(INDEXER_QUEUE_KEY);
}
@Override
public DpsHeaders getHeaders() {
if (headersMap == null) {
log.warning("Headers Map DpsHeaders is null");
}
DpsHeaders headers = this.getCoreServiceHeaders(headersMap.getHeaders());
return headers;
}
@Override
public String getUser() {
return getHeaders().getUserEmail();
}
@Override
public String getPartitionId() {
return getHeaders().getPartitionIdWithFallbackToAccountId();
}
@Override
public String getPrimaryPartitionId() {
return getHeadersMap().get(SlbHeaders.PRIMARY_PARTITION_ID);
}
@Override
public Map<String, String> getHeadersMap() {
return getHeaders().getHeaders();
}
@Override
public DpsHeaders getCoreServiceHeaders(Map<String, String> input) {
Preconditions.checkNotNull(input, "input headers cannot be null");
DpsHeaders output = DpsHeaders.createFromMap(input);
output.put(INDEXER_QUEUE_KEY,queueKey);
return output;
}
@Override
public String toString() {
return this.getHeadersMap().entrySet().stream().filter(map -> !FORBIDDEN_FROM_LOGGING.contains(map.getKey().toLowerCase())).map(Map.Entry::toString).collect(Collectors.joining(" | "));
}
}
\ No newline at end of file
provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IHeadersInfo.java 0 → 100644
+ 21
− 0
View file @ 1446a607
package org.opengroup.osdu.indexer.azure.util;
import java.util.Map;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
public interface IHeadersInfo {
DpsHeaders getHeaders();
String getUser();
String getPartitionId();
String getPrimaryPartitionId();
Map<String, String> getHeadersMap();
DpsHeaders getCoreServiceHeaders(Map<String, String> input);
}
provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/CronServiceImplTest.java
+ 1
− 1
View file @ 1446a607
......@@ -138,4 +138,4 @@ public class CronServiceImplTest {
verify(this.indicesService, times(0)).deleteIndex(any(), any());
}
}
}
\ No newline at end of file
provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/ElasticSettingServiceTest.java deleted 100644 → 0
+ 0
− 112
View file @ 4d53a194
// Copyright 2017-2019, Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.indexer.azure.service;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.opengroup.osdu.core.multitenancy.TenantInfo;
import org.opengroup.osdu.indexer.service.ElasticSettingServiceImpl;
import org.opengroup.osdu.is.core.logging.JaxRsDpsLog;
import org.opengroup.osdu.is.core.model.ClusterSettings;
import org.opengroup.osdu.is.core.provider.interfaces.cache.IElasticCredentialsCache;
import org.opengroup.osdu.is.core.provider.interfaces.persistence.ElasticRepository;
import org.opengroup.osdu.is.core.provider.interfaces.util.IHeadersInfo;
import org.opengroup.osdu.is.core.service.TenantInfoService;
import org.opengroup.osdu.is.core.util.AppException;
import org.springframework.test.context.junit4.SpringRunner;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.mockito.Mockito.when;
@RunWith(SpringRunner.class)
public class ElasticSettingServiceTest {
@Mock
private TenantInfoService tenantInfoService;
@Mock
private ElasticRepository elasticRepository;
@Mock
private IElasticCredentialsCache elasticCredentialCache;
@Mock
private TenantInfo tenantInfo;
@InjectMocks
private ElasticSettingServiceImpl sut;
@Mock
private ClusterSettings clusterSettings;
@Mock
private IHeadersInfo headersInfo;
@Mock
private JaxRsDpsLog log;
public String GAE_SERVICE = "indexer";
private final String host = "db5c51c1.us-central1.gcp.cloud.es.io";
private final int port = 9243;
private final String credentials = "name:password";
String cacheKey = "";
@Before
public void setup() {
when(tenantInfo.getName()).thenReturn("tenant1");
when(this.headersInfo.getPartitionId()).thenReturn("tenant1");
when(this.tenantInfoService.getTenantInfo()).thenReturn(tenantInfo);
sut.GAE_SERVICE = "indexer";
clusterSettings = ClusterSettings.builder().host(host).port(port).userNameAndPassword(credentials).build();
cacheKey = String.format("%s-%s", GAE_SERVICE, tenantInfo.getName());
}
@Test
public void should_getValid_clusterSettings_fromCache() {
when(this.elasticCredentialCache.get(cacheKey)).thenReturn(clusterSettings);
ClusterSettings response = this.sut.getElasticClusterInformation();
assertNotNull(response);
assertEquals(response.getHost(), host);
assertEquals(response.getPort(), port);
assertEquals(response.getUserNameAndPassword(), credentials);
}
@Test
public void should_getValid_clusterSettings_fromCosmosDB() {
when(this.elasticCredentialCache.get(cacheKey)).thenReturn(clusterSettings);
when(this.elasticRepository.getElasticClusterSettings(tenantInfo)).thenReturn(clusterSettings);
ClusterSettings response = this.sut.getElasticClusterInformation();
assertNotNull(response);
assertEquals(response.getHost(), host);
assertEquals(response.getPort(), port);
assertEquals(response.getUserNameAndPassword(), credentials);
}
@Test(expected = AppException.class)
public void should_throwAppException_when_tenantClusterInfo_not_found() throws AppException {
when(this.elasticRepository.getElasticClusterSettings(tenantInfo)).thenReturn(null);
this.sut.getElasticClusterInformation();
}
}
provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexerMappingServiceTest.java
+ 5
− 5
View file @ 1446a607
......@@ -32,7 +32,7 @@ import org.junit.runner.RunWith;
import org.mockito.ArgumentMatchers;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.opengroup.osdu.indexer.model.IndexSchema;
import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
import org.opengroup.osdu.indexer.service.IndexerMappingServiceImpl;
import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
import org.opengroup.osdu.core.common.model.http.AppException;
......@@ -168,7 +168,7 @@ public class IndexerMappingServiceTest {
doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
this.sut.updateIndexMappingForIndicesOfSameType( indices,"any field");
} catch (Exception e) {
fail("Should not throw this exception" + e.getMessage());
......@@ -202,8 +202,8 @@ public class IndexerMappingServiceTest {
doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
} catch (Exception e) {
throw e;
}
......@@ -213,7 +213,7 @@ public class IndexerMappingServiceTest {
public void should_throw_exception_if_type_of_index_is_invalid_andWeIndexfield_with_keyword() throws Exception {
try {
Set<String> indices = new HashSet<String>();
indices.add("indices 1");
indices.add("indices 1");
GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
......
provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/TenantInfoServiceTest.java deleted 100644 → 0
+ 0
− 90
View file @ 4d53a194
// Copyright 2017-2019, Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.indexer.azure.service;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.opengroup.osdu.core.api.DpsHeaders;
import org.opengroup.osdu.core.multitenancy.ITenantFactory;
import org.opengroup.osdu.core.multitenancy.TenantInfo;
import org.opengroup.osdu.indexer.service.TenantInfoServiceImpl;
import org.opengroup.osdu.is.core.provider.interfaces.util.IHeadersInfo;
import org.opengroup.osdu.is.core.util.AppException;
import org.springframework.http.HttpHeaders;
import org.springframework.test.context.junit4.SpringRunner;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.mockito.Mockito.when;
@RunWith(SpringRunner.class)
public class TenantInfoServiceTest {
private static final String HEADER_NAME = "ANY_HEADER";
private static final String HEADER_VALUE = "ANY_VALUE";
@Mock
private ITenantFactory tenantFactory;
@Mock
private IHeadersInfo headersInfo;
@InjectMocks
private TenantInfoServiceImpl sut;
@Mock
private TenantInfo info;
@Mock
private HttpHeaders httpHeaders;
@InjectMocks
private DpsHeaders HEADERS;
@Before
public void setup() {
HEADERS.put(HEADER_NAME, HEADER_VALUE);
}
@Ignore
@Test
public void should_return_validTenant_given_validAccountId() {
when(this.info.getName()).thenReturn("tenant1");
when(tenantFactory.getTenantInfo("tenant1")).thenReturn(info);
when(this.headersInfo.getHeaders()).thenReturn(HEADERS);
when(this.headersInfo.getPartitionId()).thenReturn("tenant1");
when(this.sut.getTenantInfo()).thenReturn(info);
assertNotNull(this.sut.getTenantInfo());
assertEquals("tenant1", this.sut.getTenantInfo().getName());
}
@Test(expected = AppException.class)
public void should_throwException_given_invalidAccountId() {
when(this.info.getName()).thenReturn("tenant2");
when(tenantFactory.getTenantInfo("tenant1")).thenReturn(null);
when(this.sut.getTenantInfo()).thenReturn(info);
assertNotNull(this.sut.getTenantInfo());
}
}
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment