Commit de92f9c3 authored by neelesh thakur's avatar neelesh thakur
Browse files

initial code to swap out azure partition backend

parent 3f64f421
Pipeline #10278 passed with stages
in 7 minutes and 15 seconds
......@@ -15,6 +15,7 @@
package org.opengroup.osdu.partition.api;
import org.opengroup.osdu.partition.model.PartitionInfo;
import org.opengroup.osdu.partition.model.Property;
import org.opengroup.osdu.partition.provider.interfaces.IPartitionService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
......@@ -43,7 +44,7 @@ public class PartitionApi {
@GetMapping("/{partitionId}")
@PreAuthorize("@authorizationFilter.hasPermissions()")
public ResponseEntity<Map<String, Object>> get(@PathVariable("partitionId") String partitionId) {
public ResponseEntity<Map<String, Property>> get(@PathVariable("partitionId") String partitionId) {
PartitionInfo partitionInfo = this.partitionService.getPartition(partitionId);
return ResponseEntity.ok(partitionInfo.getProperties());
}
......
......@@ -29,5 +29,5 @@ import java.util.Map;
public class PartitionInfo {
@Builder.Default
Map<String, Object> properties = new HashMap<>();
Map<String, Property> properties = new HashMap<>();
}
\ No newline at end of file
package org.opengroup.osdu.partition.model;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class Property {
@Builder.Default
private boolean sensitive = false;
private Object value;
}
......@@ -21,11 +21,14 @@ import org.mockito.Mock;
import org.mockito.junit.MockitoJUnitRunner;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.partition.model.PartitionInfo;
import org.opengroup.osdu.partition.model.Property;
import org.opengroup.osdu.partition.provider.interfaces.IPartitionService;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.junit.Assert.assertEquals;
......@@ -63,12 +66,12 @@ public class PartitionApiTest {
@Test
public void should_return200AndPartitionProperties_when_gettingPartitionIdSuccessfully() {
String partitionId = "partition1";
Map<String, Object> properties = new HashMap<>();
Map<String, Property> properties = new HashMap<>();
when(partitionService.getPartition(anyString())).thenReturn(partitionInfo);
when(partitionInfo.getProperties()).thenReturn(properties);
ResponseEntity<Map<String, Object>> result = this.sut.get(partitionId);
ResponseEntity<Map<String, Property>> result = this.sut.get(partitionId);
assertEquals(HttpStatus.OK, result.getStatusCode());
assertEquals(properties, result.getBody());
}
......
......@@ -17,7 +17,6 @@
<properties>
<azure.version>2.3.1</azure.version>
<os_core_common_version>0.0.18</os_core_common_version>
</properties>
<dependencies>
<dependency>
......@@ -65,7 +64,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-azure</artifactId>
<version>0.0.19</version>
<version>0.0.29</version>
<exclusions>
<exclusion>
<groupId>org.opengroup.osdu</groupId>
......@@ -125,6 +124,12 @@
<version>3.3.8.RELEASE</version>
</dependency>
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>azure-storage</artifactId>
<version>8.6.5</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
......
package org.opengroup.osdu.partition.provider.azure.service;
package org.opengroup.osdu.partition.provider.azure.cache;
import org.opengroup.osdu.core.common.cache.VmCache;
import org.opengroup.osdu.partition.model.PartitionInfo;
......@@ -7,7 +7,8 @@ import org.springframework.stereotype.Service;
@Service
public class PartitionServiceCacheImpl extends VmCache<String, PartitionInfo> implements IPartitionServiceCache {
public PartitionServiceCacheImpl() {
super(5*60, 1000);
super(5 * 60, 1000);
}
}
......@@ -15,7 +15,7 @@
package org.opengroup.osdu.partition.provider.azure.di;
import com.azure.security.keyvault.secrets.SecretClient;
import org.opengroup.osdu.partition.provider.azure.utils.KeyVaultFacade;
import org.opengroup.osdu.azure.KeyVaultFacade;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
......@@ -35,14 +35,14 @@ public class AzureBootstrapConfig {
}
@Bean
@Named("COSMOS_ENDPOINT")
public String cosmosEndpoint(SecretClient kv) {
return KeyVaultFacade.getKeyVaultSecret(kv, "cosmos-endpoint");
@Named("TABLE_STORAGE_ACCOUNT_NAME")
public String storageAccountName(SecretClient kv) {
return KeyVaultFacade.getSecretWithValidation(kv, "tbl-storage");
}
@Bean
@Named("COSMOS_KEY")
public String cosmosKey(SecretClient kv) {
return KeyVaultFacade.getKeyVaultSecret(kv, "cosmos-primary-key");
@Named("TABLE_STORAGE_ACCOUNT_KEY")
public String storageAccountKey(SecretClient kv) {
return KeyVaultFacade.getSecretWithValidation(kv, "tbl-storage-key");
}
}
\ No newline at end of file
package org.opengroup.osdu.partition.provider.azure.di;
import lombok.Getter;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
@Configuration
@Getter
public class CloudTableConfiguration {
@Value("${partition.cloud.table-name:PartitionInfo}")
private String cloudTableName;
}
package org.opengroup.osdu.partition.provider.azure.di;
import com.microsoft.azure.storage.CloudStorageAccount;
import com.microsoft.azure.storage.StorageException;
import com.microsoft.azure.storage.table.CloudTable;
import com.microsoft.azure.storage.table.CloudTableClient;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.common.Validators;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
import javax.inject.Named;
import java.net.URISyntaxException;
import java.security.InvalidKeyException;
@Component
public class TableStorageClient {
private final String CONNECTION_STRING = "DefaultEndpointsProtocol=https;AccountName=%s;AccountKey=%s;EndpointSuffix=core.windows.net";
@Bean
@Lazy
public CloudTableClient getCloudTableClient(
final @Named("TABLE_STORAGE_ACCOUNT_NAME") String storageAccountName,
final @Named("TABLE_STORAGE_ACCOUNT_KEY") String storageAccountKey) {
try {
Validators.checkNotNullAndNotEmpty(storageAccountName, "storageAccountName");
Validators.checkNotNullAndNotEmpty(storageAccountKey, "storageAccountKey");
final String storageConnectionString = String.format(CONNECTION_STRING, storageAccountName, storageAccountKey);
CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString);
return storageAccount.createCloudTableClient();
} catch (URISyntaxException | InvalidKeyException e) {
throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error creating cloud table storage client", e.getMessage(), e);
}
}
@Bean
@Lazy
public CloudTable getCloudTable(
final CloudTableClient cloudTableClient,
final CloudTableConfiguration tblConfiguration) {
try {
Validators.checkNotNull(cloudTableClient, "cloudTableClient");
Validators.checkNotNull(tblConfiguration, "tblConfiguration");
CloudTable cloudTable = cloudTableClient.getTableReference(tblConfiguration.getCloudTableName());
cloudTable.createIfNotExists();
return cloudTable;
} catch (URISyntaxException | StorageException e) {
throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, String.format("Error querying cloud table: %s", tblConfiguration), e.getMessage(), e);
}
}
}
package org.opengroup.osdu.partition.provider.azure.persistence;
import com.microsoft.azure.storage.StorageException;
import com.microsoft.azure.storage.table.*;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@Component
public class CloudTableStore {
private final String PARTITION_KEY = "PartitionKey";
@Autowired
private CloudTable cloudTableClient;
public boolean deleteCloudTableEntity(final Class<? extends TableEntity> clazzType, String partitionKey, String rowKey) {
try {
TableOperation retrievePartition = TableOperation.retrieve(partitionKey, rowKey, clazzType);
TableEntity partitionEntity = this.cloudTableClient.execute(retrievePartition).getResultAsType();
TableOperation deleteOperation = TableOperation.delete(partitionEntity);
this.cloudTableClient.execute(deleteOperation);
return true;
} catch (StorageException e) {
throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error querying cloud table", e.getMessage(), e);
}
}
public Iterable<? extends TableEntity> queryByPartitionId(final Class<? extends TableEntity> clazzType, String value) {
String partitionFilter = TableQuery.generateFilterCondition(
PARTITION_KEY,
TableQuery.QueryComparisons.EQUAL,
value);
TableQuery<? extends TableEntity> partitionQuery = TableQuery.from(clazzType)
.where(partitionFilter);
return this.cloudTableClient.execute(partitionQuery);
}
public void insertBatchEntities(TableBatchOperation batchOperation) {
try {
this.cloudTableClient.execute(batchOperation);
} catch (StorageException e) {
new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "error creating partition", e.getMessage(), e);
}
}
}
package org.opengroup.osdu.partition.provider.azure.persistence;
import com.microsoft.azure.storage.table.DynamicTableEntity;
public class PartitionEntity extends DynamicTableEntity {
private String partitionId;
private String name;
public PartitionEntity() {}
public PartitionEntity(String partitionId, String name) {
super(partitionId, name);
this.partitionId = partitionId;
this.name = name;
}
public String getName() {
return this.name;
}
public void setName(String name) { this.name = name; }
public String getPartitionId() {
return this.partitionId;
}
public void setPartitionId(String partitionId) { this.partitionId = partitionId; }
}
\ No newline at end of file
package org.opengroup.osdu.partition.provider.azure.persistence;
import com.microsoft.azure.storage.table.EntityProperty;
import com.microsoft.azure.storage.table.TableBatchOperation;
import org.opengroup.osdu.partition.model.PartitionInfo;
import org.opengroup.osdu.partition.model.Property;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Component
public class PartitionTableStore {
@Autowired
private CloudTableStore cloudTableStore;
public void addPartition(String partitionId, PartitionInfo partitionInfo) {
Map<String, Property> requestProperties = partitionInfo.getProperties();
TableBatchOperation batchOperation = new TableBatchOperation();
for (Map.Entry<String, Property> entry : requestProperties.entrySet()) {
String key = entry.getKey();
Property property = entry.getValue();
PartitionEntity partitionEntity = new PartitionEntity(partitionId, key);
HashMap<String, EntityProperty> properties = new HashMap<>();
if (property.isSensitive()) {
property.setValue(this.getTenantSafeSecreteId(partitionId, String.valueOf(property.getValue())));
}
properties.put("value", new EntityProperty(String.valueOf(property.getValue())));
properties.put("sensitive", new EntityProperty(property.isSensitive()));
partitionEntity.setProperties(properties);
batchOperation.insertOrReplace(partitionEntity);
}
this.cloudTableStore.insertBatchEntities(batchOperation);
}
public boolean partitionExists(String partitionId) {
List<PartitionEntity> partitionEntities = this.queryByPartitionId(partitionId);
return !partitionEntities.isEmpty();
}
public Map<String, Property> getPartition(String partitionId) {
Map<String, Property> out = new HashMap<>();
List<PartitionEntity> partitionEntities = this.queryByPartitionId(partitionId);
if (partitionEntities.isEmpty()) {
return out;
}
for (PartitionEntity pe : partitionEntities) {
Property property = Property.builder().build();
HashMap<String, EntityProperty> properties = pe.getProperties();
if (properties.containsKey("sensitive")) {
property.setSensitive(properties.get("sensitive").getValueAsBoolean());
}
if (properties.containsKey("value")) {
property.setValue(properties.get("value").getValueAsString());
}
out.put(pe.getRowKey(), property);
}
return out;
}
public List<PartitionEntity> queryByPartitionId(String partitionId) {
List<PartitionEntity> out = new ArrayList<>();
Iterable<PartitionEntity> results = (Iterable<PartitionEntity>) this.cloudTableStore.queryByPartitionId(PartitionEntity.class, partitionId);
for (PartitionEntity tableEntity : results) {
tableEntity.setPartitionId(tableEntity.getPartitionKey());
tableEntity.setName(tableEntity.getRowKey());
out.add(tableEntity);
}
return out;
}
public void deletePartition(String partitionId) {
Iterable<PartitionEntity> results = (Iterable<PartitionEntity>) this.cloudTableStore.queryByPartitionId(PartitionEntity.class, partitionId);
for (PartitionEntity tableEntity : results) {
this.cloudTableStore.deleteCloudTableEntity(PartitionEntity.class, tableEntity.getPartitionKey(), tableEntity.getRowKey());
}
}
private String getTenantSafeSecreteId(String partitionId, String secreteName) {
return String.format("%s-%s", partitionId, secreteName);
}
}
......@@ -14,108 +14,55 @@
package org.opengroup.osdu.partition.provider.azure.service;
import com.azure.security.keyvault.secrets.SecretClient;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.partition.model.PartitionInfo;
import org.opengroup.osdu.partition.provider.azure.utils.KeyVaultFacade;
import org.opengroup.osdu.partition.provider.azure.utils.ThreadPoolService;
import org.opengroup.osdu.partition.model.Property;
import org.opengroup.osdu.partition.provider.azure.persistence.PartitionTableStore;
import org.opengroup.osdu.partition.provider.interfaces.IPartitionService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Service
public class PartitionServiceImpl implements IPartitionService {
@Autowired
private SecretClient secretClient;
@Autowired
private ThreadPoolService threadPoolService;
private static final String APP_DEV_SP_USERNAME = "app-dev-sp-username";
private static final String SERVICE_PRINCIPAL_ID = "sp-appid";
private PartitionTableStore tableStore;
@Override
public PartitionInfo createPartition(String partitionId, PartitionInfo partitionInfo) {
if (this.partitionExists(partitionId)) {
if (this.tableStore.partitionExists(partitionId)) {
throw new AppException(HttpStatus.SC_CONFLICT, "partition exist", "Partition with same id exist");
}
this.addTenantSecretes(partitionId, partitionInfo);
this.tableStore.addPartition(partitionId, partitionInfo);
return partitionInfo;
}
@Override
public PartitionInfo getPartition(String partitionId) {
if (!this.partitionExists(partitionId)) {
Map<String, Property> out = new HashMap<>();
out.putAll(this.tableStore.getPartition(partitionId));
if (out.isEmpty()) {
throw new AppException(HttpStatus.SC_NOT_FOUND, "partition not found", String.format("%s partition not found", partitionId));
}
Map<String, Object> out = new HashMap<>();
out.putAll(this.getTenantSecreteInfo(partitionId));
return PartitionInfo.builder().properties(out).build();
}
@Override
public boolean deletePartition(String partitionId) {
if (!this.partitionExists(partitionId)) {
if (!this.tableStore.partitionExists(partitionId)) {
throw new AppException(HttpStatus.SC_NOT_FOUND, "partition not found", String.format("%s partition not found", partitionId));
}
this.deleteTenantSecrets(partitionId);
this.tableStore.deletePartition(partitionId);
return true;
}
private void addTenantSecretes(String partitionId, PartitionInfo partitionInfo) {
// id
KeyVaultFacade.createKeyVaultSecret(this.secretClient, getTenantSafeSecreteId(partitionId, "id"), partitionId);
// rest of keys
for (Map.Entry<String, Object> entry : partitionInfo.getProperties().entrySet()) {
String secreteName = this.getTenantSafeSecreteId(partitionId, entry.getKey());
KeyVaultFacade.createKeyVaultSecret(this.secretClient, secreteName, String.valueOf(entry.getValue()));
}
}
private Map<String, Object> getTenantSecreteInfo(String partitionId) {
Map<String, Object> out = new HashMap<>();
List<String> secreteKeys = KeyVaultFacade.getKeyVaultSecrets(secretClient, partitionId);
if (secreteKeys.isEmpty()) {
return out;
}
for (String key : secreteKeys) {
String outKey = key.replaceFirst(String.format("%s-", partitionId), "");
out.put(outKey, KeyVaultFacade.getKeyVaultSecret(this.secretClient, key));
}
out.put(SERVICE_PRINCIPAL_ID, KeyVaultFacade.getKeyVaultSecret(this.secretClient, APP_DEV_SP_USERNAME));
return out;
}
private void deleteTenantSecrets(String partitionId) {
List<String> secreteKeys = KeyVaultFacade.getKeyVaultSecrets(secretClient, partitionId);
if (secreteKeys.isEmpty()) {
return;
}
this.threadPoolService.createDeletePoolIfNeeded(secreteKeys.size());
for (String key : secreteKeys) {
this.threadPoolService.getExecutorService().submit(() -> KeyVaultFacade.deleteKeyVaultSecret(this.secretClient, key));
}
}
private String getTenantSafeSecreteId(String partitionId, String secreteName) {
return String.format("%s-%s", partitionId, secreteName);
}
private boolean partitionExists(String partitionId) {
return KeyVaultFacade.secretExists(secretClient, getTenantSafeSecreteId(partitionId, "id"));
}
}
\ No newline at end of file
// Copyright 2017-2020, Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.partition.provider.azure.utils;
import com.azure.core.exception.HttpResponseException;
import com.azure.core.exception.ResourceNotFoundException;
import com.azure.core.http.rest.PagedIterable;
import com.azure.core.http.rest.PagedResponse;
import com.azure.core.util.polling.SyncPoller;