Commit 748e0a72 authored by Sheng Wang's avatar Sheng Wang
Browse files

support partition service

parent 68741b68
Pipeline #21531 failed with stages
in 2 minutes and 25 seconds
......@@ -5,24 +5,24 @@ variables:
include:
- project: "osdu/platform/ci-cd-pipelines"
ref: "Azure-slb"
ref: "master"
file: "standard-setup.yml"
- project: "osdu/platform/ci-cd-pipelines"
ref: "Azure-slb"
ref: "master"
file: "build/maven.yml"
- project: "osdu/platform/ci-cd-pipelines"
ref: "Azure-slb"
file: "scanners/fossa.yml"
ref: "master"
file: "scanners/fossa-maven.yml"
- project: "osdu/platform/ci-cd-pipelines"
ref: "Azure-slb"
ref: "master"
file: "scanners/gitlab-ultimate.yml"
- project: "osdu/platform/ci-cd-pipelines"
ref: "Azure-slb"
file: "cloud-providers/azure-slb.yml"
ref: "master"
file: "cloud-providers/azure.yml"
......
......@@ -15,6 +15,6 @@
apiVersion: v2
name: well-planning
appVersion: "latest"
description: Helm Chart for installing storage service.
description: Helm Chart for installing well delivery service.
version: 0.1.0
type: application
......@@ -15,7 +15,7 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: well-planning-deployment
name: {{ .Release.Name }}
namespace: osdu
spec:
replicas: {{ .Values.global.replicaCount }}
......@@ -33,33 +33,25 @@ spec:
ports:
- containerPort: 8080
env:
- name: entitlements_service_endpoint
value: http://entitlements-azure/entitlements/v1
- name: entitlements_service_api_key
valueFrom:
secretKeyRef:
name: api-osdu-secret
key: entitlements_service_api_key
value: "OBSOLETE"
- name: legal_service_endpoint
valueFrom:
secretKeyRef:
name: api-osdu-secret
key: legal_service_endpoint
- name: entitlements_service_endpoint
valueFrom:
secretKeyRef:
name: api-osdu-secret
key: entitlements_service_endpoint
value: http://legal/api/legal/v1
- name: storage_service_endpoint
valueFrom:
secretKeyRef:
name: api-osdu-secret
key: storage_service_endpoint
value: http://storage/api/storage/v2
- name: schema_service_endpoint
value: http://schema-service/api/schema-service/v1
- name: partition_service_endpoint
value: http://partition/api/partition/v1
- name: KEYVAULT_URI
valueFrom:
secretKeyRef:
name: api-osdu-secret
key: schema_service_endpoint
- name: mongodb_connection_string
configMapKeyRef:
name: osdu-svc-properties
key: ENV_KEYVAULT
- name: aad_client_id
valueFrom:
secretKeyRef:
name: api-osdu-secret
key: mongodb_connection_string
\ No newline at end of file
name: active-directory
key: application-appid
......@@ -16,7 +16,7 @@ apiVersion: v1
kind: Service
metadata:
name: {{ .Chart.Name }}
namespace: ingress-nginx
namespace: osdu
spec:
type: ClusterIP
ports:
......
......@@ -110,4 +110,11 @@
</plugins>
</build>
<repositories>
<repository>
<id>gitlab-maven</id>
<url>https://community.opengroup.org/api/v4/projects/67/packages/maven</url>
</repository>
</repositories>
</project>
......@@ -41,6 +41,13 @@
</properties>
<dependencies>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-service-client</artifactId>
<version>0.0.8</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu.wd</groupId>
<artifactId>well-delivery-core</artifactId>
......@@ -54,6 +61,12 @@
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.auth0</groupId>
<artifactId>java-jwt</artifactId>
<version>3.12.0</version>
</dependency>
<!-- Azure dependencies -->
<dependency>
<groupId>com.microsoft.azure</groupId>
......
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.wd.azure.keyvault;
import com.azure.identity.DefaultAzureCredential;
import com.azure.identity.DefaultAzureCredentialBuilder;
import com.azure.security.keyvault.secrets.SecretClient;
import com.azure.security.keyvault.secrets.SecretClientBuilder;
import com.azure.security.keyvault.secrets.models.KeyVaultSecret;
import io.micrometer.core.instrument.util.StringUtils;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.client.model.http.AppException;
import java.util.logging.Level;
import java.util.logging.Logger;
public class KeyVaultFacade {
private static final Logger LOGGER = Logger.getLogger(KeyVaultFacade.class.getName());
public static String getSecretWithValidation(SecretClient client, String secretName) {
if (client == null) {
String errorMessage = "Secret Client is null";
LOGGER.log(Level.WARNING, errorMessage);
throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, errorMessage, errorMessage);
}
KeyVaultSecret secret = client.getSecret(secretName);
if (secret == null) {
String errorMessage = "Secret is null";
LOGGER.log(Level.WARNING, errorMessage);
throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, errorMessage, errorMessage);
}
String secretValue = secret.getValue();
if (StringUtils.isBlank(secretValue)) {
String errorMessage = "Secret value is null";
LOGGER.log(Level.WARNING, errorMessage);
throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, errorMessage, errorMessage);
}
return secretValue;
}
public static SecretClient keyVaultSecretClient(DefaultAzureCredential credential, String keyVaultURL) {
if (credential == null) {
String errorMessage = "Credential is null";
LOGGER.log(Level.WARNING, errorMessage);
throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, errorMessage, errorMessage);
}
if (StringUtils.isBlank(keyVaultURL)) {
String errorMessage = "KV URL is null";
LOGGER.log(Level.WARNING, errorMessage);
throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, errorMessage, errorMessage);
}
return new SecretClientBuilder()
.credential(credential)
.vaultUrl(keyVaultURL)
.buildClient();
}
public static DefaultAzureCredential azureCredential() {
return new DefaultAzureCredentialBuilder().build();
}
}
// Copyright 2020 Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.wd.azure.keyvault;
import com.azure.identity.DefaultAzureCredential;
import com.azure.security.keyvault.secrets.SecretClient;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.util.logging.Level;
import java.util.logging.Logger;
@Component
public class KeyvaultClient {
private static final Logger LOGGER = Logger.getLogger(KeyvaultClient.class.getName());
@Value("${azure.keyvault.url}")
private String keyVaultURL;
public String getSecretValue(String key) {
SecretClient secretClient = buildSecretsClient();
String value = KeyVaultFacade.getSecretWithValidation(secretClient, key);
LOGGER.log(Level.INFO, String.format("Secret %s: %s***", key, value.substring(0, 3)));
return value;
}
private SecretClient buildSecretsClient() {
LOGGER.log(Level.INFO, String.format("keyVaultURL: %s", this.keyVaultURL));
DefaultAzureCredential credential = KeyVaultFacade.azureCredential();
SecretClient secretClient = KeyVaultFacade.keyVaultSecretClient(credential, this.keyVaultURL);
return secretClient;
}
}
// Copyright 2020 Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.wd.azure.partition;
import com.azure.security.keyvault.secrets.SecretClient;
import com.google.gson.JsonElement;
import com.google.gson.Gson;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.client.model.http.AppException;
import org.opengroup.osdu.core.client.model.http.DpsHeaders;
import org.opengroup.osdu.core.client.model.partition.PartitionException;
import org.opengroup.osdu.core.client.model.partition.PartitionInfo;
import org.opengroup.osdu.core.client.partition.IPartitionFactory;
import org.opengroup.osdu.core.client.partition.IPartitionProvider;
import org.opengroup.osdu.wd.azure.keyvault.KeyVaultFacade;
import org.opengroup.osdu.wd.azure.keyvault.KeyvaultClient;
import org.opengroup.osdu.wd.azure.util.AzureServicePrincipleTokenService;
import org.opengroup.osdu.wd.core.auth.RequestInfo;
import org.opengroup.osdu.wd.core.cache.MongoConnStringCache;
import org.opengroup.osdu.wd.core.dataaccess.interfaces.IMongodbConnection;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.logging.Logger;
@Service
public class PartitionAndCacheService implements IMongodbConnection {
private static final Logger logger = Logger.getLogger(PartitionAndCacheService.class.getName());
@Autowired
private IPartitionFactory partitionFactory;
@Autowired
KeyvaultClient keyvaultClient;
@Autowired
private AzureServicePrincipleTokenService tokenService;
@Autowired
private MongoConnStringCache connCache;
@Autowired
private RequestInfo requestInfo;
private final Gson gson = new Gson();
@Override
public String get(String partitionId) {
String connString = (String) this.connCache.get(partitionId);
if (connString != null) {
return connString;
}
connString = getConnString(partitionId);
this.connCache.save(partitionId, connString);
return connString;
}
private String getConnString(String partitionId) {
try {
IPartitionProvider serviceClient = getServiceClient();
PartitionInfo partitionInfo = serviceClient.get(partitionId);
PartitionInfoAzure partitionInfoAzure = convert(partitionInfo);
String connString = getReCosmosConnection(partitionInfoAzure);
return connString;
} catch (PartitionException e) {
throw new AppException(HttpStatus.SC_FORBIDDEN, "Service unavailable", String.format("Error getting partition info for data-partition: %s", partitionId), e);
}
}
private PartitionInfoAzure convert(final PartitionInfo partitionInfo) {
JsonElement jsonElement = gson.toJsonTree(partitionInfo.getProperties());
PartitionInfoAzure infoAzure = gson.fromJson(jsonElement, PartitionInfoAzure.class);
return infoAzure;
}
private String getReCosmosConnection(PartitionInfoAzure infoAzure) {
String connection = String.valueOf(infoAzure.getReCosmosConnectionConfig().getValue());
if (infoAzure.getReCosmosConnectionConfig().isSensitive()) {
return keyvaultClient.getSecretValue(connection);
} else {
return connection;
}
}
private IPartitionProvider getServiceClient() {
this.requestInfo.getDpsHeaders().put(DpsHeaders.AUTHORIZATION, "Bearer " + this.tokenService.getAuthorizationToken());
return this.partitionFactory.create(this.requestInfo.getDpsHeaders());
}
}
// Copyright 2020 Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.wd.azure.partition;
import com.google.gson.annotations.SerializedName;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.opengroup.osdu.core.client.model.partition.Property;
/**
* Azure data partition variables.
*/
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class PartitionInfoAzure {
@SerializedName("id")
private Property idConfig;
@SerializedName("name")
private Property nameConfig;
@SerializedName("re-cosmos-connection")
private Property reCosmosConnectionConfig;
}
// Copyright 2020 Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.wd.azure.partition;
import org.opengroup.osdu.core.client.partition.IPartitionFactory;
import org.opengroup.osdu.core.client.partition.PartitionAPIConfig;
import org.opengroup.osdu.core.client.partition.PartitionFactory;
import org.opengroup.osdu.wd.core.osducoreserviceclient.SchemaClientFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component;
import java.util.logging.Level;
import java.util.logging.Logger;
@Component
public class PartitionServiceFactory {
private static final Logger LOGGER = Logger.getLogger(PartitionServiceFactory.class.getName());
@Value("${app.partition.api}")
private String PARTITON_API;
@Bean
public IPartitionFactory partitionFactory() {
LOGGER.log(Level.INFO, "PARTITON_API:" + PARTITON_API);
PartitionAPIConfig apiConfig = PartitionAPIConfig.builder()
.rootUrl(this.PARTITON_API)
.build();
return new PartitionFactory(apiConfig);
}
}
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.wd.azure.util;
import com.azure.core.http.HttpClient;
import com.azure.core.http.HttpMethod;
import com.azure.core.http.HttpRequest;
import com.azure.core.http.HttpResponse;
import com.azure.core.http.netty.NettyAsyncHttpClientBuilder;
import com.google.gson.Gson;
import com.google.gson.JsonObject;
import reactor.core.publisher.Mono;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
/**
* Class to generate the AAD authentication tokens.
*/
public final class AzureServicePrincipal {
/**
* @param sp_id AZURE CLIENT ID
* @param sp_secret AZURE CLIENT SECRET
* @param tenant_id AZURE TENANT ID
* @param app_resource_id AZURE APP RESOURCE ID
* @return AUTHENTICATION TOKEN
* @throws UnsupportedEncodingException throws UnsupportedEncodingException
*/
public String getIdToken(final String sp_id, final String sp_secret, final String tenant_id, final String app_resource_id) throws UnsupportedEncodingException {
String aadEndpoint = String.format("https://login.microsoftonline.com/%s/oauth2/token", tenant_id);
HttpRequest httpRequest = new HttpRequest(HttpMethod.POST, aadEndpoint);
Map<String, String> parameters = new HashMap<>();
parameters.put("grant_type", "client_credentials");
parameters.put("client_id", sp_id);
parameters.put("client_secret", sp_secret);
parameters.put("resource", app_resource_id);
httpRequest.setBody(getParamsString(parameters));
HttpClient client = createHttpClient();
Mono<HttpResponse> response = client.send(httpRequest);
String content = Objects.requireNonNull(response.block()).getBodyAsString().block();
Gson gson = new Gson();
JsonObject jsonObject = gson.fromJson(content, JsonObject.class);
return jsonObject.get("access_token").getAsString();
}
/**
* @param params Map of request parameters
* @return parameter string
* @throws UnsupportedEncodingException throws exception unsupported encoding is found
*/
private String getParamsString(final Map<String, String> params)
throws UnsupportedEncodingException {
StringBuilder result = new StringBuilder();
for (Map.Entry<String, String> entry : params.entrySet()) {
result.append(URLEncoder.encode(entry.getKey(), "UTF-8"));
result.append("=");
result.append(URLEncoder.encode(entry.getValue(), "UTF-8"));
result.append("&");
}
String resultString = result.toString();
return resultString.length() > 0
? resultString.substring(0, resultString.length() - 1)
: resultString;
}
/**
* @return HttpClient
*/
HttpClient createHttpClient() {
return new NettyAsyncHttpClientBuilder().build();
}
}
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0