Skip to content
Snippets Groups Projects
Commit 3d192d14 authored by Rostislav Dublin (EPAM)'s avatar Rostislav Dublin (EPAM)
Browse files

Merge branch 'unift_logic_for_credentials' into 'master' 

GCP Unify logic for credentials  (GONRG-539)

See merge request !31
parents c081dd73 73408fa8
No related branches found
No related tags found
1 merge request!31GCP Unify logic for credentials (GONRG-539)
Pipeline #11690 passed
Stage: build
Stage: containerize
Stage: scan
Stage: deploy
Stage: integration
Stage: attribution
Stage: publish
Hide whitespace changes
Inline Side-by-side
Showing
with 26 additions and 309 deletions
provider/indexer-gcp/pom.xml
+ 1
− 1
View file @ 3d192d14
......@@ -43,7 +43,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-gcp</artifactId>
<version>0.1.17</version>
<version>0.3.21</version>
</dependency>
<dependency>
......
provider/indexer-gcp/src/main/appengine/app.yaml
+ 1
− 0
View file @ 3d192d14
......@@ -39,3 +39,4 @@ env_variables:
LEGAL_HOSTNAME: "LEGAL_HOSTNAME_VAR"
REGION: "REGION_VAR"
SPRING_PROFILES_ACTIVE: 'ENVIRONMENT'
provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/AppengineLogFactory.java deleted 100644 → 0
+ 0
− 29
View file @ c081dd73
package org.opengroup.osdu.indexer.di;
import org.opengroup.osdu.core.common.logging.ILogger;
import org.opengroup.osdu.core.gcp.logging.logger.AppEngineLoggingProvider;
import org.springframework.beans.factory.FactoryBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Lazy;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
@ConditionalOnProperty(name="disable.appengine.log.factory", havingValue = "false", matchIfMissing = true )
@Component
@Primary
@Lazy
public class AppengineLogFactory implements FactoryBean<ILogger> {
private AppEngineLoggingProvider appEngineLoggingProvider = new AppEngineLoggingProvider();
@Override
public ILogger getObject() throws Exception {
return appEngineLoggingProvider.getLogger();
}
@Override
public Class<?> getObjectType() {
return ILogger.class;
}
}
\ No newline at end of file
provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/DatastoreCredentialsCacheFactory.java 0 → 100644
+ 22
− 0
View file @ 3d192d14
package org.opengroup.osdu.indexer.di;
import org.opengroup.osdu.core.common.cache.ICache;
import org.opengroup.osdu.core.common.cache.VmCache;
import org.opengroup.osdu.core.gcp.multitenancy.credentials.DatastoreCredential;
import org.springframework.beans.factory.config.AbstractFactoryBean;
import org.springframework.stereotype.Component;
@Component
public class DatastoreCredentialsCacheFactory extends
AbstractFactoryBean<ICache<String, DatastoreCredential>> {
@Override
public Class<?> getObjectType() {
return ICache.class;
}
@Override
protected ICache<String, DatastoreCredential> createInstance() throws Exception {
return new VmCache<>(5 * 60, 20);
}
}
provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/kms/KmsClient.java deleted 100644 → 0
+ 0
− 102
View file @ c081dd73
// Copyright 2017-2019, Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.indexer.kms;
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.http.javanet.NetHttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.api.services.cloudkms.v1.CloudKMS;
import com.google.api.services.cloudkms.v1.CloudKMSScopes;
import com.google.api.services.cloudkms.v1.model.DecryptRequest;
import com.google.api.services.cloudkms.v1.model.DecryptResponse;
import com.google.api.services.cloudkms.v1.model.EncryptRequest;
import com.google.api.services.cloudkms.v1.model.EncryptResponse;
import org.opengroup.osdu.core.common.provider.interfaces.IKmsClient;
import org.opengroup.osdu.core.common.search.Preconditions;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.web.context.annotation.RequestScope;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
@Component
@RequestScope
public class KmsClient implements IKmsClient {
@Value("${GOOGLE_CLOUD_PROJECT}")
private String GOOGLE_CLOUD_PROJECT;
@Value("${KMS_KEY}")
private String KMS_KEY;
@Value("${KEY_RING}")
private String KEY_RING;
private static final String KEY_NAME = "projects/%s/locations/global/keyRings/%s/cryptoKeys/%s";
/**
* Encrypts the given plaintext using the specified crypto key.
* Google KMS automatically uses the new primary key version to encrypt data, so this could be directly used for key rotation
*/
public String encryptString(String textToBeEncrypted) throws IOException {
Preconditions.checkNotNullOrEmpty(textToBeEncrypted, "textToBeEncrypted cannot be null");
byte[] plaintext = textToBeEncrypted.getBytes(StandardCharsets.UTF_8);
String resourceName = String.format(KEY_NAME, GOOGLE_CLOUD_PROJECT, KEY_RING, KMS_KEY);
CloudKMS kms = createAuthorizedClient();
EncryptRequest request = new EncryptRequest().encodePlaintext(plaintext);
EncryptResponse response = kms.projects().locations().keyRings().cryptoKeys()
.encrypt(resourceName, request)
.execute();
return response.getCiphertext();
}
/**
* Decrypts the provided ciphertext with the specified crypto key.
* Google KMS automatically uses the correct key version to decrypt data, as long as the key version is not disabled
*/
public String decryptString(String textToBeDecrypted) throws IOException {
Preconditions.checkNotNullOrEmpty(textToBeDecrypted, "textToBeDecrypted cannot be null");
CloudKMS kms = createAuthorizedClient();
String cryptoKeyName = String.format(KEY_NAME, GOOGLE_CLOUD_PROJECT, KEY_RING, KMS_KEY);
DecryptRequest request = new DecryptRequest().setCiphertext(textToBeDecrypted);
DecryptResponse response = kms.projects().locations().keyRings().cryptoKeys()
.decrypt(cryptoKeyName, request)
.execute();
return new String(response.decodePlaintext(), StandardCharsets.UTF_8).trim();
}
/**
* Creates an authorized CloudKMS client service using Application Default Credentials.
*
* @return an authorized CloudKMS client
* @throws IOException if there's an error getting the default credentials.
*/
private CloudKMS createAuthorizedClient() throws IOException {
HttpTransport transport = new NetHttpTransport();
JsonFactory jsonFactory = new JacksonFactory();
GoogleCredential credential = GoogleCredential.getApplicationDefault();
if (credential.createScopedRequired()) {
credential = credential.createScoped(CloudKMSScopes.all());
}
return new CloudKMS.Builder(transport, jsonFactory, credential)
.setApplicationName("CloudKMS snippets")
.build();
}
}
\ No newline at end of file
provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreCredential.java deleted 100644 → 0
+ 0
− 109
View file @ c081dd73
// Copyright 2017-2019, Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.indexer.persistence;
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.api.services.iam.v1.Iam;
import com.google.api.services.iam.v1.Iam.Projects.ServiceAccounts.SignJwt;
import com.google.api.services.iam.v1.model.SignJwtRequest;
import com.google.api.services.iam.v1.model.SignJwtResponse;
import com.google.auth.oauth2.AccessToken;
import com.google.auth.oauth2.GoogleCredentials;
import com.google.gson.JsonObject;
import org.apache.commons.lang3.time.DateUtils;
import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
import org.opengroup.osdu.core.common.util.Crc32c;
import org.opengroup.osdu.indexer.cache.DatastoreCredentialCache;
import java.util.Date;
public class DatastoreCredential extends GoogleCredentials {
private static final long serialVersionUID = 8344377091688956815L;
private static final JsonFactory JSON_FACTORY = new JacksonFactory();
private Iam iam;
private final TenantInfo tenant;
private final DatastoreCredentialCache cache;
protected DatastoreCredential(TenantInfo tenant, DatastoreCredentialCache cache) {
this.tenant = tenant;
this.cache = cache;
}
@Override
public AccessToken refreshAccessToken() {
String cacheKey = this.getCacheKey();
AccessToken accessToken = this.cache.get(cacheKey);
if (accessToken != null) {
return accessToken;
}
try {
SignJwtRequest signJwtRequest = new SignJwtRequest();
signJwtRequest.setPayload(this.getPayload());
String serviceAccountName = String.format("projects/-/serviceAccounts/%s", this.tenant.getServiceAccount());
SignJwt signJwt = this.getIam().projects().serviceAccounts().signJwt(serviceAccountName, signJwtRequest);
SignJwtResponse signJwtResponse = signJwt.execute();
String signedJwt = signJwtResponse.getSignedJwt();
accessToken = new AccessToken(signedJwt, DateUtils.addSeconds(new Date(), 3600));
this.cache.put(cacheKey, accessToken);
return accessToken;
} catch (Exception e) {
throw new RuntimeException("Error creating datastore credential", e);
}
}
private String getPayload() {
JsonObject payload = new JsonObject();
payload.addProperty("iss", this.tenant.getServiceAccount());
payload.addProperty("sub", this.tenant.getServiceAccount());
payload.addProperty("aud", "https://datastore.googleapis.com/google.datastore.v1.Datastore");
payload.addProperty("iat", System.currentTimeMillis() / 1000);
return payload.toString();
}
protected void setIam(Iam iam) {
this.iam = iam;
}
private Iam getIam() throws Exception {
if (this.iam == null) {
Iam.Builder builder = new Iam.Builder(GoogleNetHttpTransport.newTrustedTransport(), JSON_FACTORY,
GoogleCredential.getApplicationDefault()).setApplicationName("Search Service");
this.iam = builder.build();
}
return this.iam;
}
private String getCacheKey() {
return Crc32c.hashToBase64EncodedString(String.format("datastoreCredential:%s", this.tenant.getName()));
}
}
\ No newline at end of file
provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java deleted 100644 → 0
+ 0
− 67
View file @ c081dd73
// Copyright 2017-2019, Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.indexer.persistence;
import com.google.api.gax.retrying.RetrySettings;
import com.google.cloud.TransportOptions;
import com.google.cloud.datastore.Datastore;
import com.google.cloud.datastore.DatastoreOptions;
import com.google.cloud.http.HttpTransportOptions;
import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
import org.opengroup.osdu.indexer.cache.DatastoreCredentialCache;
import org.springframework.stereotype.Component;
import org.threeten.bp.Duration;
import javax.inject.Inject;
import java.util.HashMap;
import java.util.Map;
@Component
public class DatastoreFactory {
@Inject
private DatastoreCredentialCache cache;
private static Map<String, Datastore> DATASTORE_CLIENTS = new HashMap<>();
private static final RetrySettings RETRY_SETTINGS = RetrySettings.newBuilder()
.setMaxAttempts(6)
.setInitialRetryDelay(Duration.ofSeconds(10))
.setMaxRetryDelay(Duration.ofSeconds(32))
.setRetryDelayMultiplier(2.0)
.setTotalTimeout(Duration.ofSeconds(50))
.setInitialRpcTimeout(Duration.ofSeconds(50))
.setRpcTimeoutMultiplier(1.0)
.setMaxRpcTimeout(Duration.ofSeconds(50))
.build();
private static final TransportOptions TRANSPORT_OPTIONS = HttpTransportOptions.newBuilder()
.setReadTimeout(30000)
.build();
public Datastore getDatastoreInstance(TenantInfo tenantInfo) {
if (DATASTORE_CLIENTS.get(tenantInfo.getName()) == null) {
Datastore googleDatastore = DatastoreOptions.newBuilder()
.setCredentials(new DatastoreCredential(tenantInfo, this.cache))
.setRetrySettings(RETRY_SETTINGS)
.setTransportOptions(TRANSPORT_OPTIONS)
.setNamespace(tenantInfo.getName())
.setProjectId(tenantInfo.getProjectId())
.build().getService();
DATASTORE_CLIENTS.put(tenantInfo.getName(), googleDatastore);
}
return DATASTORE_CLIENTS.get(tenantInfo.getName());
}
}
provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryDatastore.java
+ 2
− 1
View file @ 3d192d14
......@@ -26,6 +26,7 @@ import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.provider.interfaces.IKmsClient;
import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository;
import org.opengroup.osdu.core.common.search.Preconditions;
import org.opengroup.osdu.core.gcp.multitenancy.DatastoreFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import javax.inject.Inject;
......@@ -52,7 +53,7 @@ public class ElasticRepositoryDatastore implements IElasticRepository {
@Override
public ClusterSettings getElasticClusterSettings(TenantInfo tenantInfo) {
Datastore googleDatastore = this.datastoreFactory.getDatastoreInstance(tenantInfo);
Datastore googleDatastore = this.datastoreFactory.getDatastore(tenantInfo);
Key key = googleDatastore.newKeyFactory().setKind(ELASTIC_DATASTORE_KIND).newKey(ELASTIC_DATASTORE_ID);
Entity datastoreEntity = googleDatastore.get(key);
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment