Commit 6280af16 authored by Alok Joshi's avatar Alok Joshi
Browse files

search policy integration for query api

parent 830b4294
......@@ -103,6 +103,8 @@ spec:
value: debug
- name: partition_service_endpoint
value: http://partition/api/partition/v1
- name: policy_service_endpoint
value: http://policy-service/api/policy/v1
- name: azure_istioauth_enabled
value: "true"
- name: azure_activedirectory_AppIdUri
......
Search service now supports data authorization checks via Policy service. Policy service allows dynamic policy evaluation on user requests and can
be configured per partition.
By default, Search service utilizes Elastic's authorization query filter for data authorization checks. CSP must opt-in to delegate data access to Policy Service.
Here are steps to enable Policy service for a provider:
- Enable policy configuration for desired partition:
```
PATCH /api/partition/v1/partitions/{partitionId}
{
"properties": {
"policy-service-enabled": {
"sensitive": false,
"value": "true"
}
}
}
```
- Register policy for Search service, please look at policy service [documentation](https://community.opengroup.org/osdu/platform/security-and-compliance/policy#add-policy) for more details.
- Add and provide values for following runtime configuration in `application.properties`
```
service.policy.enabled=true
POLICY_API=${policy_service_endpoint}
PARTITION_API=${partition_service_endpoint}
```
- This is an experimental feature and at this moment has following limitations
1. Support is added only for `query` api. Support for `query_with_cursor` will be added in the future
2. If the query has `returnedFields` set, it must contain all `acl, kind, legal` and `id`
3. In the current implementation, totalCount represents the number of records matching user query before the search policy is applied
......@@ -320,7 +320,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
<version>0.3.19</version>
<version>0.6.10</version>
</dependency>
</dependencies>
......
// Copyright © 2020 Amazon Web Services
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.search.provider.aws.cache;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.entitlements.Groups;
import org.opengroup.osdu.core.common.cache.RedisCache;
import org.opengroup.osdu.core.common.util.Crc32c;
import org.opengroup.osdu.search.cache.GroupCache;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
@Component("groupsCache")
public class GroupCacheImpl extends RedisCache<String, Groups> implements GroupCache {
public GroupCacheImpl(@Value("${aws.elasticache.cluster.endpoint}") final String REDIS_GROUP_HOST, @Value("${aws.elasticache.cluster.port}") final String REDIS_GROUP_PORT) {
super(REDIS_GROUP_HOST, Integer.parseInt(REDIS_GROUP_PORT), 30, String.class, Groups.class);
}
public String getCacheKey(DpsHeaders headers) {
String key = String.format("entitlement-groups:%s:%s", headers.getPartitionIdWithFallbackToAccountId(),
headers.getAuthorization());
return Crc32c.hashToBase64EncodedString(key);
}
}
......@@ -43,9 +43,12 @@ import org.opengroup.osdu.core.common.model.entitlements.AclRole;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.search.*;
import org.opengroup.osdu.search.policy.service.IPolicyService;
import org.opengroup.osdu.search.policy.service.PartitionPolicyStatusService;
import org.opengroup.osdu.search.provider.aws.service.FieldMappingTypeService;
import org.opengroup.osdu.search.provider.interfaces.IProviderHeaderService;
import org.opengroup.osdu.search.util.CrossTenantUtils;
import org.springframework.beans.factory.annotation.Autowired;
import javax.inject.Inject;
import javax.servlet.http.HttpServletResponse;
......@@ -69,6 +72,10 @@ abstract class QueryBase {
private CrossTenantUtils crossTenantUtils;
@Inject
private FieldMappingTypeService fieldMappingTypeService;
@Autowired(required = false)
private IPolicyService iPolicyService;
@Inject
private PartitionPolicyStatusService statusService;
static final String AGGREGATION_NAME = "agg";
private static final String GEO_SHAPE_INDEXED_TYPE = "geo_shape";
......@@ -115,27 +122,32 @@ abstract class QueryBase {
}
}
// apply authorization filters
String groups = dpsHeaders.getHeaders().get(providerHeaderService.getDataGroupsHeader());
String[] groupArray = groups.trim().split("\\s*,\\s*");
if (asOwner) {
authorizationQueryBuilder = boolQuery().minimumShouldMatch("1").should(termsQuery(
AclRole.OWNERS.getPath(), groupArray));
} else {
authorizationQueryBuilder = boolQuery().minimumShouldMatch("1").should(termsQuery(RecordMetaAttribute.X_ACL.getValue(), groupArray));
}
if (textQueryBuilder != null) {
queryBuilder = boolQuery().must(textQueryBuilder);
}
if (spatialQueryBuilder != null) {
queryBuilder = queryBuilder != null ? boolQuery().must(queryBuilder).must(spatialQueryBuilder) : boolQuery().must(spatialQueryBuilder);
}
if (authorizationQueryBuilder != null) {
queryBuilder = queryBuilder != null ? boolQuery().must(queryBuilder).must(authorizationQueryBuilder) : boolQuery().must(authorizationQueryBuilder);
}
return queryBuilder;
if(this.iPolicyService != null && this.statusService.policyEnabled(this.dpsHeaders.getPartitionId())) {
return queryBuilder;
} else {
// apply authorization filters
String groups = dpsHeaders.getHeaders().get(providerHeaderService.getDataGroupsHeader());
String[] groupArray = groups.trim().split("\\s*,\\s*");
if (asOwner) {
authorizationQueryBuilder = boolQuery().minimumShouldMatch("1").should(termsQuery(
AclRole.OWNERS.getPath(), groupArray));
} else {
authorizationQueryBuilder = boolQuery().minimumShouldMatch("1").should(termsQuery(RecordMetaAttribute.X_ACL.getValue(), groupArray));
}
if (authorizationQueryBuilder != null) {
queryBuilder = queryBuilder != null ? boolQuery().must(queryBuilder).must(authorizationQueryBuilder) : boolQuery().must(authorizationQueryBuilder);
}
return queryBuilder;
}
}
private QueryBuilder getSimpleQuery(String searchQuery) {
......
......@@ -28,6 +28,7 @@ import org.opengroup.osdu.search.config.SearchConfigurationProperties;
import org.opengroup.osdu.search.logging.AuditLogger;
import org.opengroup.osdu.search.provider.interfaces.IQueryService;
import org.opengroup.osdu.search.util.ElasticClientHandler;
import org.opengroup.osdu.search.util.QueryResponseUtil;
import org.springframework.stereotype.Service;
import javax.inject.Inject;
......@@ -46,6 +47,8 @@ public class QueryServiceAwsImpl extends QueryBase implements IQueryService {
private AuditLogger auditLogger;
@Inject
private SearchConfigurationProperties configurationProperties;
@Inject
private QueryResponseUtil queryResponseUtil;
@Override
public QueryResponse queryIndex(QueryRequest searchRequest) throws IOException {
......@@ -77,7 +80,7 @@ public class QueryServiceAwsImpl extends QueryBase implements IQueryService {
queryResponse.setTotalCount(searchResponse.getHits().getTotalHits().value);
if (results != null) {
queryResponse.setAggregations(aggregations);
queryResponse.setResults(results);
queryResponse.setResults(queryResponseUtil.getQueryResponseResults(results));
}
return queryResponse;
}
......
// Copyright © 2020 Amazon Web Services
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.search.provider.aws.util;
import com.amazonaws.services.simplesystemsmanagement.AWSSimpleSystemsManagement;
import com.amazonaws.services.simplesystemsmanagement.AWSSimpleSystemsManagementClientBuilder;
import com.amazonaws.services.simplesystemsmanagement.model.GetParameterRequest;
import com.amazonaws.services.simplesystemsmanagement.model.GetParameterResult;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.Setter;
import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.opengroup.osdu.core.aws.entitlements.ServicePrincipal;
import org.opengroup.osdu.core.aws.iam.IAMConfig;
import org.opengroup.osdu.core.aws.secrets.SecretsManager;
import com.amazonaws.auth.AWSCredentialsProvider;
import javax.annotation.PostConstruct;
@Component
public class ServiceAccountJwtClientImpl implements IServiceAccountJwtClient {
@Value("${aws.region}")
@Getter()
@Setter(AccessLevel.PROTECTED)
public String amazonRegion;
@Value("${aws.ssm}")
@Getter()
@Setter(AccessLevel.PROTECTED)
public Boolean ssmEnabled;
@Value("${aws.environment}")
@Getter()
@Setter(AccessLevel.PROTECTED)
public String environment;
private String awsOauthCustomScope;
String client_credentials_secret;
String client_credentials_clientid;
ServicePrincipal sp;
private AWSCredentialsProvider amazonAWSCredentials;
private AWSSimpleSystemsManagement ssmManager;
@PostConstruct
public void init() {
if (ssmEnabled) {
SecretsManager sm = new SecretsManager();
String oauth_token_url = "/osdu/" + environment + "/oauth-token-uri";
String oauth_custom_scope = "/osdu/" + environment + "/oauth-custom-scope";
String client_credentials_client_id = "/osdu/" + environment + "/client-credentials-client-id";
String client_secret_key = "client_credentials_client_secret";
String client_secret_secretName = "/osdu/" + environment + "/client_credentials_secret";
amazonAWSCredentials = IAMConfig.amazonAWSCredentials();
ssmManager = AWSSimpleSystemsManagementClientBuilder.standard()
.withCredentials(amazonAWSCredentials)
.withRegion(amazonRegion)
.build();
client_credentials_clientid = getSsmParameter(client_credentials_client_id);
client_credentials_secret = sm.getSecret(client_secret_secretName,amazonRegion,client_secret_key);
String tokenUrl = getSsmParameter(oauth_token_url);
awsOauthCustomScope = getSsmParameter(oauth_custom_scope);
sp = new ServicePrincipal(amazonRegion,environment,tokenUrl,awsOauthCustomScope);
}
}
@Override
public String getIdToken(String tenantName) {
String token= sp.getServicePrincipalAccessToken(client_credentials_clientid,client_credentials_secret);
return token;
}
private String getSsmParameter(String parameterKey) {
GetParameterRequest paramRequest = (new GetParameterRequest()).withName(parameterKey).withWithDecryption(true);
GetParameterResult paramResult = ssmManager.getParameter(paramRequest);
return paramResult.getParameter().getValue();
}
}
......@@ -50,6 +50,10 @@ AGGREGATION_SIZE=1000
awsParameterStorePropertySource.enabled=true
## AWS ElastiCache configuration
aws.elasticache.cluster.endpoint=${CACHE_CLUSTER_ENDPOINT}
aws.elasticache.cluster.port=${CACHE_CLUSTER_PORT}
aws.ssm=${SSM_ENABLED}
aws.ssm.prefix=/osdu/${ENVIRONMENT}
......@@ -61,4 +65,11 @@ server.ssl.key-store-type=PKCS12
server.ssl.key-store=${SSL_KEY_STORE_PATH:/certs/osduonaws.p12}
server.ssl.key-alias=${SSL_KEY_ALIAS:osduonaws}
server.ssl.key-password=${SSL_KEY_PASSWORD:}
server.ssl.key-store-password=${SSL_KEY_STORE_PASSWORD:}
\ No newline at end of file
server.ssl.key-store-password=${SSL_KEY_STORE_PASSWORD:}
# Policy service properties
service.policy.enabled=false
POLICY_API=${ENTITLEMENTS_BASE_URL}/api/policy/v1
POLICY_ID=search
PARTITION_API=${ENTITLEMENTS_BASE_URL}/api/partition/v1
aws.environment=${ENVIRONMENT}
\ No newline at end of file
......@@ -36,8 +36,8 @@
<failOnMissingWebXml>false</failOnMissingWebXml>
<project.main.basedir>${project.parent.basedir}</project.main.basedir>
<springboot.version>2.1.7.RELEASE</springboot.version>
<osdu.corelibazure.version>0.0.41</osdu.corelibazure.version>
<osdu.oscorecommon.version>0.3.16</osdu.oscorecommon.version>
<osdu.corelibazure.version>0.0.63</osdu.corelibazure.version>
<osdu.oscorecommon.version>0.6.9</osdu.oscorecommon.version>
<osdu.search-core.version>0.6.0-SNAPSHOT</osdu.search-core.version>
<spatial4j.version>0.7</spatial4j.version>
<jts-io-common.version>1.15.0</jts-io-common.version>
......@@ -79,7 +79,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
<version>0.3.16</version>
<version>0.6.10</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
......
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.search.provider.azure.cache;
import org.opengroup.osdu.core.common.cache.ICache;
import org.opengroup.osdu.core.common.model.entitlements.Groups;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
public interface GroupCache extends ICache<String, Groups> {
String getCacheKey(DpsHeaders headers);
}
......@@ -20,10 +20,10 @@ import org.opengroup.osdu.core.common.cache.ICache;
import org.opengroup.osdu.core.common.model.entitlements.Groups;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.util.Crc32c;
import org.opengroup.osdu.search.provider.azure.cache.GroupCache;
import org.opengroup.osdu.search.cache.GroupCache;
import org.springframework.stereotype.Component;
@Component
@Component("groupsCache")
public class GroupCacheImpl implements GroupCache {
@Resource(name = "groupCache")
......
......@@ -17,6 +17,8 @@ package org.opengroup.osdu.search.provider.azure.config;
import org.opengroup.osdu.core.common.entitlements.EntitlementsAPIConfig;
import org.opengroup.osdu.core.common.entitlements.EntitlementsFactory;
import org.opengroup.osdu.core.common.entitlements.IEntitlementsFactory;
import org.opengroup.osdu.core.common.http.json.HttpResponseBodyMapper;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
......@@ -41,6 +43,9 @@ public class AzureBootstrapConfig {
@Value("${AUTHORIZE_API_KEY}")
private String entitlementsAPIKey;
@Autowired
private HttpResponseBodyMapper mapper;
@Bean
@Named("KEY_VAULT_URL")
public String getKeyVaultURL() {
......@@ -67,6 +72,6 @@ public class AzureBootstrapConfig {
.apiKey(entitlementsAPIKey)
.rootUrl(entitlementsAPIEndpoint)
.build();
return new EntitlementsFactory(apiConfig);
return new EntitlementsFactory(apiConfig, mapper);
}
}
......@@ -45,9 +45,12 @@ import org.opengroup.osdu.core.common.model.entitlements.AclRole;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.search.*;
import org.opengroup.osdu.search.policy.service.IPolicyService;
import org.opengroup.osdu.search.policy.service.PartitionPolicyStatusService;
import org.opengroup.osdu.search.provider.azure.service.FieldMappingTypeService;
import org.opengroup.osdu.search.provider.interfaces.IProviderHeaderService;
import org.opengroup.osdu.search.util.CrossTenantUtils;
import org.springframework.beans.factory.annotation.Autowired;
import javax.inject.Inject;
import javax.servlet.http.HttpServletResponse;
......@@ -69,6 +72,11 @@ abstract class QueryBase {
@Inject
private FieldMappingTypeService fieldMappingTypeService;
@Autowired(required = false)
private IPolicyService iPolicyService;
@Inject
private PartitionPolicyStatusService statusService;
static final String AGGREGATION_NAME = "agg";
private static final String GEO_SHAPE_INDEXED_TYPE = "geo_shape";
......@@ -114,19 +122,6 @@ abstract class QueryBase {
}
}
// apply authorization filters
//bypass for BYOC implementation only.
String groups = dpsHeaders.getHeaders().get(providerHeaderService.getDataGroupsHeader());
if (groups != null) {
String[] groupArray = groups.trim().split("\\s*,\\s*");
if (asOwner) {
authorizationQueryBuilder = boolQuery().minimumShouldMatch("1").should(termsQuery(
AclRole.OWNERS.getPath(), groupArray));
} else {
authorizationQueryBuilder = boolQuery().minimumShouldMatch("1").should(termsQuery(RecordMetaAttribute.X_ACL.getValue(), groupArray));
}
}
if (textQueryBuilder != null) {
queryBuilder = boolQuery().must(textQueryBuilder);
......@@ -134,11 +129,27 @@ abstract class QueryBase {
if (spatialQueryBuilder != null) {
queryBuilder = queryBuilder != null ? boolQuery().must(queryBuilder).must(spatialQueryBuilder) : boolQuery().must(spatialQueryBuilder);
}
if (authorizationQueryBuilder != null) {
queryBuilder = queryBuilder != null ? boolQuery().must(queryBuilder).must(authorizationQueryBuilder) : boolQuery().must(authorizationQueryBuilder);
}
return queryBuilder;
if(this.iPolicyService != null && this.statusService.policyEnabled(this.dpsHeaders.getPartitionId())) {
return queryBuilder;
} else {
// apply authorization filters
//bypass for BYOC implementation only.
String groups = dpsHeaders.getHeaders().get(providerHeaderService.getDataGroupsHeader());
if (groups != null) {
String[] groupArray = groups.trim().split("\\s*,\\s*");
if (asOwner) {
authorizationQueryBuilder = boolQuery().minimumShouldMatch("1").should(termsQuery(
AclRole.OWNERS.getPath(), groupArray));
} else {
authorizationQueryBuilder = boolQuery().minimumShouldMatch("1").should(termsQuery(RecordMetaAttribute.X_ACL.getValue(), groupArray));
}
}
if (authorizationQueryBuilder != null) {
queryBuilder = queryBuilder != null ? boolQuery().must(queryBuilder).must(authorizationQueryBuilder) : boolQuery().must(authorizationQueryBuilder);
}
return queryBuilder;
}
}
private QueryBuilder getSimpleQuery(String searchQuery) {
......
......@@ -20,7 +20,6 @@ import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.aggregations.support.ValueType;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.search.*;
......@@ -28,6 +27,7 @@ import org.opengroup.osdu.search.config.SearchConfigurationProperties;
import org.opengroup.osdu.search.logging.AuditLogger;
import org.opengroup.osdu.search.provider.interfaces.IQueryService;
import org.opengroup.osdu.search.util.ElasticClientHandler;
import org.opengroup.osdu.search.util.QueryResponseUtil;
import org.springframework.stereotype.Service;
import javax.inject.Inject;
......@@ -44,6 +44,8 @@ public class QueryServiceImpl extends QueryBase implements IQueryService {
private AuditLogger auditLogger;
@Inject
private SearchConfigurationProperties configurationProperties;
@Inject
private QueryResponseUtil queryResponseUtil;
@Override
public QueryResponse queryIndex(QueryRequest searchRequest) throws IOException {
......@@ -71,7 +73,7 @@ public class QueryServiceImpl extends QueryBase implements IQueryService {
queryResponse.setTotalCount(searchResponse.getHits().getTotalHits().value);
if (results != null) {
queryResponse.setAggregations(aggregations);
queryResponse.setResults(results);
queryResponse.setResults(queryResponseUtil.getQueryResponseResults(results));
}
return queryResponse;
}
......
// Copyright 2017-2019, Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.search.provider.azure.service;
import java.util.Set;
import javax.inject.Inject;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.entitlements.IEntitlementsFactory;
import org.opengroup.osdu.core.common.entitlements.IEntitlementsService;
import org.opengroup.osdu.core.common.http.HttpResponse;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.entitlements.EntitlementsException;
import org.opengroup.osdu.core.common.model.entitlements.Groups;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.search.provider.azure.cache.GroupCache;
import org.springframework.stereotype.Service;
@Service
public class EntitlementsAndCacheServiceImpl implements EntitlementsAndCacheService {
private static final String ERROR_REASON = "Access denied";
private static final String ERROR_MSG = "The user is not authorized to perform this action";
@Inject
private IEntitlementsFactory factory;
@Inject
private GroupCache cache;
@Inject
private JaxRsDpsLog logger;