Commit 901c6537 authored by Matt Wise's avatar Matt Wise
Browse files

Merge remote-tracking branch 'upstream/master' into aws-integration

parents 465403d6 cba227ef
Pipeline #60812 failed with stages
in 35 minutes and 20 seconds
......@@ -21,6 +21,11 @@ variables:
MAVEN_PROJECTS: "-pl partition-core,provider/partition-gcp"
OSDU_GCP_TEST_SUBDIR: testing/$OSDU_GCP_SERVICE-test-$OSDU_GCP_VENDOR
OSDU_GCP_HELM_PACKAGE_CHARTS: "devops/gcp/deploy devops/gcp/configmap"
OSDU_GCP_HELM_NAMESPACE: default
OSDU_GCP_HELM_CONFIG_SERVICE_VARS: "--set data.partition_admin_accounts=$OSDU_GCP_PARTITION_ADMIN_ACCOUNTS --set data.google_cloud_project=$OSDU_GCP_PROJECT --set data.google_audiences=$GOOGLE_AUDIENCE --set data.log_level=INFO --set data.key_ring=$OSDU_GCP_PARTITION_KEY_RING --set data.kms_key=$OSDU_GCP_PARTITION_KMS_KEY"
OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS: "--set data.image=$CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA --set data.serviceAccountName=workload-identity-partition"
OSDU_GCP_HELM_CONFIG_SERVICE: partition-config
OSDU_GCP_HELM_DEPLOYMENT_SERVICE: partition-deploy
include:
- project: "osdu/platform/ci-cd-pipelines"
......@@ -45,7 +50,10 @@ include:
file: "cloud-providers/ibm.yml"
- project: "osdu/platform/ci-cd-pipelines"
file: "cloud-providers/osdu-gcp-cloudrun.yml"
file: "cloud-providers/osdu-gcp-gke.yml"
- project: "osdu/platform/ci-cd-pipelines"
file: 'cloud-providers/osdu-gcp-global.yml'
- project: "osdu/platform/ci-cd-pipelines"
file: "publishing/pages.yml"
......
......@@ -25,4 +25,4 @@ Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
\ No newline at end of file
limitations under the License.
......@@ -3,6 +3,8 @@ global:
# Service(s) Replica Count
replicaCount: 2
nodepool: services
isAutoscalingEnabled: false
################################################################################
# Specify the Gitlab branch being used for image creation
......
......@@ -28,6 +28,10 @@ spec:
app: {{ .Chart.Name }}
aadpodidbinding: osdu-identity
spec:
{{- if .Values.global.isAutoscalingEnabled }}
nodeSelector:
nodepool: {{ .Values.global.nodepool }}
{{- end }}
volumes:
- name: azure-keyvault
csi:
......
......@@ -14,6 +14,8 @@
global:
replicaCount: 1
nodepool: services
isAutoscalingEnabled: false
image:
repository: community.opengroup.org:5555/osdu/platform/system/partition
......
......@@ -7,7 +7,7 @@ metadata:
namespace: "{{ .Release.Namespace }}"
data:
GOOGLE_CLOUD_PROJECT: "{{ .Values.data.google_cloud_project }}"
PARTITION_ADMIN_ACCOUNT: "{{ .Values.data.partition_admin_account }}"
PARTITION_ADMIN_ACCOUNTS: "{{ .Values.data.partition_admin_accounts }}"
GOOGLE_AUDIENCES: "{{ .Values.data.google_audiences }}"
KEY_RING: "{{ .Values.data.key_ring }}"
KMS_KEY: "{{ .Values.data.kms_key }}"
......
data:
google_cloud_project: ""
partition_admin_account: ""
partition_admin_accounts: ""
google_audiences: ""
key_ring: ""
kms_key: ""
......
......@@ -4,7 +4,7 @@
data:
requests_cpu: "0.25"
requests_memory: "128M"
requests_memory: "256M"
limits_cpu: "1"
limits_memory: "1G"
serviceAccountName: ""
......
......@@ -15,6 +15,8 @@ tags:
description: Partition Api
- name: health-check
description: Health Check
- name: info
description: "Version info endpoint"
paths:
/actuator/health:
get:
......@@ -214,6 +216,21 @@ paths:
security:
- JWT:
- global
/info:
get:
tags:
- info
summary: "Version info"
description: "For deployment available public `/info` endpoint, \
\ which provides build and git related information."
operationId: "Version info"
produces:
- "application/json"
responses:
200:
description: "Version info."
schema:
$ref: "#/definitions/VersionInfo"
securityDefinitions:
JWT:
type: apiKey
......@@ -241,6 +258,46 @@ definitions:
cosmos-connection:
sensitive: true
value: 'cosmos-connection'
VersionInfo:
type: "object"
properties:
groupId:
type: "string"
description: "Maven artifact group ID."
actifactId:
type: "string"
description: "Maven artifact ID."
version:
type: "string"
description: "Maven artifact version"
buildTime:
type: "string"
description: "Maven artifact build time"
branch:
type: "string"
description: "Current git branch"
commitId:
type: "string"
description: "Latest commit hash"
commitMessage:
type: "string"
description: "Latest commit message"
connectedOuterServices:
type: "array"
description: "Connected outer services information"
items:
$ref: "#/definitions/ConnectedOuterService"
description: "Version info."
ConnectedOuterService:
type: "object"
properties:
name:
type: "string"
description: "Connected outer service name."
version:
type: "string"
description: "Connected outer service version."
description: "Connected outer service information."
Property:
type: object
properties:
......
......@@ -10,7 +10,7 @@
* [Update an existing partition](#update-partition)
* [Delete an existing partition](#delete-partition)
* [List of partitions](#list-partition)
* [Version info endpoint](#version-info-endpoint)
## Introduction <a name="introduction"></a>
Partition service is responsible for creating and retrieving the partition specific properties (secret and non-secret) on behalf of other services.
......@@ -200,4 +200,40 @@ A sample output is shown below.
</details>
[Back to Table of Contents](#TOC)
\ No newline at end of file
[Back to Table of Contents](#TOC)
## Version info endpoint
For deployment available public `/info` endpoint, which provides build and git related information.
#### Example response:
```json
{
"groupId": "org.opengroup.osdu",
"artifactId": "storage-gcp",
"version": "0.10.0-SNAPSHOT",
"buildTime": "2021-07-09T14:29:51.584Z",
"branch": "feature/GONRG-2681_Build_info",
"commitId": "7777",
"commitMessage": "Added copyright to version info properties file",
"connectedOuterServices": [
{
"name": "elasticSearch",
"version":"..."
},
{
"name": "postgresSql",
"version":"..."
},
{
"name": "redis",
"version":"..."
}
]
}
```
This endpoint takes information from files, generated by `spring-boot-maven-plugin`,
`git-commit-id-plugin` plugins. Need to specify paths for generated files to matching
properties:
- `version.info.buildPropertiesPath`
- `version.info.gitPropertiesPath`
[Back to table of contents](#TOC)
/*
* Copyright 2021 Google LLC
* Copyright 2021 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.partition.api;
import java.io.IOException;
import org.opengroup.osdu.core.common.info.VersionInfoBuilder;
import org.opengroup.osdu.core.common.model.info.VersionInfo;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@RestController
@RequestMapping
public class InfoApi {
@Autowired
private VersionInfoBuilder versionInfoBuilder;
@GetMapping(value = "/info", produces = MediaType.APPLICATION_JSON_VALUE)
public VersionInfo info() throws IOException {
return versionInfoBuilder.buildVersionInfo();
}
}
......@@ -26,7 +26,7 @@
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<os-core-common.version>0.10.0</os-core-common.version>
<os-core-common.version>0.11.0-SNAPSHOT</os-core-common.version>
<org.springframework.boot.version>2.4.5</org.springframework.boot.version>
</properties>
......@@ -86,9 +86,40 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<version>${org.springframework.boot.version}</version>
<executions>
<execution>
<id>build-info</id>
<goals>
<goal>build-info</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</pluginManagement>
<plugins>
<plugin>
<groupId>pl.project13.maven</groupId>
<artifactId>git-commit-id-plugin</artifactId>
<version>4.0.5</version>
<executions>
<execution>
<goals>
<goal>revision</goal>
</goals>
</execution>
</executions>
<configuration>
<verbose>true</verbose>
<dateFormat>yyyy-MM-dd'T'HH:mm:ssZ</dateFormat>
<generateGitPropertiesFile>true</generateGitPropertiesFile>
<generateGitPropertiesFilename>
${project.build.outputDirectory}/git.properties
</generateGitPropertiesFilename>
</configuration>
</plugin>
</plugins>
</build>
<modules>
......
......@@ -47,6 +47,7 @@ public class AADSecurityConfig extends WebSecurityConfigurerAdapter {
"/configuration/security",
"/swagger",
"/swagger-ui.html",
"/info",
"/webjars/**").permitAll()
.anyRequest().authenticated()
.and()
......
......@@ -22,7 +22,7 @@ In order to run the service locally or remotely, you will need to have the follo
| `AUTHORIZE_API` | ex `https://entitlements.com/entitlements/v1` | Entitlements API endpoint | no | output of infrastructure deployment |
| `GOOGLE_CLOUD_PROJECT` | ex `osdu-cicd-epam` | Google Cloud Project Id| no | output of infrastructure deployment |
| `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials |
| `PARTITION_ADMIN_ACCOUNT` | ex `admin@domen.iam.gserviceaccount.com` | Partition Admin account email | no | - |
| `PARTITION_ADMIN_ACCOUNTS` | ex `admin@domen.iam.gserviceaccount.com,osdu-gcp-sa,workload-identity` | List of partition admin account emails, could be in full form like `admin@domen.iam.gserviceaccount.com` or in `starts with` pattern like `osdu-gcp-sa`| no | - |
| `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
| `KEY_RING` | ex `csqp` | A key ring holds keys in a specific Google Cloud location and permit us to manage access control on groups of keys | yes | https://cloud.google.com/kms/docs/resource-hierarchy#key_rings |
| `KMS_KEY` | ex `partitionService` | A key exists on one key ring linked to a specific location. | yes | https://cloud.google.com/kms/docs/resource-hierarchy#key_rings |
......
/*
* Copyright 2021 Google LLC
* Copyright 2021 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.partition.provider.gcp.config;
import com.google.api.client.googleapis.auth.oauth2.GoogleIdTokenVerifier;
import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
import com.google.api.client.json.jackson2.JacksonFactory;
import java.io.IOException;
import java.security.GeneralSecurityException;
import java.util.Collections;
import lombok.RequiredArgsConstructor;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
@RequiredArgsConstructor
public class IDTokenVerifierConfiguration {
private final PropertiesConfiguration configuration;
@Bean
public GoogleIdTokenVerifier buildTokenVerifier() throws GeneralSecurityException, IOException {
return new GoogleIdTokenVerifier.Builder(
GoogleNetHttpTransport.newTrustedTransport(),
JacksonFactory.getDefaultInstance())
.setAudience(Collections.singleton(configuration.getGoogleAudiences()))
.build();
}
}
......@@ -17,6 +17,9 @@
package org.opengroup.osdu.partition.provider.gcp.config;
import java.util.List;
import java.util.Objects;
import javax.annotation.PostConstruct;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
......@@ -30,9 +33,21 @@ public class PropertiesConfiguration {
private String googleAudiences;
private String partitionAdminAccount;
private List<String> partitionAdminAccounts;
private String googleCloudProject;
private int cacheExpiration;
private int cacheMaxSize;
private String serviceAccountTail;
@PostConstruct
public void setUp() {
if (Objects.isNull(serviceAccountTail) || serviceAccountTail.isEmpty()) {
this.serviceAccountTail = googleCloudProject + ".iam.gserviceaccount.com";
}
}
}
......@@ -19,9 +19,7 @@ package org.opengroup.osdu.partition.provider.gcp.security;
import com.google.api.client.googleapis.auth.oauth2.GoogleIdToken;
import com.google.api.client.googleapis.auth.oauth2.GoogleIdTokenVerifier;
import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
import com.google.api.client.json.jackson2.JacksonFactory;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
......@@ -43,6 +41,8 @@ public class AuthorizationService implements IAuthorizationService {
private final DpsHeaders headers;
private final GoogleIdTokenVerifier verifier;
@Override
public boolean isDomainAdminServiceAccount() {
if (Objects.isNull(headers.getAuthorization()) || headers.getAuthorization().isEmpty()) {
......@@ -50,13 +50,6 @@ public class AuthorizationService implements IAuthorizationService {
}
String email = null;
try {
GoogleIdTokenVerifier verifier =
new GoogleIdTokenVerifier.Builder(
GoogleNetHttpTransport.newTrustedTransport(),
JacksonFactory.getDefaultInstance())
.setAudience(Collections.singleton(configuration.getGoogleAudiences()))
.build();
String authorization = headers.getAuthorization().replace("Bearer ", "");
GoogleIdToken googleIdToken = verifier.verify(authorization);
if (Objects.isNull(googleIdToken)) {
......@@ -64,25 +57,37 @@ public class AuthorizationService implements IAuthorizationService {
throw AppException.createUnauthorized("Unauthorized. The JWT token could not be validated");
}
email = googleIdToken.getPayload().getEmail();
String partitionAdminAccount = configuration.getPartitionAdminAccount();
if (Objects.nonNull(partitionAdminAccount) && !partitionAdminAccount.isEmpty()) {
if (email.equals(partitionAdminAccount)) {
return true;
} else {
throw AppException
.createUnauthorized(String.format("Unauthorized. The user %s is untrusted.", email));
}
List<String> partitionAdminAccounts = configuration.getPartitionAdminAccounts();
if (Objects.nonNull(partitionAdminAccounts) && !partitionAdminAccounts.isEmpty()) {
return isAllowedAccount(email);
} else {
if (StringUtils.endsWithIgnoreCase(email, "gserviceaccount.com")) {
if (StringUtils.endsWith(email, configuration.getServiceAccountTail())) {
return true;
} else {
throw AppException.createUnauthorized(
String.format("Unauthorized. The user %s is not Service Principal", email));
}
}
} catch (AppException e){
throw e;
} catch (Exception ex) {
log.warn(String.format("User %s is not unauthorized. %s.", email, ex));
throw AppException.createUnauthorized("Unauthorized. The JWT token could not be validated");
}
}
private boolean isAllowedAccount(String accountEmail) {
if (StringUtils.endsWith(accountEmail, configuration.getServiceAccountTail())) {
for (String partitionAdmin : configuration.getPartitionAdminAccounts()) {
if (partitionAdmin.equals(accountEmail)) {
return true;
}
if (StringUtils.startsWith(accountEmail, partitionAdmin)) {
return true;
}
}
}
throw AppException
.createUnauthorized(String.format("Unauthorized. The user %s is untrusted.", accountEmail));
}
}
......@@ -18,7 +18,7 @@ KEY_RING=${key-ring}
KMS_KEY=${kms-key}
GOOGLE_CLOUD_PROJECT=${google-cloud-project}
google-audiences=123.apps.googleusercontent.com
partition-admin-account=admin@domen.iam.gserviceaccount.com
partition-admin-accounts=osdu-gcp-sa
#logging configuration
logging.level.org.springframework.web=${LOG_LEVEL:DEBUG}
......
/*
* Copyright 2021 Google LLC
* Copyright 2021 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.partition.provider.gcp.security;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.when;
import com.google.api.client.googleapis.auth.oauth2.GoogleIdToken;
import com.google.api.client.googleapis.auth.oauth2.GoogleIdToken.Payload;
import com.google.api.client.googleapis.auth.oauth2.GoogleIdTokenVerifier;
import com.google.common.collect.ImmutableList;
import java.io.IOException;
import java.security.GeneralSecurityException;
import java.util.List;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.theories.DataPoints;
import org.junit.experimental.theories.FromDataPoints;
import org.junit.experimental.theories.Theories;
import org.junit.experimental.theories.Theory;
import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.partition.provider.gcp.config.PropertiesConfiguration;
@RunWith(Theories.class)
public class AuthorizationServiceTest {
private final String token = "abc";
private final String serviceAccountTail = "project-id.iam.gserviceaccount.com";
private final List<String> partitionAdminAccounts = ImmutableList.of("osdu-gcp-sa", "service.account@project-id.iam.gserviceaccount.com");
@Rule
public ExpectedException exceptionRule = ExpectedException.none();
@DataPoints("VALID_ACCOUNTS")
public static List<String> validTestSet() {
return ImmutableList.of(
"osdu-gcp-sa-first@project-id.iam.gserviceaccount.com",
"osdu-gcp-sa-second@project-id.iam.gserviceaccount.com",
"osdu-gcp-sa-third@project-id.iam.gserviceaccount.com",
"osdu-gcp-sa-fourth@project-id.iam.gserviceaccount.com");
}
@DataPoints("NOT_VALID_ACCOUNTS")
public static List<String> notValidTestSet() {
return ImmutableList.of(
"osdu-gcp-sa-first@google.com",
"osdu-gcp-sa-second@project-id.iam.gserviceaccount.com.not.valid",
"user-osdu-gcp-sa-third@project-id.iam.gserviceaccount.com");
}
@Mock
private PropertiesConfiguration configuration;