Commit 6f2bfc41 authored by Abhishek Kumar (SLB)'s avatar Abhishek Kumar (SLB)
Browse files
parents 791ec366 87e1753c
boto3==1.17.1
botocore==1.20.1
PyJWT==1.7.1
PyJWT==2.4.0
requests==2.23.00
\ No newline at end of file
......@@ -7,6 +7,7 @@
# (both environments):
# - DATA_PARTITION
# - SCHEMA_URL
# - ENTITLEMENTS_HOST
# (for gcp):
# - AUDIENCES
# (for onprem):
......@@ -19,12 +20,10 @@ set -e
source ./validate-env.sh "DATA_PARTITION"
source ./validate-env.sh "SCHEMA_URL"
source ./validate-env.sh "ENTITLEMENTS_HOST"
# FIXME find a better solution about a sidecar container readiness
echo "Waiting for a sidecar container is provisioned"
sleep 10
bootstrap_schema_onprem() {
bootstrap_schema_gettoken_onprem() {
echo "Waiting for a sidecar container is provisioned"
ID_TOKEN="$(curl --location --request POST "${OPENID_PROVIDER_URL}/protocol/openid-connect/token" \
--header "Content-Type: application/x-www-form-urlencoded" \
......@@ -33,13 +32,10 @@ bootstrap_schema_onprem() {
--data-urlencode "client_id=${OPENID_PROVIDER_CLIENT_ID}" \
--data-urlencode "client_secret=${OPENID_PROVIDER_CLIENT_SECRET}" | jq -r ".id_token")"
export BEARER_TOKEN="Bearer ${ID_TOKEN}"
echo "Bootstrap Schema Service On Prem"
python3 ./scripts/DeploySharedSchemas.py -u "${SCHEMA_URL}"/api/schema-service/v1/schemas/system
}
bootstrap_schema_gcp() {
bootstrap_schema_gettoken_gcp() {
echo "Waiting for a sidecar container is provisioned"
BEARER_TOKEN=$(gcloud auth print-identity-token --audiences="${AUDIENCES}")
export BEARER_TOKEN
......@@ -49,10 +45,27 @@ bootstrap_schema_gcp() {
# FIXME find a better solution about datastore cleaning completion
sleep 5
}
echo "Bootstrap Schema Service On GCP"
python3 ./scripts/DeploySharedSchemas.py -u "${SCHEMA_URL}"/api/schema-service/v1/schemas/system
bootstrap_schema_prechek_env() {
status_code=$(curl --retry 1 --location -globoff --request POST \
"${ENTITLEMENTS_HOST}/api/entitlements/v2/tenant-provisioning" \
--write-out "%{http_code}" --silent --output "/dev/null"\
--header 'Content-Type: application/json' \
--header "data-partition-id: ${DATA_PARTITION}" \
--header "Authorization: ${BEARER_TOKEN}")
if [ "$status_code" == 200 ]
then
echo "$status_code: Entitlements provisioning completed successfully!"
else
echo "$status_code: Entitlements provisioning failed!"
exit 1
fi
}
bootstrap_schema_deploy_shared_schemas() {
python3 ./scripts/DeploySharedSchemas.py -u "${SCHEMA_URL}"/api/schema-service/v1/schemas/system
}
if [ "${ONPREM_ENABLED}" == "true" ]
......@@ -60,10 +73,22 @@ then
source ./validate-env.sh "OPENID_PROVIDER_URL"
source ./validate-env.sh "OPENID_PROVIDER_CLIENT_ID"
source ./validate-env.sh "OPENID_PROVIDER_CLIENT_SECRET"
bootstrap_schema_onprem
# Get credentials for onprem
bootstrap_schema_gettoken_onprem
else
source ./validate-env.sh "AUDIENCES"
bootstrap_schema_gcp
# Get credentials for GCP
bootstrap_schema_gettoken_gcp
fi
# Precheck entitlements
bootstrap_schema_prechek_env
# Deploy shared schemas
bootstrap_schema_deploy_shared_schemas
touch /tmp/bootstrap_ready
......@@ -9,6 +9,7 @@ data:
DATA_PARTITION: "{{ .Values.data.data_partition_id }}"
ONPREM_ENABLED: "{{ .Values.conf.on_prem_enabled }}"
SCHEMA_URL: "{{ .Values.data.schema_host }}"
ENTITLEMENTS_HOST: "{{ .Values.data.entitlements_host }}"
{{- if not .Values.conf.on_prem_enabled }}
AUDIENCES: "{{ .Values.data.google_audiences }}"
{{- end }}
......@@ -2,7 +2,6 @@ variables:
OSDU_GCP_ENABLE_BOOTSTRAP: "true"
OSDU_GCP_SERVICE: schema
OSDU_GCP_VENDOR: gcp
OSDU_GCP_TESTS_SUBDIR: testing/schema-test-core
OSDU_GCP_HELM_CONFIG_SERVICE: schema-config
OSDU_GCP_HELM_DEPLOYMENT_SERVICE: schema-deploy
OSDU_GCP_HELM_TIMEOUT: "--timeout 15m"
......@@ -25,6 +24,14 @@ variables:
# FIXME add value below for DEV2 pipeline
# OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS_DEV2: >
osdu-gcp-containerize-bootstrap-gitlab:
variables:
BUILD_PATH: devops/$OSDU_GCP_VENDOR/bootstrap-osdu-module/Dockerfile
osdu-gcp-containerize-bootstrap-gcr:
variables:
BUILD_PATH: devops/$OSDU_GCP_VENDOR/bootstrap-osdu-module/Dockerfile
# REFACTOR to common pipeline
osdu-gcp-deploy-deployment:
needs:
......@@ -56,28 +63,45 @@ osdu-gcp-deploy-deployment:
# - echo $STATUS
# - if [[ "$STATUS" != *"met"* ]]; then echo "POD didn't start correctly" ; exit 1 ; fi
osdu-gcp-containerize-bootstrap-gitlab:
variables:
BUILD_PATH: devops/$OSDU_GCP_VENDOR/bootstrap-osdu-module/Dockerfile
osdu-gcp-containerize-bootstrap-gcr:
variables:
BUILD_PATH: devops/$OSDU_GCP_VENDOR/bootstrap-osdu-module/Dockerfile
osdu-gcp-anthos-deploy-deployment:
needs:
- osdu-gcp-containerize-gitlab
- osdu-gcp-containerize-bootstrap-gitlab
- osdu-gcp-anthos-deploy-configmap
osdu-gcp-test:
script:
- $MAVEN_BUILD . test-results.log verify -q -f $OSDU_GCP_TESTS_SUBDIR/pom.xml
- $MAVEN_BUILD . test-results.log verify -q -f testing/schema-test-core/pom.xml
artifacts:
when: always
paths:
- test-results.log
- $OSDU_GCP_TESTS_SUBDIR/target/*/TEST-*.xml
- testing/schema-test-core/target/*/TEST-*.xml
reports:
junit:
- $OSDU_GCP_TESTS_SUBDIR/target/*/TEST-*.xml
- testing/schema-test-core/target/*/TEST-*.xml
osdu-gcp-anthos-deploy-deployment:
needs:
- osdu-gcp-containerize-gitlab
- osdu-gcp-containerize-bootstrap-gitlab
- osdu-gcp-anthos-deploy-configmap
# FIXME for DEV2
osdu-gcp-dev2-test:
script:
- $MAVEN_BUILD . test-results.log verify -q -f testing/schema-test-core/pom.xml
artifacts:
when: always
paths:
- test-results.log
- testing/schema-test-core/target/*/TEST-*.xml
reports:
junit:
- testing/schema-test-core/target/*/TEST-*.xml
osdu-gcp-anthos-test:
script:
- $MAVEN_BUILD . test-results.log verify -q -f testing/schema-test-core/pom.xml
artifacts:
when: always
paths:
- test-results.log
- testing/schema-test-core/target/*/TEST-*.xml
reports:
junit:
- testing/schema-test-core/target/*/TEST-*.xml
......@@ -94,36 +94,10 @@ Once the above Prerequisite are done, we can follow the below steps to run the s
You can access the service APIs by following the service contract in [schema.yaml](docs/api/schema.yaml)
## Testing
### Running E2E Tests
This section describes how to run cloud OSDU E2E tests (testing/schema-test-core).
You will need to have the following environment variables defined.
| name | value | description | sensitive? | source |
| --- | --- | --- | --- | --- |
| `INTEGRATION_TEST_AUDIENCE` | `*****.apps.googleusercontent.com` | client application ID | yes | https://console.cloud.google.com/apis/credentials |
| `VENDOR` | `gcp` | Use value 'gcp' to run gcp tests | no | - |
| `HOST` | ex`http://localhost:8080` | Schema service host | no | - |
| `INTEGRATION_TESTER` | `********` | Service account base64 encoded string for API calls. Note: this user must have entitlements configured already | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
| `PRIVATE_TENANT2` | ex`opendes` | OSDU tenant used for testing | no | - |
| `PRIVATE_TENANT1` | ex`osdu` | OSDU tenant used for testing | no | - |
| `SHARED_TENANT` | ex`common` | OSDU tenant used for testing | no | - |
**Entitlements configuration for integration accounts**
| INTEGRATION_TESTER |
| --- |
| users<br/>service.schema-service.system-admin<br/>service.entitlements.user<br/>service.schema-service.viewers<br/>service.schema-service.editors<br/>data.integration.test<br/>data.test1 |
Execute following command to build code and run all the integration tests:
```bash
# Note: this assumes that the environment variables for integration tests as outlined
# above are already exported in your environment.
# build + install integration test core
$ (cd testing/schema-test-core/ && mvn clean test)
```
#### Anthos:
[Anthos Testing](docs/anthos/README.md)
#### GCP:
[Gcp Testing](docs/gcp/README.md)
## Deployment
......
......@@ -39,6 +39,38 @@ and usage in mixed mode was not tested. Usage of spring profiles is preferred.
| `OBMDRIVER` | `gcs` or `minio` | Obm driver mode that defines which object storage will be used | no | - |
| `SERVICE_TOKEN_PROVIDER` | `GCP` or `OPENID` |Service account token provider, `GCP` means use Google service account `OPEIND` means use OpenId provider like `Keycloak` | no | - |
## Testing
### Running E2E Tests
This section describes how to run cloud OSDU E2E tests (testing/schema-test-core).
You will need to have the following environment variables defined.
| name | value | description | sensitive? | source |
| --- | --- | --- | --- | --- |
| `VENDOR` | `anthos` | Use value 'gcp' to run gcp tests | no | - |
| `HOST` | ex`http://localhost:8080` | Schema service host | no | - |
| `PRIVATE_TENANT2` | ex`opendes` | OSDU tenant used for testing | no | - |
| `PRIVATE_TENANT1` | ex`osdu` | OSDU tenant used for testing | no | - |
| `SHARED_TENANT` | ex`common` | OSDU tenant used for testing | no | - |
| `TEST_OPENID_PROVIDER_CLIENT_ID` | `********` | Client Id for `$INTEGRATION_TESTER` | yes | -- |
| `TEST_OPENID_PROVIDER_CLIENT_SECRET` | `********` | | Client secret for `$INTEGRATION_TESTER` | -- |
| `TEST_OPENID_PROVIDER_URL` | `https://keycloak.com/auth/realms/osdu` | OpenID provider url | yes | -- |
**Entitlements configuration for integration accounts**
| INTEGRATION_TESTER |
| --- |
| users<br/>service.schema-service.system-admin<br/>service.entitlements.user<br/>service.schema-service.viewers<br/>service.schema-service.editors<br/>data.integration.test<br/>data.test1 |
Execute following command to build code and run all the integration tests:
```bash
# Note: this assumes that the environment variables for integration tests as outlined
# above are already exported in your environment.
# build + install integration test core
$ (cd testing/schema-test-core/ && mvn clean test)
```
### Properties set in Partition service:
Note that properties can be set in Partition as `sensitive` in that case in property `value` should be present not value itself, but ENV variable name.
......@@ -449,4 +481,4 @@ For shared tenant only:
<td>ListObjects, CRUDObject
</td>
</tr>
</table>
\ No newline at end of file
</table>
......@@ -36,6 +36,37 @@ and usage in mixed mode was not tested. Usage of spring profiles is preferred.
| `OBMDRIVER` | `gcs` or `minio` | Obm driver mode that defines which object storage will be used | no | - |
| `SERVICE_TOKEN_PROVIDER` | `GCP` or `OPENID` |Service account token provider, `GCP` means use Google service account `OPEIND` means use OpenId provider like `Keycloak` | no | - |
## Testing
### Running E2E Tests
This section describes how to run cloud OSDU E2E tests (testing/schema-test-core).
You will need to have the following environment variables defined.
| name | value | description | sensitive? | source |
| --- | --- | --- | --- | --- |
| `INTEGRATION_TEST_AUDIENCE` | `*****.apps.googleusercontent.com` | client application ID | yes | https://console.cloud.google.com/apis/credentials |
| `VENDOR` | `gcp` | Use value 'gcp' to run gcp tests | no | - |
| `HOST` | ex`http://localhost:8080` | Schema service host | no | - |
| `INTEGRATION_TESTER` | `********` | Service account base64 encoded string for API calls. Note: this user must have entitlements configured already | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
| `PRIVATE_TENANT2` | ex`opendes` | OSDU tenant used for testing | no | - |
| `PRIVATE_TENANT1` | ex`osdu` | OSDU tenant used for testing | no | - |
| `SHARED_TENANT` | ex`common` | OSDU tenant used for testing | no | - |
**Entitlements configuration for integration accounts**
| INTEGRATION_TESTER |
| --- |
| users<br/>service.schema-service.system-admin<br/>service.entitlements.user<br/>service.schema-service.viewers<br/>service.schema-service.editors<br/>data.integration.test<br/>data.test1 |
Execute following command to build code and run all the integration tests:
```bash
# Note: this assumes that the environment variables for integration tests as outlined
# above are already exported in your environment.
# build + install integration test core
$ (cd testing/schema-test-core/ && mvn clean test)
```
## Datastore configuration:
There must be a namespace `dataecosystem`.
......
......@@ -12,7 +12,7 @@
<properties>
<start-class>org.opengroup.osdu.schema.provider.ibm.app.SchemaIBMApplication</start-class>
<os-core-lib-ibm.version>0.15.0-rc1</os-core-lib-ibm.version>
<os-core-lib-ibm.version>0.15.0-rc2</os-core-lib-ibm.version>
</properties>
<dependencyManagement>
......
......@@ -19,6 +19,7 @@
<module>schema-test-core</module>
<module>schema-test-gcp</module>
<module>schema-test-azure</module>
<module>schema-test-anthos</module>
</modules>
<repositories>
......
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-schema-test</artifactId>
<version>0.15.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-schema-test-anthos</artifactId>
<version>0.15.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>schema-test-gcp</name>
<description>Anthos implementation of test project for schema service</description>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%yellow([%thread]) %highlight(| %-5level |) %green(%d) %cyan(| %logger{15} |) %highlight(%msg) %n</pattern>
<charset>utf8</charset>
</encoder>
</appender>
<root level="INFO">
<appender-ref ref="CONSOLE" />
</root>
</configuration>
......@@ -2,8 +2,9 @@ package org.opengroup.osdu.schema.util;
import com.google.common.base.Strings;
import org.opengroup.osdu.azure.util.AzureServicePrincipal;
import org.opengroup.osdu.core.aws.cognito.AWSCognitoClient;
import org.opengroup.osdu.core.ibm.util.IdentityClient;
import org.opengroup.osdu.schema.util.gcp.GoogleServiceAccount;
import org.opengroup.osdu.schema.util.gcp.OpenIDTokenProvider;
public class AuthUtil {
......@@ -25,6 +26,8 @@ public class AuthUtil {
token = new AzureServicePrincipal().getIdToken(sp_id, sp_secret, tenant_id, app_resource_id);
} else if (Strings.isNullOrEmpty(token) && vendor.equals("ibm")) {
token = IdentityClient.getTokenForUserWithAccess();
} else if (Strings.isNullOrEmpty(token) && vendor.equals("anthos")){
token = new OpenIDTokenProvider().getToken();
}
return "Bearer " + token;
}
......
package org.opengroup.osdu.schema.util;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Base64;
import java.util.HashMap;
import java.util.Map;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import com.google.auth.oauth2.ServiceAccountCredentials;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import io.jsonwebtoken.JwtBuilder;
import io.jsonwebtoken.Jwts;
import io.jsonwebtoken.SignatureAlgorithm;
public class GoogleServiceAccount {
final ServiceAccountCredentials serviceAccount;
public GoogleServiceAccount(String serviceAccountEncoded) throws IOException {
this(Base64.getDecoder().decode(serviceAccountEncoded));
}
public GoogleServiceAccount(byte[] serviceAccountJson) throws IOException {
try (InputStream inputStream = new ByteArrayInputStream(serviceAccountJson)) {
this.serviceAccount = ServiceAccountCredentials.fromStream(inputStream);
}
}
public String getEmail() {
return this.serviceAccount.getClientEmail();
}
public String getAuthToken(String audience) throws IOException {
JwtBuilder jwtBuilder = Jwts.builder();
Map<String, Object> header = new HashMap<>();
header.put("type", "JWT");
header.put("alg", "RS256");
jwtBuilder.setHeader(header);
Map<String, Object> claims = new HashMap<>();
claims.put("target_audience", audience);
claims.put("exp", System.currentTimeMillis() / 1000 + 3600);
claims.put("iat", System.currentTimeMillis() / 1000);
claims.put("iss", this.getEmail());
claims.put("aud", "https://www.googleapis.com/oauth2/v4/token");
jwtBuilder.addClaims(claims);
jwtBuilder.signWith(SignatureAlgorithm.RS256, this.serviceAccount.getPrivateKey());
String jwt = jwtBuilder.compact();
HttpPost httpPost = new HttpPost("https://www.googleapis.com/oauth2/v4/token");
ArrayList<NameValuePair> postParameters = new ArrayList<>();
postParameters.add(new BasicNameValuePair("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer"));
postParameters.add(new BasicNameValuePair("assertion", jwt));
HttpClient client = new DefaultHttpClient();
httpPost.setEntity(new UrlEncodedFormEntity(postParameters, "UTF-8"));
httpPost.setHeader("Content-Type", "application/x-www-form-urlencoded");
HttpResponse response = client.execute(httpPost);
String responseEntity = EntityUtils.toString(response.getEntity());
JsonObject content = new JsonParser().parse(responseEntity).getAsJsonObject();
return content.get("id_token").getAsString();
}
}
/*
* Copyright 2020-2022 Google LLC
* Copyright 2020-2022 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.schema.util.gcp;
import com.google.auth.oauth2.ServiceAccountCredentials;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import io.jsonwebtoken.JwtBuilder;
import io.jsonwebtoken.Jwts;
import io.jsonwebtoken.SignatureAlgorithm;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Base64;
import java.util.HashMap;
import java.util.Map;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
public class GoogleServiceAccount {
final ServiceAccountCredentials serviceAccount;
public GoogleServiceAccount(String serviceAccountEncoded) throws IOException {
this(Base64.getDecoder().decode(serviceAccountEncoded));
}
public GoogleServiceAccount(byte[] serviceAccountJson) throws IOException {
try (InputStream inputStream = new ByteArrayInputStream(serviceAccountJson)) {
this.serviceAccount = ServiceAccountCredentials.fromStream(inputStream);
}
}
public String getEmail() {
return this.serviceAccount.getClientEmail();
}
public String getAuthToken(String audience) throws IOException {
JwtBuilder jwtBuilder = Jwts.builder();
Map<String, Object> header = new HashMap<>();
header.put("type", "JWT");
header.put("alg", "RS256");
jwtBuilder.setHeader(header);
Map<String, Object> claims = new HashMap<>();
claims.put("target_audience", audience);
claims.put("exp", System.currentTimeMillis() / 1000 + 3600);
claims.put("iat", System.currentTimeMillis() / 1000);
claims.put("iss", this.getEmail());
claims.put("aud", "https://www.googleapis.com/oauth2/v4/token");
jwtBuilder.addClaims(claims);
jwtBuilder.signWith(SignatureAlgorithm.RS256, this.serviceAccount.getPrivateKey());
String jwt = jwtBuilder.compact();
HttpPost httpPost = new HttpPost("https://www.googleapis.com/oauth2/v4/token");
ArrayList<NameValuePair> postParameters = new ArrayList<>();
postParameters.add(
new BasicNameValuePair("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer"));
postParameters.add(new BasicNameValuePair("assertion", jwt));
HttpClient client = new DefaultHttpClient();
httpPost.setEntity(new UrlEncodedFormEntity(postParameters, "UTF-8"));
httpPost.setHeader("Content-Type", "application/x-www-form-urlencoded");
HttpResponse response = client.execute(httpPost);
String responseEntity = EntityUtils.toString(response.getEntity());
JsonObject content = new JsonParser().parse(responseEntity).getAsJsonObject();
return content.get("id_token").getAsString();
}
}
/*
* Copyright 2020-2022 Google LLC
* Copyright 2020-2022 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/