Commit 3f994fb5 authored by Abhishek Kumar (SLB)'s avatar Abhishek Kumar (SLB)
Browse files
parents d47c9085 2a849b6a
......@@ -44,6 +44,7 @@ variables:
OSDU_GCP_HELM_CONFIG_SERVICE: schema-config
OSDU_GCP_HELM_DEPLOYMENT_SERVICE: schema-deploy
OSDU_GCP_HELM_CONFIG_SERVICE_VARS: "--set data.log_level=INFO --set data.authorize_api=$OSDU_GCP_AUTHORIZE_API --set data.google_audiences=$GOOGLE_AUDIENCE --set data.partition_api=$OSDU_GCP_PARTITION_API --set data.shared_tenant_name=$OSDU_GCP_TENANT"
OSDU_GCP_HELM_CONFIG_SERVICE_VARS_DEV2: "--set data.log_level=INFO --set data.authorize_api=$OSDU_GCP_AUTHORIZE_API --set data.google_audiences=$GOOGLE_AUDIENCE --set data.partition_api=$OSDU_GCP_PARTITION_API --set data.shared_tenant_name=$OSDU_GCP_TENANT"
OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS: "--set data.image=$CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA --set data.serviceAccountName=$OSDU_GCP_SERVICE-k8s"
include:
......
This diff is collapsed.
......@@ -31,6 +31,10 @@ Below command can be run through azure-pipeline.yml after setting environment va
Schema service as part of deployment deploys pre-defined OSDU schemas so end users can get community accepted schemas to refer. Such schemas are present in [folder](./deployments/shared-schemas/osdu) and script to deploy the schema are present [here](deployments/scripts).
Details to deploy shared schemas can be found under [README.md](./deployments/shared-schemas/README.md)
##AWS
Instructions for running and testing this service can be found [here](./provider/schema-aws/README.md)
## License
......
......@@ -5,6 +5,6 @@ COPY ./deployments /opt/
RUN chmod 775 /opt/bootstrap_schema.sh
RUN pip3 install -r /opt/scripts/requirements.txt && pip3 install -r /opt/scripts/gcp-deployment-requirements.txt
RUN pip3 install --upgrade pip && pip3 install -r /opt/scripts/requirements.txt && pip3 install -r /opt/scripts/gcp-deployment-requirements.txt
CMD ["/bin/bash", "-c", "source /opt/bootstrap_schema.sh"]
osdu_gcp_bootstrap:
osdu-gcp-bootstrap:
stage: bootstrap
image: python:3.8
needs: ["osdu-gcp-deploy-deployment"]
......@@ -13,16 +13,46 @@ osdu_gcp_bootstrap:
- pip install -r $OSDU_GCP_DEPLOYMENTS_SCRIPTS_SUBDIR/gcp-deployment-requirements.txt
- export BEARER_TOKEN=$(python $OSDU_GCP_DEPLOYMENTS_SUBDIR/Token.py)
- python $OSDU_GCP_DEPLOYMENTS_SCRIPTS_SUBDIR/DeploySharedSchemas.py -u $OSDU_GCP_SCHEMA_URL/schema
rules:
- if: '$OSDU_GCP == "true"'
when: always
only:
variables:
- $OSDU_GCP == 'true'
osdu-gcp-bootstrap-dev2:
stage: bootstrap
image: python:3.8
needs: ["osdu-gcp-dev2-deploy-deployment"]
extends: .osdu-gcp-dev2-variables
variables:
JSON_KEY: $OSDU_GCP_INTEGRATION_TESTER
AUDIENCE: $GOOGLE_AUDIENCE
DATA_PARTITION: $OSDU_GCP_TENANT
script:
- pip install -r $OSDU_GCP_DEPLOYMENTS_SUBDIR/requirements.txt
- pip install -r $OSDU_GCP_DEPLOYMENTS_SCRIPTS_SUBDIR/gcp-deployment-requirements.txt
- export BEARER_TOKEN=$(python $OSDU_GCP_DEPLOYMENTS_SUBDIR/Token.py)
- python $OSDU_GCP_DEPLOYMENTS_SCRIPTS_SUBDIR/DeploySharedSchemas.py -u $OSDU_GCP_SCHEMA_URL/schema
rules:
- if: '$OSDU_GCP == "true" && $CI_COMMIT_BRANCH =~ /^release$/'
when: always
- if: '$OSDU_GCP == "true" && $CI_COMMIT_TAG'
when: always
# The variable DEV2="true" should be specified manually in GitLab before running a pipeline to test this job against a protected branch
- if: '$OSDU_GCP == "true" && $DEV2 == "true"'
when: always
osdu-gcp-test:
needs: ["osdu_gcp_bootstrap"]
only:
variables:
- $OSDU_GCP == 'true'
needs: ["osdu-gcp-bootstrap"]
osdu-gcp-dev2-test:
needs: ["osdu-gcp-bootstrap-dev2"]
rules:
- if: '$OSDU_GCP == "true" && $CI_COMMIT_BRANCH =~ /^release$/'
when: always
- if: '$OSDU_GCP == "true" && $CI_COMMIT_TAG'
when: always
# The variable DEV2="true" should be specified manually in GitLab before running a pipeline to test this job against a protected branch
- if: '$OSDU_GCP == "true" && $DEV2 == "true"'
when: always
containerize_osdu_gcp_schema_bootstrap:
stage: containerize
......
......@@ -20,6 +20,7 @@ spec:
containers:
- name: "{{ .Values.conf.app_name }}"
image: "{{ .Values.data.image }}"
imagePullPolicy: "{{ .Values.data.imagePullPolicy }}"
envFrom:
- configMapRef:
name: "{{ .Values.conf.configmap }}"
......
......@@ -8,6 +8,7 @@ data:
limits_cpu: "1"
limits_memory: "1G"
serviceAccountName: ""
imagePullPolicy: "IfNotPresent"
image: ""
conf:
......
# Schema Service
The OSDU on AWS energy data platform registers new schemas describing ‘Kinds of data’ via the Schema Service.The Schema service enables easier creation, management and sharing of schemas.
The Schema service supports the deployment of ‘Shared’ as well as ‘Private’ schemas. ‘Shared’ schemas will help make teams aware of the different schemas being used and will facilitate reuse.
## Running Locally
These instructions will get you a copy of the project up and running on your local machine for development and testing purposes.
### Prerequisites
Pre-requisites
* JDK 8 (https://docs.aws.amazon.com/corretto/latest/corretto-8-ug/downloads-list.html)
* Maven 3.8.3 or later
* Lombok 1.16 or later
* OSDU Instance deployed on AWS
### Service Configuration
In order to run the service locally or remotely, you will need to have the following environment variables defined.
| name | example value | required | description | sensitive? |
| --- | --- | --- | --- | --- |
| `APPLICATION_PORT` | `8080` | yes | The port the service will be hosted on. | no |
| `AWS_REGION` | `us-east-1` | yes | The region where resources needed by the service are deployed | no |
| `AWS_ACCESS_KEY_ID` | `ASIAXXXXXXXXXXXXXX` | yes | The AWS Access Key for a user with access to Backend Resources required by the service | yes |
| `AWS_SECRET_ACCESS_KEY` | `super-secret-key==` | yes | The AWS Secret Key for a user with access to Backend Resources required by the service | yes |
| `AWS_SESSION_TOKEN` | `session-token-xxxxxxxxxx` | no | AWS Session token needed if using an SSO user session to authenticate | yes |
| `ENVIRONMENT` | `osdu-prefix` | yes | The Resource Prefix defined during deployment | no |
| `LOG_LEVEL` | `DEBUG` | yes | The Log Level severity to use (https://www.tutorialspoint.com/log4j/log4j_logging_levels.htm) | no |
| `SSM_ENABLED` | `true` | yes | Set to 'true' to use SSM to resolve config properties, otherwise use env vars | no |
| `SSL_ENABLED` | `false` | no | Set to 'false' to disable SSL for local development | no |
| `ENTITLEMENTS_BASE_URL` | `http://localhost:8081` or `https://some-hosted-url` | yes | Specify the base url for an entitlements service instance. Can be run locally or remote | no |
| `PARTITION_BASE_URL` | `http://localhost:8081` or `https://some-hosted-url` | yes | Specify the base url for a partitions service instance. Can be run locally or remote | no |
### Run Locally
Check that maven is installed:
example:
```bash
$ mvn --version
Apache Maven 3.8.3 (ff8e977a158738155dc465c6a97ffaf31982d739)
Maven home: /usr/local/Cellar/maven/3.8.3/libexec
Java version: 1.8.0_312, vendor: Amazon.com Inc., runtime: /Library/Java/JavaVirtualMachines/amazon-corretto-8.jdk/Contents/Home/jre
...
```
You may need to configure access to the remote maven repository that holds the OSDU dependencies. Copy one of the below files' content to your .m2 folder
* For development against the OSDU GitLab environment, leverage: `<REPO_ROOT>~/.mvn/community-maven.settings.xml`
* For development in an AWS Environment, leverage: `<REPO_ROOT>/provider/schema-aws/maven/settings.xml`
* Navigate to the service's root folder and run:
```bash
mvn clean package -pl schema-core,provider/schema-aws
```
* If you wish to build the project without running tests
```bash
mvn clean package -pl schema-core,provider/schema-aws -DskipTests
```
After configuring your environment as specified above, you can follow these steps to run the application. These steps should be invoked from the *repository root.*
<br/>
<br/>
NOTE: If not on osx/linux: Replace `*` with version numbers as defined in the provider/schema-aws/pom.xml file
```bash
java -jar provider/schema-aws/target/schema-aws-*.*.*-SNAPSHOT-spring-boot.jar
```
## Testing
### Running Integration Tests
This section describes how to run OSDU Integration tests (testing/schema-test-aws).
You will need to have the following environment variables defined.
| name | example value | description | sensitive?
| --- | --- | --- | --- |
| `AWS_ACCESS_KEY_ID` | `ASIAXXXXXXXXXXXXXX` | The AWS Access Key for a user with access to Backend Resources required by the service | yes |
| `AWS_SECRET_ACCESS_KEY` | `super-secret-key==` | The AWS Secret Key for a user with access to Backend Resources required by the service | yes |
| `AWS_SESSION_TOKEN` | `session-token-xxxxxxxxx` | AWS Session token needed if using an SSO user session to authenticate | yes |
| `AWS_COGNITO_USER_POOL_ID` | `us-east-1_xxxxxxxx` | User Pool Id for the reference cognito | no |
| `AWS_COGNITO_CLIENT_ID` | `xxxxxxxxxxxx` | Client ID for the Auth Flow integrated with the Cognito User Pool | no |
| `AWS_COGNITO_AUTH_FLOW` | `USER_PASSWORD_AUTH` | Auth flow used by reference cognito deployment | no |
| `AWS_COGNITO_AUTH_PARAMS_USER` | `int-test-user@testing.com` | Int Test Username | no |
| `AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS` | `no-access-user@testing.com` | Int Test No Access Username | no |
| `AWS_COGNITO_AUTH_PARAMS_PASSWORD` | `some-secure-password` | Int Test User/NoAccessUser Password | yes |
| `HOST` | `http://localhost:8080` | The url where the Schema API is hosted | no |
| `PRIVATE_TENANT1` | `opendes` | Data Partition Id used by int tests | no |
| `PRIVATE_TENANT2` | `tenant2` | Data Partition Id used by int tests | no |
| `SHARED_TENANT` | `common` | Shared Data Partition Id used by int tests | no |
| `VENDOR` | `aws` | CSP running these tests | no |
**Creating a new user to use for integration tests**
```
aws cognito-idp admin-create-user --user-pool-id ${AWS_COGNITO_USER_POOL_ID} --username ${AWS_COGNITO_AUTH_PARAMS_USER} --user-attributes Name=email,Value=${AWS_COGNITO_AUTH_PARAMS_USER} Name=email_verified,Value=True --message-action SUPPRESS
aws cognito-idp initiate-auth --auth-flow ${AWS_COGNITO_AUTH_FLOW} --client-id ${AWS_COGNITO_CLIENT_ID} --auth-parameters USERNAME=${AWS_COGNITO_AUTH_PARAMS_USER},PASSWORD=${AWS_COGNITO_AUTH_PARAMS_PASSWORD}
```
**Entitlements group configuration for integration accounts**
<br/>
In order to add user entitlements, run entitlements bootstrap scripts in the entitlements project
| AWS_COGNITO_AUTH_PARAMS_USER | AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS |
| --- | --- |
| service.schema-service.editors | service.entitlements.user |
| service.schema-service.viewers |
Execute following command to build code and run all the integration tests:
### Run Tests simulating Pipeline
* Prior to running tests, scripts must be executed locally to generate pipeline env vars
```bash
testing/schema-test-aws/build-aws/prepare-dist.sh
#Set Neccessary ENV Vars here as defined in run-tests.sh
dist/testing/integration/build-aws/run-tests.sh
```
### Run Tests using mvn
Set required env vars and execute the following:
```
mvn clean package -f testing/pom.xml -pl schema-test-core -DskipTests
mvn verify -f testing/schema-test-core/pom.xml -Dcucumber.options="--plugin junit:target/junit-report.xml --tags @SchemaService"
```
## License
Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
[http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0)
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment