diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9f3d130e626ec91fb02b80acab85278473338b64..74e61e601301a552058bbdd596130fda09a7a7d5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -33,3 +33,8 @@ include: - project: "osdu/platform/deployment-and-operations/infra-gcp-provisioning" ref: "master" file: "prod-deploy/admin-ui/osdu-gcp-admin-ui.yml" + + # Pipeline to c Admin UI service to preship and community environments + - project: "osdu/platform/deployment-and-operations/infra-gcp-provisioning" + ref: "master" + file: "bootstrap_infra/bootstrap_infra.yml" diff --git a/README.md b/README.md index c6e9eafadc67c1a0282081e3ea44ade1d320bacc..91f6d95c300d06164f0df90c88c8c9264a43e21d 100644 --- a/README.md +++ b/README.md @@ -1,40 +1,46 @@ # OSDU module/deploument framework -Creates fully functional GCP based OSDU installation. +Creates fully functional GCP based OSDU installation. ## Prerequisites + - **Terraform** (version: v1.0.0) - **Gcloud** (version: Google Cloud SDK 345.0.0) - **Kubectl** (version: v1.21.0) ## Before install + At the respective GCP project we need to create OAuth 2.0 Client ID. According to GCP requirement you need to first create OAuth consent screen, and then activate `API & Services` so you could create OAuth 2.0 Client ID of type Web application. ### **To create OAuth consent screen:** -1. Go to **GCP navigation menu** -> **API & Services** -> **OAuth consent screen** +Go to **GCP navigation menu** -> **API & Services** -> **OAuth consent screen** - - choose type `Internal` and click `CREATE` - - fillin `App name`, `User support email`, `Authorized domains` and `Developer contact information` fields. - - press `SAVE AND CONTINUE`, next step press `SAVE AND CONTINUE`. +``` +- choose type `Internal` and click `CREATE` +- fillin `App name`, `User support email`, `Authorized domains` and `Developer contact information` fields. +- press `SAVE AND CONTINUE`, next step press `SAVE AND CONTINUE`. +``` ![alt text](img/gcp-consent-screen.png "Content Screen") +### **To create OAuth 2.0 Client ID of type Web application:** -### **To create OAuth 2.0 Client ID of type Web application** - -1. Go to **GCP navigation menu** -> **API & Services** -> **Credentials** +Go to **GCP navigation menu** -> **API & Services** -> **Credentials** - - click `+ CREATE CREDENTIALS` -> OAuth client ID -> Application type (choose `Web application`) -> fillin `Name` field like `project_name-audiences` -> click `CREATE` +``` +- click `+ CREATE CREDENTIALS` -> OAuth client ID -> Application type (choose `Web application`) -> fillin `Name` field like `project_name-audiences` -> click `CREATE` +``` ## How to run **osdu-module** with **Terraform** -Make shure that your gcloud client is configured with oppropriate GCP **project** and **region** -Then you need to fill-in **mandatory** variables at **variables.tf** file. +Make shure that your gcloud client is configured with oppropriate GCP **project** and **region** +Then you need to fill-in **mandatory** variables at **variables.tf** file. > these variables are: + ``` project_id (your GCP project ID) data_partition_id (Data Partition ID and also Tenant Name) @@ -46,6 +52,7 @@ audiences (these variable is ClientID from previous step of creating OAuth 2.0 C ``` ### To start installation use the following commands + ``` $ git clone git@community.opengroup.org:osdu/platform/deployment-and-operations/infra-gcp-provisioning.git $ cd infra-gcp-provisioning @@ -56,5 +63,6 @@ $ terraform apply ``` ## Post install steps + After successfull installation please look at the **outputs** of terraform execution and -get **ingress_ip** variable value. Use this ip-address for our DNS domain name. +get **ingress_ip** variable value. Use this ip-address for our DNS domain name. diff --git a/bootstrap_infra/Dockerfile b/bootstrap_infra/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..1f1db9ad8f8ab34107f5e3b4921545b3e941649c --- /dev/null +++ b/bootstrap_infra/Dockerfile @@ -0,0 +1,24 @@ +FROM python:3.8-buster + +COPY ./bootstrap_infra/requirements.txt ./ +COPY ./bootstrap_infra ./opt + +ENV PREFIX=$(pwd) + +RUN apt-get update && \ + apt-get install -yqq \ + gnupg2 \ + gnupg \ + gnupg1 \ + curl \ + wget \ + postgresql-client + +RUN pip install -U pip && pip install -r requirements.txt --use-feature=2020-resolver + +# Google SDK install +RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - && apt-get update -y && apt-get install google-cloud-sdk -y && apt-get install kubectl -y +RUN wget https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 -O /opt/sql_bootstrap/cloud_sql_proxy && \ + chmod +x /opt/sql_bootstrap/cloud_sql_proxy + +CMD ["/bin/bash", "-c", "/opt/autostart.sh"] diff --git a/bootstrap_infra/autostart.sh b/bootstrap_infra/autostart.sh new file mode 100755 index 0000000000000000000000000000000000000000..bcbcb16dbafae4ac98bec5dcf2ee22d015e1a75f --- /dev/null +++ b/bootstrap_infra/autostart.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +set -ex + +cd /opt + +source ./validate-env.sh "$GCLOUD_PROJECT" +source ./validate-env.sh "$GOOGLE_PROJECT_REGION" +source ./validate-env.sh "$CLOUDSDK_COMPUTE_ZONE" +source ./validate-env.sh "$GCLOUD_REGION" +source ./validate-env.sh "$ENVIRONMENT" +source ./validate-env.sh "$DATA_PARTITION_ID" +source ./validate-env.sh "$COMPLIANCE_RULE_SET" +source ./validate-env.sh "$CRM_ACCOUNT_IDS" +source ./validate-env.sh "$NEW_TENANT" +source ./validate-env.sh "$GOOGLE_DOMAIN" +source ./validate-env.sh "$ELASTIC_ADMIN_USER" +source ./validate-env.sh "$ELASTIC_ADMIN_PASSWORD" +source ./validate-env.sh "$ELASTIC_PORT" +source ./validate-env.sh "$SECURITY_HTTPS_CERTIFICATE_TRUST" +source ./validate-env.sh "$ELASTIC_HOST" +source ./validate-env.sh "$KEYRING" +source ./validate-env.sh "$TENANT_NAME" + +#validate TENNANT_NAME +source ./validate-env-tenant-name.sh "$TENANT_NAME" +source ./validate-env-tenant-name.sh "$DATA_PARTITION_ID" + +echo "Create Indexes in Datastore" +pushd create-definitions > /dev/null +./create-index-definitions.sh +popd > /dev/null + +echo "Create gcp.datastore rows for ingestion-strategy" +pushd ingestion-strategy > /dev/null +python ingest-strategy.py --namespace=$TENANT_NAME --createdby=test_user --creationtimestamp=`date +%s` --dagname=Osdu_ingest --description="Default workflow for manifest-based ingestion" --version=1 --workflowname=Osdu_ingest +echo "Done" +popd > /dev/null + +echo "Configure Elasticsearch" +pushd elastic-search-settings > /dev/null +./create-search-service-user.sh +popd > /dev/null + +echo "Bootstrap SQL Database" +pushd sql_bootstrap > /dev/null +./cloud_sql_proxy -instances=${PGADDRESS}=tcp:5432 & +sleep 5 + +psql -h 127.0.0.1 -U $PGUSER -d $PGDATABASE -f create-tables.sql +popd > /dev/null + +echo "Finished successfully" diff --git a/bootstrap_infra/bootstrap_infra.yml b/bootstrap_infra/bootstrap_infra.yml new file mode 100644 index 0000000000000000000000000000000000000000..71aaeb153614e4e59a6934916280e63d5584e24f --- /dev/null +++ b/bootstrap_infra/bootstrap_infra.yml @@ -0,0 +1,22 @@ +containerize_osdu_deploy_gcp: + stage: containerize + image: docker:19.03.15 + tags: ["osdu-medium"] + services: + - docker:19.03.15-dind + variables: + IMAGE_NAME: osdu-bootstrap_infra + SHORT_SHA_IMAGE: ${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:${CI_COMMIT_SHORT_SHA} + LATEST_IMAGE: ${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:latest + before_script: + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + script: + # Gitlab Container Registry + - docker build -t $IMAGE_NAME:local -f bootstrap_infra/Dockerfile . + - docker tag $IMAGE_NAME:local $SHORT_SHA_IMAGE + - docker tag $IMAGE_NAME:local $LATEST_IMAGE + - docker push $SHORT_SHA_IMAGE + - docker push $LATEST_IMAGE + only: + refs: + - master diff --git a/bootstrap_infra/create-definitions/create-index-definitions.sh b/bootstrap_infra/create-definitions/create-index-definitions.sh new file mode 100755 index 0000000000000000000000000000000000000000..e7cbc5ef80d25f5c77d3067bcbc987f72c038dfc --- /dev/null +++ b/bootstrap_infra/create-definitions/create-index-definitions.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# Copyright 2020 Google LLC +# Copyright 2017-2019, Schlumberger +# Copyright 2020 EPAM +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Exit as soon as command fails (add -v for debugging strict) + +set -ex + +source ../validate-env.sh $GCLOUD_REGION + +create-datastore-indexes() { + echo "Creating new indexes for datastore in the project" + gcloud datastore indexes create --project=$TENANT_PROJECT index.yaml --quiet + #echo "done" +} + +if [ "${NEW_TENANT}" == "true" ] +then + source ../validate-env.sh $TENANT_PROJECT + set +e + echo "You are creating an app for project [${GCLOUD_PROJECT}]" + APPENGINE_MSG=$(gcloud app create --region=${GCLOUD_REGION} 2>&1) + APPENGINE_MSG_ERR=$(echo "$APPENGINE_MSG" | grep ERROR:) + if [[ "$APPENGINE_MSG_ERR" == *"already contains an App Engine application"* ]] + then + echo "The project [${GCLOUD_PROJECT}] already contains an App Engine application" + elif [[ "$APPENGINE_MSG_ERR" == "" ]] + then + echo "$APPENGINE_MSG" + else + echo "$APPENGINE_MSG_ERR" + exit 1 + fi + set -e + gcloud beta datastore databases create --region=${GCLOUD_REGION} + create-datastore-indexes +else + while read line + do + export TENANT_PROJECT=$(echo $line | cut -d',' -f2) + create-datastore-indexes + done <../datastore-table-info.txt +fi diff --git a/bootstrap_infra/create-definitions/index.yaml b/bootstrap_infra/create-definitions/index.yaml new file mode 100644 index 0000000000000000000000000000000000000000..caa5fdb990dd6cbc1ab9a06684b673118f9fc536 --- /dev/null +++ b/bootstrap_infra/create-definitions/index.yaml @@ -0,0 +1,78 @@ +# Index configuration for Google Cloud Datastore +indexes: +- kind: StorageRecord + properties: + - name: kind + - name: version + - name: bucket + +- kind: StorageRecord + properties: + - name: __key__ + - name: bucket + - name: kind + - name: version + +- kind: StorageRecord + properties: + - name: status + - name: kind + +- kind: MainIngestionJobStatus + ancestor: no + properties: + - name: status + direction: asc + - name: timestamp + direction: desc + +- kind: IngestionPipelineJobInfo + ancestor: no + properties: + - name: masterStatus + - name: creationDate + direction: desc + +# MPTC indexes +- kind: log + properties: + - name: owner + - name: fullName + +- kind: log + properties: + - name: tags + - name: fullName + +- kind: log + properties: + - name: type + - name: owner + +- kind: log + properties: + - name: owner + - name: __key__ + - name: type + +- kind: log + properties: + - name: customProperties + - name: fullName + +- kind: log + properties: + - name: __key__ + direction: desc + +- kind: log + properties: + - name: owner + - name: __key__ + direction: desc + +- kind: history + properties: + - name: logid + - name: time + direction: asc diff --git a/bootstrap_infra/elastic-search-settings/create-search-service-user.sh b/bootstrap_infra/elastic-search-settings/create-search-service-user.sh new file mode 100755 index 0000000000000000000000000000000000000000..1713cf8a094e957b9b2556ff9533c2621a0d77f8 --- /dev/null +++ b/bootstrap_infra/elastic-search-settings/create-search-service-user.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# Copyright 2020 Google LLC +# Copyright 2017-2019, Schlumberger +# Copyright 2020 EPAM +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +source ../validate-env.sh $ELASTIC_HOST +source ../validate-env.sh $ELASTIC_PORT +source ../validate-env.sh $ELASTIC_ADMIN_USER +source ../validate-env.sh $ELASTIC_ADMIN_PASSWORD +source ../validate-env.sh $SECURITY_HTTPS_CERTIFICATE_TRUST + +export SEARCH_SERVICE_USER="search-service" +export INDEXER_SERVICE_USER="indexer-service" +export ELASTIC_HOST=$ELASTIC_HOST +export ELASTIC_PORT=$ELASTIC_PORT +export ACCOUNT_EMAIL=$(printf '%s@opengroup.org' $(openssl rand -hex 16)) +export SEARCH_SERVICE_PASSWORD=$(openssl rand -base64 32) +export INDEXER_SERVICE_PASSWORD=$(openssl rand -base64 32) + +export ADMIN_USERANDPASSWORD=$(echo $ELASTIC_ADMIN_USER:$ELASTIC_ADMIN_PASSWORD) +export SEARCH_ROLE_CREATION_REQUEST=$(printf 'https://%s:%s/_xpack/security/role/search_service' "$ELASTIC_HOST" "$ELASTIC_PORT") +export SEARCH_USER_CREATION_REQUEST=$(printf 'https://%s:%s/_xpack/security/user/%s' "$ELASTIC_HOST" "$ELASTIC_PORT" "$SEARCH_SERVICE_USER") +export INDEXER_ROLE_CREATION_REQUEST=$(printf 'https://%s:%s/_xpack/security/role/indexer_service' "$ELASTIC_HOST" "$ELASTIC_PORT") +export INDEXER_USER_CREATION_REQUEST=$(printf 'https://%s:%s/_xpack/security/user/%s' "$ELASTIC_HOST" "$ELASTIC_PORT" "$INDEXER_SERVICE_USER") + +INSECURE="" +if (( "$SECURITY_HTTPS_CERTIFICATE_TRUST" == "false")); then + INSECURE="--insecure"; +fi + +# create new role that only contain minimum privileges for search and indexer api +echo '{ + "cluster": ["monitor"], + "indices": [ + { + "names": [ "*" ], + "privileges": ["read", "delete_index", "view_index_metadata"] + } + ] + }' > role_payload.json + +set +e +status_code=$(curl ${INSECURE} --write-out %{http_code} --silent --output /dev/null -d @role_payload.json --user "$ADMIN_USERANDPASSWORD" -H "Content-Type: application/json" -X POST "$SEARCH_ROLE_CREATION_REQUEST") +if [[ "$status_code" -gt 299 || "$status_code" -eq 000 ]] ; then + echo "Status Code: $status_code | An error occurred while creating roles in elastic cluster. Exiting..." + exit 1 +fi +set -e + +echo '{ + "cluster": ["monitor"], + "indices": [ + { + "names": [ "*" ], + "privileges": ["create_index", "delete_index", "create", "delete", "index", "write", "view_index_metadata", "read", "monitor"] + } + ] + }' > role_payload.json +set +e +status_code=$(curl ${INSECURE} --write-out %{http_code} --silent --output /dev/null -d @role_payload.json --user "$ADMIN_USERANDPASSWORD" -H "Content-Type: application/json" -X POST "$INDEXER_ROLE_CREATION_REQUEST") +if [[ "$status_code" -gt 299 || "$status_code" -eq 000 ]] ; then + echo "Status Code: $status_code | An error occurred while creating roles in elastic cluster. Exiting..." + exit 1 +fi +set -e + +# create new user for search and indexer api +printf '{ + "password": "%s", + "roles": ["search_service"], + "full_name": "%s", + "email": "%s" + }' "$SEARCH_SERVICE_PASSWORD" "$SEARCH_SERVICE_USER" "$ACCOUNT_EMAIL" > user_payload.json +set +e +status_code=$(curl ${INSECURE} --write-out %{http_code} --silent --output /dev/null -d @user_payload.json --user "$ADMIN_USERANDPASSWORD" -H "Content-Type: application/json" -X POST "$SEARCH_USER_CREATION_REQUEST") +if [[ "$status_code" -gt 299 || "$status_code" -eq 000 ]] ; then + echo "Status Code: $status_code | An error occurred while creating $SEARCH_SERVICE_USER user in elastic cluster. Exiting..." + exit 1 +fi +set -e + +printf '{ + "password": "%s", + "roles": ["indexer_service"], + "full_name": "%s", + "email": "%s" + }' "$INDEXER_SERVICE_PASSWORD" "$INDEXER_SERVICE_USER" "$ACCOUNT_EMAIL" > user_payload.json +set +e +status_code=$(curl ${INSECURE} --write-out %{http_code} --silent --output /dev/null -d @user_payload.json --user "$ADMIN_USERANDPASSWORD" -H "Content-Type: application/json" -X POST "$INDEXER_USER_CREATION_REQUEST") +if [[ "$status_code" -gt 299 || "$status_code" -eq 000 ]] ; then + echo "Status Code: $status_code | An error occurred while creating $INDEXER_SERVICE_USER user in elastic cluster. Exiting..." + exit 1 +fi +set -e + +# clean up +rm -rf role_payload.json +rm -rf user_payload.json +echo "Calling elastic-cluster-credentials.sh" +./elastic-cluster-credentials.sh \ No newline at end of file diff --git a/bootstrap_infra/elastic-search-settings/datastore-entity.py b/bootstrap_infra/elastic-search-settings/datastore-entity.py new file mode 100644 index 0000000000000000000000000000000000000000..a232b4b9c0a64bfd6126e43c8c205d01622d9e92 --- /dev/null +++ b/bootstrap_infra/elastic-search-settings/datastore-entity.py @@ -0,0 +1,63 @@ +# Copyright 2020 Google LLC +# Copyright 2017-2019, Schlumberger +# Copyright 2020 EPAM +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.cloud import datastore +import datetime +import argparse + +class SearchSettings: + def __init__(self, host, port,configuration): + self.host = str(host) + self.port = str(port) + self.configuration = str(configuration) + self.lastupdated = datetime.datetime.now() + + + def getDataStoreEntity(self,client,kind,key): + entity_key = client.key(kind,key) + entity = datastore.Entity(key=entity_key) + entity['host'] = self.host + entity['port'] = self.port + entity['configuration'] = self.configuration + entity['lastupdated'] = self.lastupdated + + return entity + +def create_client(project_id,namespace): + return datastore.Client(project=project_id, namespace=namespace) + +def addEntity(client, entity): + client.put(entity) + +def delete_entity(client, kind, identity): + key = client.key(kind, identity) + client.delete(key) + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--project-id', help='Your cloud project ID.') + parser.add_argument('--namespace', help='Your cloud project namespace.') + parser.add_argument('--kind', help='Your entity kind.') + parser.add_argument('--key', help='Your elastic key.') + parser.add_argument('--host', help='Your elastic host.') + parser.add_argument('--port', help='Your elastic port.') + parser.add_argument('--configuration', help='Your elastic configuration.') + args = parser.parse_args() + + client = create_client(args.project_id,args.namespace) + delete_entity(client,args.kind,args.key) + search=SearchSettings(host=args.host,port=args.port, configuration=args.configuration) + addEntity(client,search.getDataStoreEntity(client=client,kind=args.kind,key=args.key)) diff --git a/bootstrap_infra/elastic-search-settings/elastic-cluster-credentials.sh b/bootstrap_infra/elastic-search-settings/elastic-cluster-credentials.sh new file mode 100755 index 0000000000000000000000000000000000000000..686324df9c86d012c9f92efd1b4bd69c22671a8f --- /dev/null +++ b/bootstrap_infra/elastic-search-settings/elastic-cluster-credentials.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# Copyright 2020 Google LLC +# Copyright 2017-2019, Schlumberger +# Copyright 2020 EPAM +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# This bootstrap script is for setting up a new tenant's elastic cluster credentials on tenant's project datastore + +source ../validate-env.sh $SEARCH_SERVICE_PASSWORD +source ../validate-env.sh $INDEXER_SERVICE_PASSWORD +source ../validate-env.sh $DATA_PARTITION_ID +source ../validate-env.sh $TENANT_NAME + +# TENANT_NAME is evaluated from DATA_PARTITION_ID +#source ../set-tenant-name.sh + +# A new entry named +# "encrypted-credentials" will be created in datastore under [default] -> elastic and have all the encryption data +export ELASTIC_HOST=$ELASTIC_HOST +export ELASTIC_PORT=$ELASTIC_PORT +#export KEYRING="csqp" +export KEY="searchService" + +set +e +KERING_CHECK=$(gcloud kms keyrings list --location=global | grep $KEYRING) +if [[ "$KERING_CHECK" == "" ]] + then + echo "Error. The project [$GCLOUD_PROJECT] doesn't contain the keyring [$KEYRING]" + exit 1 +fi +KEY_CHECK=$(gcloud kms keys list --keyring=$KEYRING --location=global | grep $KEY | grep "ENABLED") +if [[ "$KEY_CHECK" == "" ]] + then + echo "Error. The keyring [$KEYRING] in project [$GCLOUD_PROJECT] doesn't contain an active key [$KEY]" + exit 1 +fi +set -e +# encrypt the credentials by master project's encryption key +export HOST_ENCRYPTION=$(echo $ELASTIC_HOST | gcloud kms encrypt --location=global --keyring=$KEYRING --key=$KEY --plaintext-file=- --ciphertext-file=- --project=$GCLOUD_PROJECT | base64 -w 0) +export PORT_ENCRYPTION=$(echo $ELASTIC_PORT | gcloud kms encrypt --location=global --keyring=$KEYRING --key=$KEY --plaintext-file=- --ciphertext-file=- --project=$GCLOUD_PROJECT | base64 -w 0) +export SEARCH_USERANDPASSWORD_ENCRYPTION=$(echo $SEARCH_SERVICE_USER:$SEARCH_SERVICE_PASSWORD | gcloud kms encrypt --location=global --keyring=$KEYRING --key=$KEY --plaintext-file=- --ciphertext-file=- --project=$GCLOUD_PROJECT | base64 -w 0) +export INDEXER_USERANDPASSWORD_ENCRYPTION=$(echo $INDEXER_SERVICE_USER:$INDEXER_SERVICE_PASSWORD | gcloud kms encrypt --location=global --keyring=$KEYRING --key=$KEY --plaintext-file=- --ciphertext-file=- --project=$GCLOUD_PROJECT | base64 -w 0) + +# insert the indexer credentials into tenant's datastore +python datastore-entity.py --project-id "${TENANT_PROJECT}" --namespace "${TENANT_NAME}" --kind SearchSettings --key indexer-service --host "${HOST_ENCRYPTION}" --port "${PORT_ENCRYPTION}" --configuration "${INDEXER_USERANDPASSWORD_ENCRYPTION}" +echo "Created indexer-service user in datastore" + +# insert the search credentials into tenant's datastore +python datastore-entity.py --project-id "${TENANT_PROJECT}" --namespace "${TENANT_NAME}" --kind SearchSettings --key search-service --host "${HOST_ENCRYPTION}" --port "${PORT_ENCRYPTION}" --configuration "${SEARCH_USERANDPASSWORD_ENCRYPTION}" +echo "Created search-service user in datastore" diff --git a/bootstrap_infra/ingestion-strategy/README.md b/bootstrap_infra/ingestion-strategy/README.md new file mode 100644 index 0000000000000000000000000000000000000000..821d6ba9c8b4f38d711502c78ca4a006f0901bc0 --- /dev/null +++ b/bootstrap_infra/ingestion-strategy/README.md @@ -0,0 +1,13 @@ +# Create gcp.datastore with kind ingestion-strategy + +## About +Ingestion service in datastore mode requires gcp.datastore table ingest-datastore. +From which it could determine which DAG it should to run. + +### Usage +Provide authentication credentials to your application code by setting the environment variable `GOOGLE_APPLICATION_CREDENTIALS`. Replace [PATH] with the file path of the JSON file that contains your service account key. +Be aware that option `--namespace` is project tenant specific, and should be changed accordingly. + +``` +$ python ingest-strategy.py --namespace= --createdby="gcp-default-sa@default-project" --creationtimestamp=1611939505831 --dagname=Osdu_ingest --description="Default workflow for manifest-based ingestion" --version=1 --workflowname=Osdu_ingest +``` diff --git a/bootstrap_infra/ingestion-strategy/ingest-strategy.py b/bootstrap_infra/ingestion-strategy/ingest-strategy.py new file mode 100644 index 0000000000000000000000000000000000000000..596ecd38cd51b8daf126d66522349baa301c4ebc --- /dev/null +++ b/bootstrap_infra/ingestion-strategy/ingest-strategy.py @@ -0,0 +1,76 @@ +# Imports the Google Cloud client library +from google.cloud import datastore +import argparse + +# Create variables +parser = argparse.ArgumentParser() +parser.add_argument('--namespace', help='Your cloud project namespace.') +parser.add_argument('--createdby', help='Your service account name', default=None, type=str) +parser.add_argument('--creationtimestamp', help='Timestamp value', default=None, type=int) +parser.add_argument('--dagname', help='Your DagName field.', default=None, type=str) +parser.add_argument('--description', help='Description for Dag', default=None, type=str) +parser.add_argument('--version', help='Your Dag version', default=None, type=int) +parser.add_argument('--workflowname', help='Your WorkflowName field.', default=None, type=str) + +args = parser.parse_args() + +# Instantiates a client +namespace = args.namespace +datastore_client = datastore.Client(namespace=namespace) + +# The kind for the new entity +kind = 'workflow' +# The name/ID for the new entity could be added as datastore_client.key(kind, name) +# In our case datastore creates autogenerated id +# The Cloud Datastore key for the new entity +task_key = datastore_client.key(kind) + +# Prepares the new entity +# If we create field - task['DAGName'] = Default_dag ; +# This will be presented in datastore as DAGName - dURlZmF1bHRfZGFn , basically as blob variable; +# Correct syntax should be task['DAGName'] = u'Default_dag' , which means convert to unicode string. + +# if statement on whether we need to change variable to unicode-string or keep it None type + +if args.createdby is None: + CreatedBy = args.createdby +else: + #CreatedBy = unicode(args.createdby, "utf-8") + CreatedBy = bytes.decode(args.createdby.encode('utf-8')) +if args.creationtimestamp is None: + CreationTimestamp = args.creationtimestamp +else: + #CreationTimestamp = unicode(args.creationtimestamp, "utf-8") + CreationTimestamp = args.creationtimestamp +if args.dagname is None: + DagName = args.dagname +else: + #DagName = unicode(args.dagname, "utf-8") + DagName = bytes.decode(args.dagname.encode('utf-8')) +if args.description is None: + Description = args.description +else: + #Description = unicode(args.description, "utf-8") + Description = bytes.decode(args.description.encode('utf-8')) +if args.version is None: + Version = args.version +else: + #Version = unicode(args.version, "utf-8") + Version = args.version +if args.workflowname is None: + WorkflowName = args.workflowname +else: + #WorkflowName = unicode(args.workflowname, "utf-8") + WorkflowName = bytes.decode(args.workflowname.encode('utf-8')) + +# Initialize entity, and create fields +task = datastore.Entity(key=task_key) +task['CreatedBy'] = CreatedBy +task['CreationTimestamp'] = CreationTimestamp +task['DagName']= DagName +task['Description'] = Description +task['Version']= Version +task['WorkflowName']= WorkflowName + +# Saves the entity to datastore +datastore_client.put(task) diff --git a/bootstrap_infra/ingestion-strategy/usage-ingest-strategy.txt b/bootstrap_infra/ingestion-strategy/usage-ingest-strategy.txt new file mode 100644 index 0000000000000000000000000000000000000000..672c353b74ab8751bbded53967fa4cf0452b604c --- /dev/null +++ b/bootstrap_infra/ingestion-strategy/usage-ingest-strategy.txt @@ -0,0 +1,3 @@ +Hot-to use: +If datatype for variable should be null, just omit passing this variable to script, like in example below with --userid +$ python strategy.py --namespace=opendes --dagname=Ingest_dag --datatype=osdu --workflowtype=ingest-str diff --git a/bootstrap_infra/requirements.txt b/bootstrap_infra/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..189e0f31ed4582204783c506b0b841c0b15dd1ae --- /dev/null +++ b/bootstrap_infra/requirements.txt @@ -0,0 +1,8 @@ +setuptools==50.3.0 +google-cloud-datastore==1.15.0 +google-api-python-client==1.11.0 +google-auth==1.24.0 +pyyaml==5.3.1 +google-cloud==0.34.0 +google-cloud-pubsub==1.7.0 +google-cloud-runtimeconfig==0.32.0 diff --git a/bootstrap_infra/sql_bootstrap/create-tables.sql b/bootstrap_infra/sql_bootstrap/create-tables.sql new file mode 100644 index 0000000000000000000000000000000000000000..e5479f627871a6d18e621dfbe91639784dbde4da --- /dev/null +++ b/bootstrap_infra/sql_bootstrap/create-tables.sql @@ -0,0 +1,91 @@ +CREATE TABLE IF NOT EXISTS public."group" +( +id bigint NOT NULL GENERATED ALWAYS AS IDENTITY ( INCREMENT 1 START 1 MINVALUE 1 MAXVALUE 9223372036854775807 CACHE 1 ), +name character varying COLLATE pg_catalog."default", +description text COLLATE pg_catalog."default", +email character varying COLLATE pg_catalog."default", +partition_id character varying COLLATE pg_catalog."default", +CONSTRAINT group_pkey PRIMARY KEY (id), +CONSTRAINT group_email_key UNIQUE (email), +CONSTRAINT group_name_key UNIQUE (name) +) + +TABLESPACE pg_default; + +ALTER TABLE public."group" +OWNER to postgres; + +CREATE TABLE IF NOT EXISTS public.app_id +( + id bigint NOT NULL GENERATED ALWAYS AS IDENTITY, + group_id bigint, + app_id character varying, + PRIMARY KEY (id), +CONSTRAINT app_id_group_fk FOREIGN KEY (group_id) +REFERENCES public."group" (id) MATCH SIMPLE +ON UPDATE NO ACTION +ON DELETE NO ACTION +NOT VALID +); + +ALTER TABLE public.app_id +OWNER to postgres; + +CREATE TABLE IF NOT EXISTS public.member +( + id bigint NOT NULL GENERATED ALWAYS AS IDENTITY ( INCREMENT 1 START 1 MINVALUE 1 MAXVALUE 9223372036854775807 CACHE 1 ), +email character varying COLLATE pg_catalog."default", +partition_id character varying COLLATE pg_catalog."default", +CONSTRAINT member_pkey PRIMARY KEY (id), +CONSTRAINT member_email_key UNIQUE (email) +) + +TABLESPACE pg_default; + +ALTER TABLE public.member +OWNER to postgres; + +CREATE TABLE IF NOT EXISTS public.member_to_group +( + group_id bigint NOT NULL, + member_id bigint NOT NULL, + role character varying COLLATE pg_catalog."default", + CONSTRAINT member_to_group_pkey PRIMARY KEY (group_id, member_id), +CONSTRAINT group_as_member_holder_fk FOREIGN KEY (group_id) +REFERENCES public."group" (id) MATCH SIMPLE +ON UPDATE NO ACTION +ON DELETE NO ACTION +NOT VALID, +CONSTRAINT user_as_member_fk FOREIGN KEY (member_id) +REFERENCES public.member (id) MATCH SIMPLE +ON UPDATE NO ACTION +ON DELETE NO ACTION +NOT VALID +) + +TABLESPACE pg_default; + +ALTER TABLE public.member_to_group +OWNER to postgres; + +CREATE TABLE IF NOT EXISTS public.embedded_group +( + parent_id bigint NOT NULL, + child_id bigint NOT NULL, + CONSTRAINT embedded_group_pk PRIMARY KEY (parent_id, child_id), +CONSTRAINT group_as_child_fk FOREIGN KEY (child_id) +REFERENCES public."group" (id) MATCH SIMPLE +ON UPDATE NO ACTION +ON DELETE NO ACTION +NOT VALID, +CONSTRAINT group_as_parent_fk FOREIGN KEY (parent_id) +REFERENCES public."group" (id) MATCH SIMPLE +ON UPDATE NO ACTION +ON DELETE NO ACTION +NOT VALID +) + +TABLESPACE pg_default; + +ALTER TABLE public.embedded_group +OWNER to postgres; diff --git a/bootstrap_infra/validate-env-tenant-name.sh b/bootstrap_infra/validate-env-tenant-name.sh new file mode 100755 index 0000000000000000000000000000000000000000..c094c14fa8e87747efe30a63a7cf69897235ea20 --- /dev/null +++ b/bootstrap_infra/validate-env-tenant-name.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Copyright 2020 Google LLC +# Copyright 2017-2019, Schlumberger +# Copyright 2020 EPAM +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +TENANT_NAME_ORIG=$1 +TENANT_NAME_CHECK="${TENANT_NAME_ORIG//[^a-z0-9]}" +if [[ $TENANT_NAME_ORIG != "$TENANT_NAME_CHECK" ]] +then + echo "ERROR: Invalid variable value TENANT_NAME/DATA_PARTITION_ID. Validate it using regexp [^a-z0-9]." + exit 1 +fi +# tenant name should not be longer than 25 chracters +CHAR_LIMIT=25 +TENANT_NAME_LENGTH=${#TENANT_NAME_ORIG} +if [[ $TENANT_NAME_LENGTH -gt "$CHAR_LIMIT" ]] +then + echo "ERROR: The TENANT_NAME/DATA_PARTITION_ID variable is too long, must be less than $(( CHAR_LIMIT+1 )) characters." + exit 1 +fi diff --git a/bootstrap_infra/validate-env.sh b/bootstrap_infra/validate-env.sh new file mode 100755 index 0000000000000000000000000000000000000000..742df8214780ffde26ab7a6a56c242144d480f54 --- /dev/null +++ b/bootstrap_infra/validate-env.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright 2020 Google LLC +# Copyright 2017-2019, Schlumberger +# Copyright 2020 EPAM +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +if [ "$1" = "" ] +then + echo "Missing environment variable. Please provide all variables and try again" + exit 1 +fi \ No newline at end of file diff --git a/modules/osdu/bootstrap_infra.tf b/modules/osdu/bootstrap_infra.tf index 36267a4c175876492695c1d013dacf2f9ca36dd8..9257017b59f1782a8d36ec4db60adc68428c61bb 100644 --- a/modules/osdu/bootstrap_infra.tf +++ b/modules/osdu/bootstrap_infra.tf @@ -119,7 +119,7 @@ resource "kubernetes_job" "infra_config" { service_account_name = kubernetes_service_account.config.metadata[0].name container { name = "config" - image = "eu.gcr.io/osdu-cicd-epam/os-infra/osdu-deploy-gcp:3a75d1ed" + image = "community.opengroup.org:5555/osdu/platform/deployment-and-operations/infra-gcp-provisioning/osdu-bootstrap_infra" env_from { config_map_ref { name = "${each.value}-infra-config" diff --git a/prod-deploy/admin-ui/osdu-gcp-admin-ui.yml b/prod-deploy/admin-ui/osdu-gcp-admin-ui.yml index 5f9de607614ca5473753801ad858b619e3e2d52b..f0503a42d93086d0bdb4da12402d6517127fe60b 100644 --- a/prod-deploy/admin-ui/osdu-gcp-admin-ui.yml +++ b/prod-deploy/admin-ui/osdu-gcp-admin-ui.yml @@ -29,7 +29,6 @@ build_osdu_gcp: only: refs: - master - - merge_requests build_osdu_gcp_preship: stage: build @@ -72,7 +71,6 @@ deploy_osdu_gcp: only: refs: - master - - merge_requests deploy_osdu_gcp_preship: stage: deploy diff --git a/prod-setup.yml b/prod-setup.yml index cfee61173a23ddf16282bccab9bd651e9028178c..259e52deccbe5688465579aa425bed787970d784 100644 --- a/prod-setup.yml +++ b/prod-setup.yml @@ -1,5 +1,6 @@ stages: - scripts_check - build + - containerize - configuration - deploy