Commit a3354884 authored by Yauheni  Rykhter (EPAM)'s avatar Yauheni Rykhter (EPAM)
Browse files

Merge branch 'master' of...

Merge branch 'master' of https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-gcp-provisioning into GONRG-3676-Segy-to-Zgy-Conversion
parents 6ff3bd8c d39b0c7d
Pipeline #79684 passed with stage
in 26 seconds
# OSDU
This guide describes creating an OSDU set of services within a GCP project.
## Prerequisites
Before you start, ensure that you have configured the evnironment, see
[packages](https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-gcp-provisioning/-/tree/master/examples/simple_osdu#packages) and
[manual configuration](https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-gcp-provisioning/-/tree/master/examples/simple_osdu#manual-actions)
## OSDU Installation Prerequisites
Before OSDU deployment, download in a separate directory the following files:
- main.tf
- outputs.tf
- variables.tf
Then, configure the variables in the variables.tf file:
| Name | Description | Type | Default value | Required |
|------|-------------|------|---------|:--------:|
`service_google_project` | Google project id | string | none | required
`service_google_project_region` | Project region, for example, `us-central1` | string | none | required
`gcloud_region` | Appengine region, for example, `us-central` | string | none | required
`data_google_projects` | List of data Google project ids | list | none | required
`service_google_project_zone` | Project zone, for example, `us-central1-c` | string | none | required
`data_partition_id` | Data partition, for example, `osdu`. The variable should contain only lowercase letters and numbers. Maximum lenght is 25 characters | string | `dataid` | required
`domain` | Domain for OSDU installation | string | none | required
`log_level` | Filter for log entries, for example, INFO | string | `INFO` | required
`bucket_lifetime` | How many days the archived files will be stored in buckets | number | `366` | required
`bucket_location` | Location for buckets in the project. See also: https://cloud.google.com/storage/docs/locations | string | `US` | required
`bucket_storage_class` | Storage classes of the Cloud Storage | string | `MULTI_REGIONAL` | required
`configmap_env` | Environment type | string | `dev` | required
`configmap_rule_set` | Compliance rule set | string | `shared` | required
`configmap_account_ids` | CRM account ids | string | `temp` | required
`elastic_user` | Elasticsearch user | string | `elastic` | required
`elastic_pass` | Elasticsearch password | string | none | required
`elastic_host` | Elasticsearch FQDN, for example, `instance-name.es.us-central1.cloud.es.io` | string | none | required
`elastic_port` | Elasticsearch port | string | `9243` | required
`audiences` | GCP audiences id, for example, `763519302943-n9310xxxxxto6r5lfjsr4is6kjb4.apps.googleusercontent.com. To obtain this variable`, see [creating OAuth 2.0 Client ID of type Web application](https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-gcp-provisioning/-/tree/master/examples/simple_osdu#to-create-oauth-20-client-id-of-type-web-application) | string | none | required
`vpc_network_name` | VPC network name | string | `default` | required
`instance_tier` | Machine type | srting | `db-f1-micro` | required
`database_name` | Database name | string | `entitlements` | required
`sql_deletion_protection` | Database instance deletion protection | bool | `false` | required
`cloud_task_name` | A part of Cloud Tasks queue name | string | `indexer-queue-osdu` | required
`cloud_task_concurrent_dispatches` | The maximum number of concurrent tasks that Cloud Tasks allows to be dispatched for this queue | number | `400` | required
`cloud_task_dispatches_per_second` | The maximum rate at which tasks are dispatched from this queue | number | `100` | required
`cloud_task_retry_attempts` | Number of attempts per task | number | `5` | required
`admin_user_email` | Admin user email in the project or at google identity service | string | `user@example.com` | required
`service_wks_mapping` | WKS kind | string | `WksMapping` | required
`composer_image` | Airflow image version | string | `composer-1.17.0-preview.10-airflow-2.1.1` | required
`gke_version_prefix` | GKE version. If you want to use a specific GKE cluster version you should set a valid prefix (1.X, 1.X.Y, or 1.X.Y-gke.Z) and set the variable `gke_version_attribute` to `latest_version` value | string | none | optional
`gke_release_channel` | GKE release channel.</br> Allowable value:<ul><li> `RAPID`</li><li> `REGULAR`</li><li> `STABLE`</li></ul> To use a specific default release chanel version, set the `gke_version_attribute` equal to `release_channel_default_version` | string | `REGULAR` | required
`gke_version_attribute` | GKE version attribute.</br> Allowable values:<ul><li> `release_channel_default_version`</li><li> `default_version`</li><li> `latest_version`</li></ul> | string | `release_channel_default_version` | required
`gke_node_count` | The number of nodes per instance group | number | `3` | required
`gke_machine_type` | GKE machine type | string | `e2-standard-4` | required
#
To provide variables in command line, use the `-var` option of the `terraform plan` and the `terrafrom apply` commands, for example, `terraform apply -var="domain=example.com"`
## Outputs
| Name | Description |
|------|-------------|
`ingress_ip` | IP address |
`ingress_domain` | Domain name |
`dags_folder` | Airflow DAGs folder |
`airflow_url` | Airflow URL |
`backup_endpoint` | Service backup info endpoint |
`entitlements_endpoint` | Service entitlements info endpoint|
`file_endpoint` | Service file info endpoint |
`indexer_queue_endpoint` | Service indexer queue info endpoint |
`indexer_endpoint` | Service indexer info endpoint |
`legal_endpoint` | Service legal info endpoint |
`notification_endpoint` | Service notification info endpoint |
`partition_endpoint` | Service partition info endpoint |
`policy_endpoint` | Service policy info endpoint |
`register_endpoint` | Service register info endpoint |
`schema_endpoint` | Service schema info endpoint |
`search_endpoint` | Service search info endpoint |
`storage_endpoint` | Service storage info endpoint |
`unit_endpoint` | Service unit info endpoint |
`crs_catalog_endpoint` | Service CRS catalog info endpoint |
`crs_converter_endpoint` | Service CRS converter info endpoint |
`GKE_version` | Full GKE version |
## Provisioning
To provision this example, run the following from the local current directory:
- `terraform init` to install the plugins
![terraform init step](./images/init.png)
- `terraform plan` to view the infrastructure plan
- `terraform apply` to apply the infrastructure build. To confirm, type `yes`.
![terraform apply step](./images/apply.png)
The result is fully provisioned infrastructure and services.
![Provisioning result](./images/result.png)
The IP address and domain dame from the output will be used in [Post install steps](https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-gcp-provisioning/-/tree/master/examples/simple_osdu#post-install-steps).
## Destroying
To destroy built infrastructure run `terraform destroy`. To confirm, type `yes`.
\ No newline at end of file
/**
* Copyright 2021 Google LLC
* Copyright 2021 EPAM
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
provider "google" {
project = var.service_google_project
region = var.service_google_project_region
zone = var.service_google_project_zone
}
module "osdu" {
source = "git::https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-gcp-provisioning.git//modules/osdu"
service_google_project_region = var.service_google_project_region
gcloud_region = var.gcloud_region
service_google_project = var.service_google_project
data_google_projects = var.data_google_projects
data_partition_id = var.data_partition_id
ingress_domain = var.domain
service_google_project_zone = var.service_google_project_zone
log_level = var.log_level
bucket_lifetime = var.bucket_lifetime
bucket_location = var.bucket_location
bucket_storage_class = var.bucket_storage_class
configmap_env = var.configmap_env
configmap_rule_set = var.configmap_rule_set
configmap_account_ids = var.configmap_account_ids
elastic_user = var.elastic_user
elastic_pass = var.elastic_pass
elastic_host = var.elastic_host
elastic_port = var.elastic_port
audiences = var.audiences
vpc_network_name = var.vpc_network_name
instance_tier = var.instance_tier
database_name = var.database_name
sql_deletion_protection = var.sql_deletion_protection
cloud_task_name = var.cloud_task_name
cloud_task_concurrent_dispatches = var.cloud_task_concurrent_dispatches
cloud_task_dispatches_per_second = var.cloud_task_dispatches_per_second
cloud_task_retry_attempts = var.cloud_task_retry_attempts
admin_user_email = var.admin_user_email
service_wks_mapping = var.service_wks_mapping
composer_image = var.composer_image
gke_version_prefix = var.gke_version_prefix
gke_release_channel = var.gke_release_channel
gke_version_attribute = var.gke_version_attribute
gke_node_count = var.gke_node_count
gke_machine_type = var.gke_machine_type
}
/**
* Copyright 2021 Google LLC
* Copyright 2021 EPAM
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
output "ingress_ip" {
value = module.osdu.ingress_ip
}
output "ingress_domain" {
value = module.osdu.ingress_domain
}
output "dags_folder" {
value = module.osdu.dags_folder
}
output "airflow_url" {
value = module.osdu.airflow_url
}
output "backup_endpoint" {
value = "https://${module.osdu.ingress_domain}/api/backup/v1/info"
}
output "entitlements_endpoint" {
value = "https://${module.osdu.ingress_domain}/api/entitlements/v2/info"
}
output "file_endpoint" {
value = "https://${module.osdu.ingress_domain}/api/file/v2/info"
}
output "indexer_queue_endpoint" {
value = "https://${module.osdu.ingress_domain}/api/indexer-queue/v1/info"
}
output "indexer_endpoint" {
value = "https://${module.osdu.ingress_domain}/api/indexer/v2/info"
}
output "legal_endpoint" {
value = "https://${module.osdu.ingress_domain}/api/legal/v1/info"
}
output "notification_endpoint" {
value = "https://${module.osdu.ingress_domain}/api/notification/v1/info"
}
output "partition_endpoint" {
value = "https://${module.osdu.ingress_domain}/api/partition/v1/info"
}
output "policy_endpoint" {
value = "https://${module.osdu.ingress_domain}/api/policy/v1/health"
}
output "register_endpoint" {
value = "https://${module.osdu.ingress_domain}/api/register/v1/info"
}
output "schema_endpoint" {
value = "https://${module.osdu.ingress_domain}/api/schema-service/v1/info"
}
output "search_endpoint" {
value = "https://${module.osdu.ingress_domain}/api/search/v2/info"
}
output "storage_endpoint" {
value = "https://${module.osdu.ingress_domain}/api/storage/v2/info"
}
output "unit_endpoint" {
value = "https://${module.osdu.ingress_domain}/api/unit/info"
}
output "crs_catalog_endpoint" {
value = "https://${module.osdu.ingress_domain}/api/crs/catalog/info"
}
output "crs_converter_endpoint" {
value = "https://${module.osdu.ingress_domain}/api/crs/converter/info"
}
output "GKE_version" {
value = module.osdu.gke_version
}
/**
* Copyright 2021 Google LLC
* Copyright 2021 EPAM
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
# Global inputs #
variable "service_google_project_region" {
description = "Service project region"
type = string
}
variable "gcloud_region" {
description = "Appengine region"
type = string
}
variable "service_google_project" {
description = "The Service Google Project ID"
type = string
}
variable "data_google_projects" {
type = list(string)
description = "List of Data Google Project IDs"
}
variable "data_partition_id" {
description = "Data Partition ID"
type = string
default = "dataid"
}
variable "domain" {
description = "ingress URL address"
type = string
}
variable "service_google_project_zone" {
description = "service project zone"
type = string
}
variable "log_level" {
description = "Log Level"
type = string
default = "INFO"
}
# End of Global inputs #
# Buckets' inputs #
variable "bucket_lifetime" {
description = "How many days your archived files will be stored in buckets"
type = number
default = 366
}
variable "bucket_location" {
description = "Location for buckets in project. Info: https://cloud.google.com/storage/docs/locations"
type = string
default = "US"
}
variable "bucket_storage_class" {
description = "Storage classes offered by Cloud Storage"
type = string
default = "MULTI_REGIONAL"
}
# End of Buckets' inputs #
# Configmap and GKE Secrets' vars #
variable "configmap_env" {
description = "Environment type"
type = string
default = "dev"
}
variable "configmap_rule_set" {
description = "Compliance rule set"
type = string
default = "shared"
}
variable "configmap_account_ids" {
description = "CRM account ids"
type = string
default = "temp"
}
variable "elastic_user" {
description = "User for Elasticsearch"
type = string
default = "elastic"
}
variable "elastic_pass" {
description = "Password to connect to Elasticsearch"
type = string
}
variable "elastic_host" {
description = "Domain name of ElasticSearch instance. It should looks like instance-name.es.us-central1.cloud.es.io"
type = string
}
variable "elastic_port" {
description = "Elasticsearch port"
type = string
default = "9243"
}
# End of configmap variables #
# Legal service vars
variable "audiences" {
description = "audiences GCP credencials client ID"
type = string
}
# End of Legal service vars
# postgresql
variable "vpc_network_name" {
description = "VPC network name"
type = string
default = "default"
}
variable "instance_tier" {
description = "Machine type"
type = string
default = "db-f1-micro"
}
variable "database_name" {
description = "Database name"
type = string
default = "entitlements"
}
variable "sql_deletion_protection" {
description = "Database instance deletion protection"
type = bool
default = false
}
# End of postgresql variables #
# cloud task variables
variable "cloud_task_name" {
type = string
default = "indexer-queue-osdu"
}
variable "cloud_task_concurrent_dispatches" {
type = number
default = 400
}
variable "cloud_task_dispatches_per_second" {
type = number
default = 100
}
variable "cloud_task_retry_attempts" {
type = number
default = 5
}
# end of cloud task variables
variable "admin_user_email" {
description = "Admin person user email in project or at google identity service"
type = string
default = "user@example.com"
}
variable "service_wks_mapping" {
type = string
default = "WksMapping"
}
variable "composer_image" {
type = string
default = "composer-1.17.0-preview.10-airflow-2.1.1"
}
# GKE Cluster variables
variable "gke_version_prefix" {
description = "GKE version"
type = string
default = ""
# If you want to use a specific GKE cluster version you should set a valid prefix (1.X, 1.X.Y, or 1.X.Y-gke.Z) and set the variable "gke_version_attribute" to "latest_version" value
}
variable "gke_release_channel" {
description = "GKE release channel"
type = string
default = "REGULAR"
# Can be set to "RAPID", "REGULAR", "STABLE". If you want to use a specific default release chanel version you should also set "gke_version_attribute" variable to "release_channel_default_version" value
}
variable "gke_version_attribute" {
type = string
default = "release_channel_default_version"
# Can be set to "release_channel_default_version", "default_version", "latest_version"
}
variable "gke_node_count" {
type = number
default = 3
}
variable "gke_machine_type" {
type = string
default = "e2-standard-4"
}
# end of GKE Cluster variables
......@@ -2,8 +2,9 @@
This example illustrates how to create an osdu set of services within a single GCP project.
## Prerequisites
### Operation system
The code works in Debian-based Linux (Debian 10 and Ubuntu 20.04) and Windows WSL 2. Also, it works but is not guaranteed in Google Cloud Shell. All other operating systems, including macOS, are not verified and supported.
### Packages
Packages are only needed for installation from local computer. Please use Google Cloud Shell as it is preconfigured and contains all the required dependencies and software.
......@@ -18,7 +19,7 @@ OSDU requires Elasticsearch engine.
Recomended version for ElasticSearch is **7.10 or higher**. Currently on 18.10.2021, version **8** is not supported.
It can be configured as managed service at [Elactic Cloud](https://cloud.elastic.co/) or on-premises.
From the OSDU module perspective it is vital to get elasticsearch FQDN and password to provision infrastructure.
User name is hardcoded as "elastic" (default Elasticsearch user name).
User name is hardcoded as "elastic" (default Elasticsearch user name).
### Manual actions
......@@ -75,7 +76,7 @@ Go to **GCP navigation menu** -> **API & Services** -> **OAuth consent screen**
![alt text](img/web-credentials.png "Web application credentials")
### OSDU Installation Prerequisites:
### OSDU Installation Prerequisites:
To start installation, use the following commands and steps.
- **Activate Google Cloud Shell in Google Cloud Console**
......@@ -148,7 +149,7 @@ Note the IP address from the output of the command.
### Post install steps
Go to `IAM & Admin` -> `Service Accounts` -> find service account with a name `osdu-sa-airflow-composer`
Then use Actions field three vertical dots, and choose `Manage keys`, proceed with `ADD KEY` button
`Create New Key`, key type pick `JSON`, click `CREATE`. Service account json file will be dowloaded to your PC.
`Create New Key`, key type pick `JSON`, click `CREATE`. Service account json file will be dowloaded to your PC.
Go to `Cloud Storage` -> find bucket with a name ending with `-airflow-sa-bucket` click on this bucket
use `UPLOAD FILES` button, and upload service account json file to the bucket.
......
locals {
project_list = setunion(tolist([var.service_google_project]), var.data_google_projects)
}
module "buckets_for_all_projects" {
bucket_name = [
"${each.value}-entitlements",
"${each.value}-backup-service",
"${each.value}-records",
"${each.value}-${var.data_partition_id}-legal-service-configuration",
"${each.value}-logstore",
"${each.value}-unit-catalog-bucket",
"${each.value}-staging-area",
"${each.value}-persistent-area",
"${each.value}-schema",
"${each.value}-file",
"${each.value}-wks-mapping-definitions",
"${each.value}-airflow-sa-bucket",
"${each.value}-file-dms-bucket"
]
source = "./modules/buckets"
for_each = local.project_list
gcp_project = each.value
bucket_location = var.bucket_location
bucket_lifetime = var.bucket_lifetime
bucket_storage_class = var.bucket_storage_class
}
locals {
project_list = setunion(tolist([var.service_google_project]), var.data_google_projects)
}
module "buckets_for_all_projects" {
bucket_name = [
"${each.value}-backup-service",
"${each.value}-records",
"${each.value}-${var.data_partition_id}-legal-service-configuration",
"${each.value}-logstore",
"${each.value}-unit-catalog-bucket",
"${each.value}-staging-area",
"${each.value}-persistent-area",
"${each.value}-schema",
"${each.value}-file",
"${each.value}-wks-mapping-definitions",
"${each.value}-airflow-sa-bucket",
"${each.value}-file-dms-bucket"
]
source = "./modules/buckets"
for_each = local.project_list
gcp_project = each.value
bucket_location = var.bucket_location
bucket_lifetime = var.bucket_lifetime
bucket_storage_class = var.bucket_storage_class
}
......@@ -25,7 +25,7 @@ module "kubectl" {
cluster_location = google_container_cluster.primary.location
additional_components = ["kubectl", "kpt", "beta"]
kubectl_create_command = "${path.module}/scripts/install_asm.sh ${var.service_google_project} ${google_container_cluster.primary.name} ${google_container_cluster.primary.location} asm-downloads"
kubectl_destroy_command = "${path.module}/scripts/destroy_asm.sh"
kubectl_destroy_command = "${path.module}/scripts/destroy_asm.sh ${var.service_google_project} ${google_container_cluster.primary.name}"
}
resource "null_resource" "service-gateway" {
......
......@@ -2,9 +2,29 @@
set -e
PROJECT=$1
CLUSTER_NAME=$2
#kubectl delete ns asm-system istio-system --ignore-not-found
#kubectl label namespaces --all istio-injection-