Commit 04d831f4 authored by Abhishek Kumar (SLB)'s avatar Abhishek Kumar (SLB)
Browse files

Merge remote-tracking branch 'origin/master' into jackson-databind-whitesource

parents 7873463f 10a70686
Pipeline #107903 failed with stages
in 22 seconds
......@@ -60,6 +60,7 @@ variables:
--set data.serviceAccountName=$OSDU_GCP_SERVICE-k8s
--set data.bootstrap_image=$CI_REGISTRY_IMAGE/osdu-gcp-bootstrap-$OSDU_GCP_SERVICE:$CI_COMMIT_SHORT_SHA
--set data.bootstrapServiceAccountName=datafier-workload-identity-gke
--timeout 15m
# FIXME add value below for DEV2 pipeline
# OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS_DEV2: >
......@@ -125,7 +126,7 @@ include:
- local: "/devops/azure/gitlab-bootstrap.yml"
- local: "/devops/aws/bootstrap.yaml"
- local: "/devops/ibm/bootstrap.yaml"
osdu-gcp-containerize-bootstrap-gitlab:
variables:
BUILD_PATH: devops/$OSDU_GCP_VENDOR/bootstrap-osdu-module/Dockerfile
......
......@@ -7,5 +7,6 @@ COPY ./deployments ./
RUN apt-get update && apt-get install jq -y
RUN pip3 install --upgrade pip && pip3 install -r ./scripts/requirements.txt && pip3 install -r ./scripts/gcp-deployment-requirements.txt
RUN chmod +x /opt/bootstrap_schema.sh
CMD ["/bin/bash", "-c", "source ./bootstrap_schema.sh && sleep 365d"]
CMD ["/bin/bash", "-c", "/opt/bootstrap_schema.sh && sleep 365d"]
#!/usr/bin/env bash
#
# Script that bootstraps schema service using Python scripts, that make requests to schema service
# Contains logic for both onprem and gcp version
#
# Expected environment variables:
# (both environments):
# - DATA_PARTITION
# - SCHEMA_URL
# (for gcp):
# - AUDIENCES
# (for onprem):
# - OPENID_PROVIDER_URL
# - OPENID_PROVIDER_CLIENT_ID
# - OPENID_PROVIDER_CLIENT_SECRET
#
set -e
source ./validate-env.sh "DATA_PARTITION"
source ./validate-env.sh "SCHEMA_URL"
# FIXME find a better solution about a sidecar container readiness
echo "Waiting for a sidecar container is provisioned"
sleep 10
bootstrap_schema_onprem() {
export BEARER_TOKEN="$(curl --location --request POST "${OPENID_PROVIDER_URL}/protocol/openid-connect/token" \
BEARER_TOKEN="$(curl --location --request POST "${OPENID_PROVIDER_URL}/protocol/openid-connect/token" \
--header "Content-Type: application/x-www-form-urlencoded" \
--data-urlencode "grant_type=client_credentials" \
--data-urlencode "scope=openid" \
--data-urlencode "client_id=${OPENID_PROVIDER_CLIENT_ID}" \
--data-urlencode "client_secret=${OPENID_PROVIDER_CLIENT_SECRET}" | jq -r ".id_token")"
export BEARER_TOKEN
echo "Bootstrap Schema Service"
python3 ./scripts/DeploySharedSchemas.py -u ${SCHEMA_URL}/api/schema-service/v1/schemas/system
echo "Bootstrap Schema Service On Prem"
python3 ./scripts/DeploySharedSchemas.py -u "${SCHEMA_URL}"/api/schema-service/v1/schemas/system
}
bootstrap_schema_gcp() {
export BEARER_TOKEN=`gcloud auth print-identity-token --audiences=${AUDIENCES}`
BEARER_TOKEN=$(gcloud auth print-identity-token --audiences="${AUDIENCES}")
export BEARER_TOKEN
echo "Clean-up for Datastore schemas"
python3 ./scripts/GcpDatastoreCleanUp.py
# FIXME find a better solution about datastore cleaning completion
sleep 5
echo "Bootstrap Schema Service"
python3 ./scripts/DeploySharedSchemas.py -u ${SCHEMA_URL}/api/schema-service/v1/schemas/system
echo "Bootstrap Schema Service On GCP"
python3 ./scripts/DeploySharedSchemas.py -u "${SCHEMA_URL}"/api/schema-service/v1/schemas/system
}
......@@ -43,3 +65,5 @@ else
source ./validate-env.sh "AUDIENCES"
bootstrap_schema_gcp
fi
touch /tmp/bootstrap_ready
set -e
#!/bin/bash
# Copyright 2020 Google LLC
# Copyright 2017-2019, Schlumberger
# Copyright 2022 EPAM
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
ENV_VAR_NAME=$1
......
......@@ -20,6 +20,11 @@ spec:
containers:
- name: "{{ .Values.conf.app_name }}-bootstrap"
image: "{{ .Values.data.bootstrap_image }}"
readinessProbe:
exec:
command:
- cat
- /tmp/bootstrap_ready
imagePullPolicy: "{{ .Values.data.imagePullPolicy }}"
envFrom:
- configMapRef:
......
......@@ -102,6 +102,9 @@ curl -L -X PATCH 'http://partition.com/api/partition/v1/partitions/opendes' -H '
### Schema configuration:
```
CREATE SCHEMA IF NOT EXISTS dataecosystem AUTHORIZATION <SCHEMA_POSTGRESQL_USERNAME>;
```
For private tenants:
```
......@@ -117,7 +120,7 @@ CREATE TABLE IF NOT EXISTS dataecosystem.authority
)
TABLESPACE pg_default;
ALTER TABLE IF EXISTS dataecosystem.authority
OWNER to postgres;
OWNER to <SCHEMA_POSTGRESQL_USERNAME>;
-- Index: authority_datagin
-- DROP INDEX IF EXISTS dataecosystem.authority_datagin;
CREATE INDEX IF NOT EXISTS authority_datagin
......@@ -136,16 +139,16 @@ CREATE TABLE IF NOT EXISTS dataecosystem."entityType"
)
TABLESPACE pg_default;
ALTER TABLE IF EXISTS dataecosystem."entityType"
OWNER to postgres;
OWNER to <SCHEMA_POSTGRESQL_USERNAME>;
-- Index: entitytype_datagin
-- DROP INDEX IF EXISTS dataecosystem.entitytype_datagin;
CREATE INDEX IF NOT EXISTS entitytype_datagin
ON dataecosystem."entityType" USING gin
(data)
TABLESPACE pg_default;
-- Table: dataecosystem.schema-osm
-- DROP TABLE IF EXISTS dataecosystem."schema-osm";
CREATE TABLE IF NOT EXISTS dataecosystem."schema-osm"
-- Table: dataecosystem.schema_osm
-- DROP TABLE IF EXISTS dataecosystem."schema_osm";
CREATE TABLE IF NOT EXISTS dataecosystem."schema_osm"
(
id text COLLATE pg_catalog."default" NOT NULL,
pk bigint NOT NULL GENERATED ALWAYS AS IDENTITY ( INCREMENT 1 START 1 MINVALUE 1 MAXVALUE 9223372036854775807 CACHE 1 ),
......@@ -154,12 +157,12 @@ CREATE TABLE IF NOT EXISTS dataecosystem."schema-osm"
CONSTRAINT schemarequest_id UNIQUE (id)
)
TABLESPACE pg_default;
ALTER TABLE IF EXISTS dataecosystem."schema-osm"
OWNER to postgres;
ALTER TABLE IF EXISTS dataecosystem."schema_osm"
OWNER to <SCHEMA_POSTGRESQL_USERNAME>;
-- Index: schemarequest_datagin
-- DROP INDEX IF EXISTS dataecosystem.schemarequest_datagin;
CREATE INDEX IF NOT EXISTS schemarequest_datagin
ON dataecosystem."schema-osm" USING gin
ON dataecosystem."schema_osm" USING gin
(data)
TABLESPACE pg_default;
-- Table: dataecosystem.source
......@@ -174,18 +177,16 @@ CREATE TABLE IF NOT EXISTS dataecosystem.source
)
TABLESPACE pg_default;
ALTER TABLE IF EXISTS dataecosystem.source
OWNER to postgres;
OWNER to <SCHEMA_POSTGRESQL_USERNAME>;
-- Index: source_datagin
-- DROP INDEX IF EXISTS dataecosystem.source_datagin;
CREATE INDEX IF NOT EXISTS source_datagin
ON dataecosystem.source USING gin
(data)
TABLESPACE pg_default;
```
For shared tenant:
-- For shared tenant:
```
-- Table: dataecosystem.system_authority
-- DROP TABLE IF EXISTS dataecosystem.system_authority;
......@@ -194,35 +195,35 @@ CREATE TABLE IF NOT EXISTS dataecosystem.system_authority
id text COLLATE pg_catalog."default" NOT NULL,
pk bigint NOT NULL GENERATED ALWAYS AS IDENTITY ( INCREMENT 1 START 1 MINVALUE 1 MAXVALUE 9223372036854775807 CACHE 1 ),
data jsonb NOT NULL,
CONSTRAINT "Authority_pkey" PRIMARY KEY (pk),
CONSTRAINT authority_id UNIQUE (id)
CONSTRAINT "Authority_pkey_system" PRIMARY KEY (pk),
CONSTRAINT authority_id_system UNIQUE (id)
)
TABLESPACE pg_default;
ALTER TABLE IF EXISTS dataecosystem.system_authority
OWNER to postgres;
OWNER to <SCHEMA_POSTGRESQL_USERNAME>;
-- Index: system_authority_datagin
-- DROP INDEX IF EXISTS dataecosystem.system_authority_datagin;
CREATE INDEX IF NOT EXISTS system_authority_datagin
ON dataecosystem.system_authority USING gin
(data)
TABLESPACE pg_default;
-- Table: dataecosystem.system_entityType
-- DROP TABLE IF EXISTS dataecosystem."system_entityType";
CREATE TABLE IF NOT EXISTS dataecosystem."system_entityType"
-- Table: dataecosystem.system_entity_type
-- DROP TABLE IF EXISTS dataecosystem."system_entity_type";
CREATE TABLE IF NOT EXISTS dataecosystem."system_entity_type"
(
id text COLLATE pg_catalog."default" NOT NULL,
pk bigint NOT NULL GENERATED ALWAYS AS IDENTITY ( INCREMENT 1 START 1 MINVALUE 1 MAXVALUE 9223372036854775807 CACHE 1 ),
data jsonb NOT NULL,
CONSTRAINT "EntityType_pkey" PRIMARY KEY (pk),
CONSTRAINT entitytype_id UNIQUE (id)
CONSTRAINT "EntityType_pkey_system" PRIMARY KEY (pk),
CONSTRAINT entitytype_id_system UNIQUE (id)
)
TABLESPACE pg_default;
ALTER TABLE IF EXISTS dataecosystem."system_entityType"
OWNER to postgres;
-- Index: system_entityType_datagin
-- DROP INDEX IF EXISTS dataecosystem.system_entityType_datagin;
CREATE INDEX IF NOT EXISTS system_entityType_datagin
ON dataecosystem."system_entityType" USING gin
ALTER TABLE IF EXISTS dataecosystem."system_entity_type"
OWNER to <SCHEMA_POSTGRESQL_USERNAME>;
-- Index: system_entity_type_datagin
-- DROP INDEX IF EXISTS dataecosystem.system_entity_type_datagin;
CREATE INDEX IF NOT EXISTS system_entity_type_datagin
ON dataecosystem."system_entity_type" USING gin
(data)
TABLESPACE pg_default;
-- Table: dataecosystem.system_schema_osm
......@@ -232,12 +233,12 @@ CREATE TABLE IF NOT EXISTS dataecosystem."system_schema_osm"
id text COLLATE pg_catalog."default" NOT NULL,
pk bigint NOT NULL GENERATED ALWAYS AS IDENTITY ( INCREMENT 1 START 1 MINVALUE 1 MAXVALUE 9223372036854775807 CACHE 1 ),
data jsonb NOT NULL,
CONSTRAINT "Schema_pkey" PRIMARY KEY (pk),
CONSTRAINT schemarequest_id UNIQUE (id)
CONSTRAINT "Schema_pkey_system" PRIMARY KEY (pk),
CONSTRAINT schemarequest_id_system UNIQUE (id)
)
TABLESPACE pg_default;
ALTER TABLE IF EXISTS dataecosystem."system_schema_osm"
OWNER to postgres;
OWNER to <SCHEMA_POSTGRESQL_USERNAME>;
-- Index: schemarequest_datagin
-- DROP INDEX IF EXISTS dataecosystem.schemarequest_datagin;
CREATE INDEX IF NOT EXISTS schemarequest_datagin
......@@ -251,22 +252,20 @@ CREATE TABLE IF NOT EXISTS dataecosystem.system_source
id text COLLATE pg_catalog."default" NOT NULL,
pk bigint NOT NULL GENERATED ALWAYS AS IDENTITY ( INCREMENT 1 START 1 MINVALUE 1 MAXVALUE 9223372036854775807 CACHE 1 ),
data jsonb NOT NULL,
CONSTRAINT "Source_pkey" PRIMARY KEY (pk),
CONSTRAINT source_id UNIQUE (id)
CONSTRAINT "Source_pkey_system" PRIMARY KEY (pk),
CONSTRAINT source_id_system UNIQUE (id)
)
TABLESPACE pg_default;
ALTER TABLE IF EXISTS dataecosystem.system_source
OWNER to postgres;
OWNER to <SCHEMA_POSTGRESQL_USERNAME>;
-- Index: system_source_datagin
-- DROP INDEX IF EXISTS dataecosystem.system_source_datagin;
CREATE INDEX IF NOT EXISTS system_source_datagin
ON dataecosystem.system_source USING gin
(data)
TABLESPACE pg_default;
```
## RabbitMQ configuration:
### Properties set in Partition service:
......@@ -428,7 +427,7 @@ These buckets must be defined in tenants’ dedicated object store servers. OBM
</td>
</tr>
<tr>
<td>&lt;PartitionInfo.projectId>-<strong>schema</strong>
<td>&lt;PartitionInfo.projectId-PartitionInfo.name>-<strong>schema</strong>
</td>
<td>ListObjects, CRUDObject
</td>
......@@ -445,7 +444,7 @@ For shared tenant only:
</td>
</tr>
<tr>
<td>&lt;PartitionInfo.projectId><strong>-system-schema</strong>
<td>&lt;PartitionInfo.projectId-PartitionInfo.name><strong>-system-schema</strong>
</td>
<td>ListObjects, CRUDObject
</td>
......
......@@ -77,7 +77,7 @@ At Google cloud storage should be created bucket:
</td>
</tr>
<tr>
<td>&lt;PartitionInfo.projectId><strong>-schema</strong>
<td>&lt;PartitionInfo.projectId-PartitionInfo.name><strong>-schema</strong>
</td>
<td>ListObjects, CRUDObject
</td>
......@@ -96,7 +96,7 @@ At Google cloud storage should be created bucket:
</td>
</tr>
<tr>
<td>&lt;PartitionInfo.projectId><strong>-system-schema</strong>
<td>&lt;PartitionInfo.projectId-PartitionInfo.name><strong>-system-schema</strong>
</td>
<td>ListObjects, CRUDObject
</td>
......
/*
Copyright 2021 Google LLC
Copyright 2021 EPAM Systems, Inc
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
* Copyright 2020-2022 Google LLC
* Copyright 2020-2022 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.schema;
......
/*
Copyright 2021 Google LLC
Copyright 2021 EPAM Systems, Inc
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
* Copyright 2020-2022 Google LLC
* Copyright 2020-2022 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.schema.configuration;
......
/*
* Copyright 2020-2022 Google LLC
* Copyright 2020-2022 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.schema.configuration;
import lombok.Getter;
......@@ -11,5 +28,5 @@ import org.springframework.context.annotation.Configuration;
@Setter
public class PropertiesConfiguration {
private String sharedTenantName;
private String sharedTenantName;
}
......@@ -22,38 +22,42 @@ import org.opengroup.osdu.core.gcp.oqm.model.OqmDestination;
public interface DestinationProvider<DestinationT> {
/**
* The method used only for destinations without kind (i.e. {@link OqmDestination})
* @param partitionId the id of partition
* @return the destination for OSDU Mappers
*/
DestinationT getDestination(String partitionId);
/**
* The method used only for destinations without kind (i.e. {@link OqmDestination})
*
* @param partitionId the id of partition
* @return the destination for OSDU Mappers
*/
DestinationT getDestination(String partitionId);
/**
* The method used only for destinations with kind (i.e. {@link org.opengroup.osdu.core.gcp.osm.model.Destination})
* You can also pass empty string for the kind name if you want to use this method for building other destinations
* @param partitionId the id of partition
* @param kindName the name of the kind
* @return the destination for OSDU Mappers
*/
DestinationT getDestination(String partitionId, String kindName);
/**
* The method used only for destinations with kind (i.e. {@link org.opengroup.osdu.core.gcp.osm.model.Destination})
* You can also pass empty string for the kind name if you want to use this method for building
* other destinations
*
* @param partitionId the id of partition
* @param kindName the name of the kind
* @return the destination for OSDU Mappers
*/
DestinationT getDestination(String partitionId, String kindName);
/**
* The method used if tenantInfo already acknowledged and there is no need to call TenantFactory to get the info
*
* @param tenantInfo tenant info got from somewhere
* @param kindName the name of the kind
* @return the destination for OSDU Mappers
*/
DestinationT getDestination(TenantInfo tenantInfo, String kindName);
/**
* The method used if tenantInfo already acknowledged and there is no need to call TenantFactory
* to get the info
*
* @param tenantInfo tenant info got from somewhere
* @param kindName the name of the kind
* @return the destination for OSDU Mappers
*/
DestinationT getDestination(TenantInfo tenantInfo, String kindName);
/**
* The method is used only for custom namespace and kind usage
*
* @param partitionId partitionId for destination
* @param namespace custom namespace
* @param kindName the name of the kind
* @return the destination for OSDU Mappers
*/
DestinationT getDestination(String partitionId, String namespace, String kindName);
/**
* The method is used only for custom namespace and kind usage
*
* @param partitionId partitionId for destination
* @param namespace custom namespace
* @param kindName the name of the kind
* @return the destination for OSDU Mappers
*/
DestinationT getDestination(String partitionId, String namespace, String kindName);
}
......@@ -28,46 +28,47 @@ import org.springframework.stereotype.Service;
@Service
@Slf4j
public abstract class DestinationProviderImpl<DestinationT> implements DestinationProvider<DestinationT> {
public abstract class DestinationProviderImpl<DestinationT> implements
DestinationProvider<DestinationT> {
private final ITenantFactory tenantFactory;
private final ITenantFactory tenantFactory;
@Autowired
public DestinationProviderImpl(ITenantFactory tenantFactory) {
this.tenantFactory = tenantFactory;
}
@Autowired
protected DestinationProviderImpl(ITenantFactory tenantFactory) {
this.tenantFactory = tenantFactory;
}
@Override
public DestinationT getDestination(String partitionId) {
TenantInfo tenantInfo = tenantFactory.getTenantInfo(partitionId);
return getDestination(tenantInfo, "");
}
@Override
public DestinationT getDestination(String partitionId) {
TenantInfo tenantInfo = tenantFactory.getTenantInfo(partitionId);
return getDestination(tenantInfo, "");
}
@Override
public DestinationT getDestination(String partitionId, String kindName) {
TenantInfo tenantInfo = tenantFactory.getTenantInfo(partitionId);
return getDestination(tenantInfo, kindName);
}
@Override
public DestinationT getDestination(String partitionId, String kindName) {
TenantInfo tenantInfo = tenantFactory.getTenantInfo(partitionId);
return getDestination(tenantInfo, kindName);
}
@Override
public DestinationT getDestination(TenantInfo tenantInfo, String kindName) {
log.debug("Providing destination for the tenant: " + tenantInfo.getName());
String partitionId = tenantInfo.getDataPartitionId();
String namespace = tenantInfo.getName();
return getDestination(partitionId, namespace, kindName);
}
@Override
public DestinationT getDestination(TenantInfo tenantInfo, String kindName) {
log.debug("Providing destination for the tenant: " + tenantInfo.getName());
String partitionId = tenantInfo.getDataPartitionId();
String namespace = tenantInfo.getName();
return getDestination(partitionId, namespace, kindName);
}
@Override
public DestinationT getDestination(String partitionId, String namespace, String kindName) {
@Override
public DestinationT getDestination(String partitionId, String namespace, String kindName) {
DestinationInstructions instructions = DestinationInstructions.builder()
.dataPartition(partitionId)
.namespace(new Namespace(namespace))
.kind(new Kind(kindName))
.build();
DestinationInstructions instructions = DestinationInstructions.builder()
.dataPartition(partitionId)
.namespace(new Namespace(namespace))
.kind(new Kind(kindName))
.build();
return buildDestination(instructions);
}
return buildDestination(instructions);
}
protected abstract DestinationT buildDestination(DestinationInstructions instructions);
protected abstract DestinationT buildDestination(DestinationInstructions instructions);
}
......@@ -17,6 +17,8 @@
package org.opengroup.osdu.schema.destination.provider.impl;
import static org.opengroup