Commit e7edc9a3 authored by Riabokon Stanislav(EPAM)[GCP]'s avatar Riabokon Stanislav(EPAM)[GCP]
Browse files

Rename buckets. Updated Schema configuration for Anthos in README.md.

Refactoring code.
parent 5ebf1dff
Pipeline #107037 failed with stages
in 108 minutes and 33 seconds
......@@ -102,6 +102,9 @@ curl -L -X PATCH 'http://partition.com/api/partition/v1/partitions/opendes' -H '
### Schema configuration:
```
CREATE SCHEMA IF NOT EXISTS dataecosystem AUTHORIZATION <SCHEMA_POSTGRESQL_USERNAME>;
```
For private tenants:
```
......@@ -117,7 +120,7 @@ CREATE TABLE IF NOT EXISTS dataecosystem.authority
)
TABLESPACE pg_default;
ALTER TABLE IF EXISTS dataecosystem.authority
OWNER to postgres;
OWNER to <SCHEMA_POSTGRESQL_USERNAME>;
-- Index: authority_datagin
-- DROP INDEX IF EXISTS dataecosystem.authority_datagin;
CREATE INDEX IF NOT EXISTS authority_datagin
......@@ -136,16 +139,16 @@ CREATE TABLE IF NOT EXISTS dataecosystem."entityType"
)
TABLESPACE pg_default;
ALTER TABLE IF EXISTS dataecosystem."entityType"
OWNER to postgres;
OWNER to <SCHEMA_POSTGRESQL_USERNAME>;
-- Index: entitytype_datagin
-- DROP INDEX IF EXISTS dataecosystem.entitytype_datagin;
CREATE INDEX IF NOT EXISTS entitytype_datagin
ON dataecosystem."entityType" USING gin
(data)
TABLESPACE pg_default;
-- Table: dataecosystem.schema-osm
-- DROP TABLE IF EXISTS dataecosystem."schema-osm";
CREATE TABLE IF NOT EXISTS dataecosystem."schema-osm"
-- Table: dataecosystem.schema_osm
-- DROP TABLE IF EXISTS dataecosystem."schema_osm";
CREATE TABLE IF NOT EXISTS dataecosystem."schema_osm"
(
id text COLLATE pg_catalog."default" NOT NULL,
pk bigint NOT NULL GENERATED ALWAYS AS IDENTITY ( INCREMENT 1 START 1 MINVALUE 1 MAXVALUE 9223372036854775807 CACHE 1 ),
......@@ -154,12 +157,12 @@ CREATE TABLE IF NOT EXISTS dataecosystem."schema-osm"
CONSTRAINT schemarequest_id UNIQUE (id)
)
TABLESPACE pg_default;
ALTER TABLE IF EXISTS dataecosystem."schema-osm"
OWNER to postgres;
ALTER TABLE IF EXISTS dataecosystem."schema_osm"
OWNER to <SCHEMA_POSTGRESQL_USERNAME>;
-- Index: schemarequest_datagin
-- DROP INDEX IF EXISTS dataecosystem.schemarequest_datagin;
CREATE INDEX IF NOT EXISTS schemarequest_datagin
ON dataecosystem."schema-osm" USING gin
ON dataecosystem."schema_osm" USING gin
(data)
TABLESPACE pg_default;
-- Table: dataecosystem.source
......@@ -174,18 +177,16 @@ CREATE TABLE IF NOT EXISTS dataecosystem.source
)
TABLESPACE pg_default;
ALTER TABLE IF EXISTS dataecosystem.source
OWNER to postgres;
OWNER to <SCHEMA_POSTGRESQL_USERNAME>;
-- Index: source_datagin
-- DROP INDEX IF EXISTS dataecosystem.source_datagin;
CREATE INDEX IF NOT EXISTS source_datagin
ON dataecosystem.source USING gin
(data)
TABLESPACE pg_default;
```
For shared tenant:
-- For shared tenant:
```
-- Table: dataecosystem.system_authority
-- DROP TABLE IF EXISTS dataecosystem.system_authority;
......@@ -194,35 +195,35 @@ CREATE TABLE IF NOT EXISTS dataecosystem.system_authority
id text COLLATE pg_catalog."default" NOT NULL,
pk bigint NOT NULL GENERATED ALWAYS AS IDENTITY ( INCREMENT 1 START 1 MINVALUE 1 MAXVALUE 9223372036854775807 CACHE 1 ),
data jsonb NOT NULL,
CONSTRAINT "Authority_pkey" PRIMARY KEY (pk),
CONSTRAINT authority_id UNIQUE (id)
CONSTRAINT "Authority_pkey_system" PRIMARY KEY (pk),
CONSTRAINT authority_id_system UNIQUE (id)
)
TABLESPACE pg_default;
ALTER TABLE IF EXISTS dataecosystem.system_authority
OWNER to postgres;
OWNER to <SCHEMA_POSTGRESQL_USERNAME>;
-- Index: system_authority_datagin
-- DROP INDEX IF EXISTS dataecosystem.system_authority_datagin;
CREATE INDEX IF NOT EXISTS system_authority_datagin
ON dataecosystem.system_authority USING gin
(data)
TABLESPACE pg_default;
-- Table: dataecosystem.system_entityType
-- DROP TABLE IF EXISTS dataecosystem."system_entityType";
CREATE TABLE IF NOT EXISTS dataecosystem."system_entityType"
-- Table: dataecosystem.system_entity_type
-- DROP TABLE IF EXISTS dataecosystem."system_entity_type";
CREATE TABLE IF NOT EXISTS dataecosystem."system_entity_type"
(
id text COLLATE pg_catalog."default" NOT NULL,
pk bigint NOT NULL GENERATED ALWAYS AS IDENTITY ( INCREMENT 1 START 1 MINVALUE 1 MAXVALUE 9223372036854775807 CACHE 1 ),
data jsonb NOT NULL,
CONSTRAINT "EntityType_pkey" PRIMARY KEY (pk),
CONSTRAINT entitytype_id UNIQUE (id)
CONSTRAINT "EntityType_pkey_system" PRIMARY KEY (pk),
CONSTRAINT entitytype_id_system UNIQUE (id)
)
TABLESPACE pg_default;
ALTER TABLE IF EXISTS dataecosystem."system_entityType"
OWNER to postgres;
-- Index: system_entityType_datagin
-- DROP INDEX IF EXISTS dataecosystem.system_entityType_datagin;
CREATE INDEX IF NOT EXISTS system_entityType_datagin
ON dataecosystem."system_entityType" USING gin
ALTER TABLE IF EXISTS dataecosystem."system_entity_type"
OWNER to <SCHEMA_POSTGRESQL_USERNAME>;
-- Index: system_entity_type_datagin
-- DROP INDEX IF EXISTS dataecosystem.system_entity_type_datagin;
CREATE INDEX IF NOT EXISTS system_entity_type_datagin
ON dataecosystem."system_entity_type" USING gin
(data)
TABLESPACE pg_default;
-- Table: dataecosystem.system_schema_osm
......@@ -232,12 +233,12 @@ CREATE TABLE IF NOT EXISTS dataecosystem."system_schema_osm"
id text COLLATE pg_catalog."default" NOT NULL,
pk bigint NOT NULL GENERATED ALWAYS AS IDENTITY ( INCREMENT 1 START 1 MINVALUE 1 MAXVALUE 9223372036854775807 CACHE 1 ),
data jsonb NOT NULL,
CONSTRAINT "Schema_pkey" PRIMARY KEY (pk),
CONSTRAINT schemarequest_id UNIQUE (id)
CONSTRAINT "Schema_pkey_system" PRIMARY KEY (pk),
CONSTRAINT schemarequest_id_system UNIQUE (id)
)
TABLESPACE pg_default;
ALTER TABLE IF EXISTS dataecosystem."system_schema_osm"
OWNER to postgres;
OWNER to <SCHEMA_POSTGRESQL_USERNAME>;
-- Index: schemarequest_datagin
-- DROP INDEX IF EXISTS dataecosystem.schemarequest_datagin;
CREATE INDEX IF NOT EXISTS schemarequest_datagin
......@@ -251,22 +252,20 @@ CREATE TABLE IF NOT EXISTS dataecosystem.system_source
id text COLLATE pg_catalog."default" NOT NULL,
pk bigint NOT NULL GENERATED ALWAYS AS IDENTITY ( INCREMENT 1 START 1 MINVALUE 1 MAXVALUE 9223372036854775807 CACHE 1 ),
data jsonb NOT NULL,
CONSTRAINT "Source_pkey" PRIMARY KEY (pk),
CONSTRAINT source_id UNIQUE (id)
CONSTRAINT "Source_pkey_system" PRIMARY KEY (pk),
CONSTRAINT source_id_system UNIQUE (id)
)
TABLESPACE pg_default;
ALTER TABLE IF EXISTS dataecosystem.system_source
OWNER to postgres;
OWNER to <SCHEMA_POSTGRESQL_USERNAME>;
-- Index: system_source_datagin
-- DROP INDEX IF EXISTS dataecosystem.system_source_datagin;
CREATE INDEX IF NOT EXISTS system_source_datagin
ON dataecosystem.system_source USING gin
(data)
TABLESPACE pg_default;
```
## RabbitMQ configuration:
### Properties set in Partition service:
......@@ -428,7 +427,7 @@ These buckets must be defined in tenants’ dedicated object store servers. OBM
</td>
</tr>
<tr>
<td>&lt;PartitionInfo.projectId>-<strong>schema</strong>
<td>&lt;PartitionInfo.projectId-PartitionInfo.name>-<strong>schema</strong>
</td>
<td>ListObjects, CRUDObject
</td>
......@@ -445,7 +444,7 @@ For shared tenant only:
</td>
</tr>
<tr>
<td>&lt;PartitionInfo.projectId><strong>-system-schema</strong>
<td>&lt;PartitionInfo.projectId-PartitionInfo.name><strong>-system-schema</strong>
</td>
<td>ListObjects, CRUDObject
</td>
......
......@@ -77,7 +77,7 @@ At Google cloud storage should be created bucket:
</td>
</tr>
<tr>
<td>&lt;PartitionInfo.projectId><strong>-schema</strong>
<td>&lt;PartitionInfo.projectId-PartitionInfo.name><strong>-schema</strong>
</td>
<td>ListObjects, CRUDObject
</td>
......@@ -96,7 +96,7 @@ At Google cloud storage should be created bucket:
</td>
</tr>
<tr>
<td>&lt;PartitionInfo.projectId><strong>-system-schema</strong>
<td>&lt;PartitionInfo.projectId-PartitionInfo.name><strong>-system-schema</strong>
</td>
<td>ListObjects, CRUDObject
</td>
......
/*
Copyright 2021 Google LLC
Copyright 2021 EPAM Systems, Inc
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
* Copyright 2020-2022 Google LLC
* Copyright 2020-2022 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.schema;
......
/*
Copyright 2021 Google LLC
Copyright 2021 EPAM Systems, Inc
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
* Copyright 2020-2022 Google LLC
* Copyright 2020-2022 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.schema.configuration;
......
/*
* Copyright 2020-2022 Google LLC
* Copyright 2020-2022 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.schema.configuration;
import lombok.Getter;
......@@ -11,5 +28,5 @@ import org.springframework.context.annotation.Configuration;
@Setter
public class PropertiesConfiguration {
private String sharedTenantName;
private String sharedTenantName;
}
......@@ -22,38 +22,42 @@ import org.opengroup.osdu.core.gcp.oqm.model.OqmDestination;
public interface DestinationProvider<DestinationT> {
/**
* The method used only for destinations without kind (i.e. {@link OqmDestination})
* @param partitionId the id of partition
* @return the destination for OSDU Mappers
*/
DestinationT getDestination(String partitionId);
/**
* The method used only for destinations without kind (i.e. {@link OqmDestination})
*
* @param partitionId the id of partition
* @return the destination for OSDU Mappers
*/
DestinationT getDestination(String partitionId);
/**
* The method used only for destinations with kind (i.e. {@link org.opengroup.osdu.core.gcp.osm.model.Destination})
* You can also pass empty string for the kind name if you want to use this method for building other destinations
* @param partitionId the id of partition
* @param kindName the name of the kind
* @return the destination for OSDU Mappers
*/
DestinationT getDestination(String partitionId, String kindName);
/**
* The method used only for destinations with kind (i.e. {@link org.opengroup.osdu.core.gcp.osm.model.Destination})
* You can also pass empty string for the kind name if you want to use this method for building
* other destinations
*
* @param partitionId the id of partition
* @param kindName the name of the kind
* @return the destination for OSDU Mappers
*/
DestinationT getDestination(String partitionId, String kindName);
/**
* The method used if tenantInfo already acknowledged and there is no need to call TenantFactory to get the info
*
* @param tenantInfo tenant info got from somewhere
* @param kindName the name of the kind
* @return the destination for OSDU Mappers
*/
DestinationT getDestination(TenantInfo tenantInfo, String kindName);
/**
* The method used if tenantInfo already acknowledged and there is no need to call TenantFactory
* to get the info
*
* @param tenantInfo tenant info got from somewhere
* @param kindName the name of the kind
* @return the destination for OSDU Mappers
*/
DestinationT getDestination(TenantInfo tenantInfo, String kindName);
/**
* The method is used only for custom namespace and kind usage
*
* @param partitionId partitionId for destination
* @param namespace custom namespace
* @param kindName the name of the kind
* @return the destination for OSDU Mappers
*/
DestinationT getDestination(String partitionId, String namespace, String kindName);
/**
* The method is used only for custom namespace and kind usage
*
* @param partitionId partitionId for destination
* @param namespace custom namespace
* @param kindName the name of the kind
* @return the destination for OSDU Mappers
*/
DestinationT getDestination(String partitionId, String namespace, String kindName);
}
......@@ -28,46 +28,47 @@ import org.springframework.stereotype.Service;
@Service
@Slf4j
public abstract class DestinationProviderImpl<DestinationT> implements DestinationProvider<DestinationT> {
public abstract class DestinationProviderImpl<DestinationT> implements
DestinationProvider<DestinationT> {
private final ITenantFactory tenantFactory;
private final ITenantFactory tenantFactory;
@Autowired
public DestinationProviderImpl(ITenantFactory tenantFactory) {
this.tenantFactory = tenantFactory;
}
@Autowired
protected DestinationProviderImpl(ITenantFactory tenantFactory) {
this.tenantFactory = tenantFactory;
}
@Override
public DestinationT getDestination(String partitionId) {
TenantInfo tenantInfo = tenantFactory.getTenantInfo(partitionId);
return getDestination(tenantInfo, "");
}
@Override
public DestinationT getDestination(String partitionId) {
TenantInfo tenantInfo = tenantFactory.getTenantInfo(partitionId);
return getDestination(tenantInfo, "");
}
@Override
public DestinationT getDestination(String partitionId, String kindName) {
TenantInfo tenantInfo = tenantFactory.getTenantInfo(partitionId);
return getDestination(tenantInfo, kindName);
}
@Override
public DestinationT getDestination(String partitionId, String kindName) {
TenantInfo tenantInfo = tenantFactory.getTenantInfo(partitionId);
return getDestination(tenantInfo, kindName);
}
@Override
public DestinationT getDestination(TenantInfo tenantInfo, String kindName) {
log.debug("Providing destination for the tenant: " + tenantInfo.getName());
String partitionId = tenantInfo.getDataPartitionId();
String namespace = tenantInfo.getName();
return getDestination(partitionId, namespace, kindName);
}
@Override
public DestinationT getDestination(TenantInfo tenantInfo, String kindName) {
log.debug("Providing destination for the tenant: " + tenantInfo.getName());
String partitionId = tenantInfo.getDataPartitionId();
String namespace = tenantInfo.getName();
return getDestination(partitionId, namespace, kindName);
}
@Override
public DestinationT getDestination(String partitionId, String namespace, String kindName) {
@Override
public DestinationT getDestination(String partitionId, String namespace, String kindName) {
DestinationInstructions instructions = DestinationInstructions.builder()
.dataPartition(partitionId)
.namespace(new Namespace(namespace))
.kind(new Kind(kindName))
.build();
DestinationInstructions instructions = DestinationInstructions.builder()
.dataPartition(partitionId)
.namespace(new Namespace(namespace))
.kind(new Kind(kindName))
.build();
return buildDestination(instructions);
}
return buildDestination(instructions);
}
protected abstract DestinationT buildDestination(DestinationInstructions instructions);
protected abstract DestinationT buildDestination(DestinationInstructions instructions);
}
......@@ -17,6 +17,8 @@
package org.opengroup.osdu.schema.destination.provider.impl;
import static org.opengroup.osdu.schema.destination.provider.model.DestinationInstructions.toObmDestination;
import lombok.extern.slf4j.Slf4j;
import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
import org.opengroup.osdu.core.gcp.obm.persistence.ObmDestination;
......@@ -25,19 +27,17 @@ import org.opengroup.osdu.schema.destination.provider.model.DestinationInstructi
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import static org.opengroup.osdu.schema.destination.provider.model.DestinationInstructions.toObmDestination;
@Component
@Slf4j
public class ObmDestinationProvider extends DestinationProviderImpl<ObmDestination> {
@Autowired
public ObmDestinationProvider(ITenantFactory tenantFactory) {
super(tenantFactory);
}
@Autowired
public ObmDestinationProvider(ITenantFactory tenantFactory) {
super(tenantFactory);
}
@Override
protected ObmDestination buildDestination(DestinationInstructions instructions) {
return toObmDestination(instructions);
}
@Override
protected ObmDestination buildDestination(DestinationInstructions instructions) {
return toObmDestination(instructions);
}
}
......@@ -17,6 +17,8 @@
package org.opengroup.osdu.schema.destination.provider.impl;
import static org.opengroup.osdu.schema.destination.provider.model.DestinationInstructions.toOqmDestination;
import lombok.extern.slf4j.Slf4j;
import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
import org.opengroup.osdu.core.gcp.oqm.model.OqmDestination;
......@@ -25,19 +27,17 @@ import org.opengroup.osdu.schema.destination.provider.model.DestinationInstructi
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import static org.opengroup.osdu.schema.destination.provider.model.DestinationInstructions.toOqmDestination;
@Component
@Slf4j
public class OqmDestinationProvider extends DestinationProviderImpl<OqmDestination> {
@Autowired
public OqmDestinationProvider(ITenantFactory tenantFactory) {
super(tenantFactory);
}
@Autowired
public OqmDestinationProvider(ITenantFactory tenantFactory) {
super(tenantFactory);
}
@Override
protected OqmDestination buildDestination(DestinationInstructions instructions) {
return toOqmDestination(instructions);
}
@Override
protected OqmDestination buildDestination(DestinationInstructions instructions) {
return toOqmDestination(instructions);
}
}
......@@ -17,6 +17,8 @@
package org.opengroup.osdu.schema.destination.provider.impl;
import static org.opengroup.osdu.schema.destination.provider.model.DestinationInstructions.toOsmDestination;
import lombok.extern.slf4j.Slf4j;
import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
import org.opengroup.osdu.core.gcp.osm.model.Destination;
......@@ -25,19 +27,17 @@ import org.opengroup.osdu.schema.destination.provider.model.DestinationInstructi
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import static org.opengroup.osdu.schema.destination.provider.model.DestinationInstructions.toOsmDestination;
@Service
@Slf4j
public class OsmDestinationProvider extends DestinationProviderImpl<Destination> {
@Autowired
public OsmDestinationProvider(ITenantFactory tenantFactory) {
super(tenantFactory);
}
@Autowired
public OsmDestinationProvider(ITenantFactory tenantFactory) {
super(tenantFactory);
}
@Override
protected Destination buildDestination(DestinationInstructions instructions) {
return toOsmDestination(instructions);
}
@Override
protected Destination buildDestination(DestinationInstructions instructions) {
return toOsmDestination(instructions);
}
}
......@@ -28,27 +28,28 @@ import org.opengroup.osdu.core.gcp.osm.model.Namespace;
@Data
@Builder
public class DestinationInstructions {
private String dataPartition;
private Namespace namespace;
private Kind kind;
public static Destination toOsmDestination(DestinationInstructions instructions){