diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f03a7d4989795cac0acbcbbb43dd28911d1e05bd..d9257ff4ac063855237c645e4da619dbe1ecc58b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,7 +19,7 @@ variables: OSDU_GCP_APPLICATION_NAME: os-indexer OSDU_GCP_PROJECT: nice-etching-277309 OSDU_GCP_TENANT_NAME: osdu - OSDU_GCP_STORAGE_SCHEMA_HOST: https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/schemas + OSDU_GCP_SCHEMA_HOST: https://os-schema-dot-nice-etching-277309.uc.r.appspot.com/api/schema-service/v1/schema OSDU_SECURITY_HTTPS_CERTIFICATE_TRUST: 'true' IBM_BUILD_SUBDIR: provider/indexer-ibm diff --git a/devops/azure/chart/templates/deployment.yaml b/devops/azure/chart/templates/deployment.yaml index 970010316c2313463716c5cbae79631f5da3ecce..77103ebed9532bd8ed32f7cdf373925b776452da 100644 --- a/devops/azure/chart/templates/deployment.yaml +++ b/devops/azure/chart/templates/deployment.yaml @@ -96,10 +96,12 @@ spec: value: http://entitlements-azure/entitlements/v1 - name: entitlements_service_api_key value: "OBSOLETE" + - name: schema_service_url + value: http://storage/api/schema-service/v1 + - name: SCHEMA_HOST + value: http://schema/api/schema-service/v1/schema - name: storage_service_url value: http://storage/api/storage/v2 - - name: STORAGE_SCHEMA_HOST - value: http://storage/api/storage/v2/schemas - name: STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST value: http://storage/api/storage/v2/query/records:batch - name: STORAGE_QUERY_RECORD_HOST diff --git a/devops/azure/release.yaml b/devops/azure/release.yaml index 540d39689a9503daf6db8fa8529570eeca3deb5e..7c5645257e8861c17762ea116337806442a803d5 100644 --- a/devops/azure/release.yaml +++ b/devops/azure/release.yaml @@ -154,8 +154,8 @@ spec: value: "OBSOLETE" - name: storage_service_url value: http://storage/api/storage/v2 - - name: STORAGE_SCHEMA_HOST - value: http://storage/api/storage/v2/schemas + - name: SCHEMA_HOST + value: http://schema/api/schema-service/v1/schema - name: STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST value: http://storage/api/storage/v2/query/records:batch - name: STORAGE_QUERY_RECORD_HOST diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java index 60466c3f1eecce1d1cb863820a4f12922182ec12..c8b38cd4e9e0c9515a9f1f0c61b3efd5732d44f4 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java @@ -18,6 +18,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.AccessLevel; import lombok.experimental.FieldDefaults; +import org.opengroup.osdu.indexer.schema.converter.interfaces.SchemaToStorageFormat; import org.opengroup.osdu.indexer.schema.converter.tags.*; import org.springframework.stereotype.Component; @@ -30,7 +31,7 @@ import java.util.stream.Collectors; */ @Component @FieldDefaults(makeFinal=true, level= AccessLevel.PRIVATE) -public class SchemaToStorageFormatImpl { +public class SchemaToStorageFormatImpl implements SchemaToStorageFormat { ObjectMapper objectMapper; @@ -41,6 +42,7 @@ public class SchemaToStorageFormatImpl { this.objectMapper = objectMapper; } + @Override public String convertToString(final String schemaServiceFormat, String kind) { assert schemaServiceFormat!= null; assert kind!= null; diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/interfaces/SchemaToStorageFormat.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/interfaces/SchemaToStorageFormat.java new file mode 100644 index 0000000000000000000000000000000000000000..41fe22b7ca06af59c52f51f0044160d4a310a10f --- /dev/null +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/interfaces/SchemaToStorageFormat.java @@ -0,0 +1,5 @@ +package org.opengroup.osdu.indexer.schema.converter.interfaces; + +public interface SchemaToStorageFormat { + String convertToString(String schemaServiceFormat, String kind); +} diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/impl/SchemaServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/impl/SchemaServiceImpl.java index b74de7cb3636cb2cd6b8580433bdcb76214a10af..ffdbd9593749ef66ca22edf7c1f5a486ca10ed8a 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/impl/SchemaServiceImpl.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/impl/SchemaServiceImpl.java @@ -20,6 +20,7 @@ import org.opengroup.osdu.core.common.http.FetchServiceHttpRequest; import org.opengroup.osdu.core.common.http.IUrlFetchService; import org.opengroup.osdu.core.common.model.http.HttpResponse; import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo; +import org.opengroup.osdu.indexer.schema.converter.interfaces.SchemaToStorageFormat; import org.opengroup.osdu.indexer.service.SchemaService; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; @@ -31,29 +32,32 @@ import java.net.URLEncoder; import java.nio.charset.StandardCharsets; /** - * Provides implementation of the Service that retrieves schemas from the Schema Service + * Provides implementation of the client service that retrieves schemas from the Schema Service */ @Component public class SchemaServiceImpl implements SchemaService { @Inject private IUrlFetchService urlFetchService; - @Value("${STORAGE_SCHEMA_HOST}") - private String STORAGE_SCHEMA_HOST; + @Value("${SCHEMA_HOST}") + private String SCHEMA_HOST; @Inject private IRequestInfo requestInfo; + @Inject + private SchemaToStorageFormat schemaToStorageFormat; + @Override public String getSchema(String kind) throws URISyntaxException, UnsupportedEncodingException { - // this is temporary implementation that still uses storage service - String url = String.format("%s/%s", STORAGE_SCHEMA_HOST, URLEncoder.encode(kind, StandardCharsets.UTF_8.toString())); + String url = String.format("%s/%s", SCHEMA_HOST, URLEncoder.encode(kind, StandardCharsets.UTF_8.toString())); FetchServiceHttpRequest request = FetchServiceHttpRequest.builder() .httpMethod(HttpMethods.GET) .headers(this.requestInfo.getHeadersMap()) .url(url) .build(); HttpResponse response = this.urlFetchService.sendRequest(request); - return response.getResponseCode() != HttpStatus.SC_OK ? null : response.getBody(); + return response.getResponseCode() != HttpStatus.SC_OK ? null : + schemaToStorageFormat.convertToString(response.getBody(), kind); } } diff --git a/provider/indexer-aws/src/main/resources/application.properties b/provider/indexer-aws/src/main/resources/application.properties index 6e7d9c6f32304d2fb255c95e63239ec8f2964a55..31706b87e72a40110e99a5aa237536675b54e1ad 100644 --- a/provider/indexer-aws/src/main/resources/application.properties +++ b/provider/indexer-aws/src/main/resources/application.properties @@ -22,7 +22,7 @@ aws.es.serviceName=es GAE_SERVICE=indexer -STORAGE_SCHEMA_HOST=${STORAGE_HOST}/api/storage/v2/schemas +SCHEMA_HOST=${SCHEMA_HOST}/api/schema-service/v1/schema STORAGE_QUERY_RECORD_HOST=${STORAGE_HOST}/api/storage/v2/query/records STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=${STORAGE_HOST}/api/storage/v2/query/records:batch STORAGE_RECORDS_BATCH_SIZE=20 diff --git a/provider/indexer-azure/.envrc.template b/provider/indexer-azure/.envrc.template index e07a7890090f4b5af016ff3c7161c9b5aefb2065..153291ee94ad2bd56f7863314d628f96e84d0384 100644 --- a/provider/indexer-azure/.envrc.template +++ b/provider/indexer-azure/.envrc.template @@ -1,8 +1,9 @@ ## # Needed to run the service ## +export schema_service_url= +export SCHEMA_HOST= export storage_service_url= -export STORAGE_SCHEMA_HOST= export STORAGE_QUERY_RECORD_HOST= export STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST= export servicebus_namespace_name= diff --git a/provider/indexer-azure/README.md b/provider/indexer-azure/README.md index 6772abca8ff69d7dc5f90e8456805f5ee709a734..3151ea79d665a6c90d414be1e67f64294ca61f1a 100644 --- a/provider/indexer-azure/README.md +++ b/provider/indexer-azure/README.md @@ -40,8 +40,9 @@ az keyvault secret show --vault-name $KEY_VAULT_NAME --name $KEY_VAULT_SECRET_NA | name | value | description | sensitive? | source | | --- | --- | --- | --- | --- | | `server.servlet.contextPath` | `/api/indexer/v2/` | Servlet context path | no | - | +| `schema_service_url` | ex `https://schema.azurewebsites.net` | Endpoint of schema service | no | output of infrastructure deployments | +| `SCHEMA_HOST` | `${schema_service_url}/api/schema-service/v1/schema` | Endpoint of schema API | no | - | | `storage_service_url` | ex `https://storage.azurewebsites.net` | Endpoint of storage service | no | output of infrastructure deployments | -| `STORAGE_SCHEMA_HOST` | `${storage_service_url}/schemas` | Endpoint of schema API | no | - | | `STORAGE_QUERY_RECORD_HOST` | `${storage_service_url}/query/records` | Endpoint of records API | no | - | | `STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST` | `${storage_service_url}/query/records:batch` | Endpoint of records batch API | no | - | | `KEYVAULT_URI` | ex `https://foo-kv.vault.azure.net/` | . | . | . | diff --git a/provider/indexer-azure/src/main/resources/application.properties b/provider/indexer-azure/src/main/resources/application.properties index f6cb597dbcc58293ffd9897faadace106a77c4f1..ba8b8c91cb5e09899b1f4d40a4e424cc65b6f6f4 100644 --- a/provider/indexer-azure/src/main/resources/application.properties +++ b/provider/indexer-azure/src/main/resources/application.properties @@ -34,8 +34,10 @@ KINDS_REDIS_DATABASE=1 CRON_INDEX_CLEANUP_THRESHOLD_DAYS=3 CRON_EMPTY_INDEX_CLEANUP_THRESHOLD_DAYS=7 +schema_service_url=${schema_service_endpoint} +SCHEMA_HOST=${schema_service_url}/schema + storage_service_url=${storage_service_endpoint} -STORAGE_SCHEMA_HOST=${storage_service_url}/schemas STORAGE_QUERY_RECORD_HOST=${storage_service_url}/query/records STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=${storage_service_url}/query/records:batch STORAGE_RECORDS_BATCH_SIZE=20 diff --git a/provider/indexer-gcp/README.md b/provider/indexer-gcp/README.md index 23c482574d2515459a8f962b197687472e1bba52..249ebe314c89f388ac898aeff1ae6d9887986ccd 100644 --- a/provider/indexer-gcp/README.md +++ b/provider/indexer-gcp/README.md @@ -26,7 +26,7 @@ In order to run the service locally or remotely, you will need to have the follo | `INDEXER_QUEUE_HOST` | ex `https://os-indexer-queue-dot-opendes.appspot.com/_dps/task-handlers/enqueue` | Indexer-Queue API endpoint | no | output of infrastructure deployment | | `CRS_API` | ex `https://crs-converter-gae-dot-opendes.appspot.com/api/crs/v1` | CRS API endpoint | no | https://console.cloud.google.com/memorystore/redis/instances | | `STORAGE_HOSTNAME` | ex `os-storage-dot-opendes.appspot.com` | Storage Host | no | output of infrastructure deployment | -| `STORAGE_SCHEMA_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/schemas` | Storage API endpoint 'schemas' | no | https://console.cloud.google.com/apis/credentials | +| `SCHEMA_HOST` | ex `https://os-schema-dot-opendes.appspot.com/api/schema-service/v1/schema` | Storage API endpoint 'schemas' | no | https://console.cloud.google.com/apis/credentials | | `STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records:batch` | Storage API endpoint 'records' | no | https://console.cloud.google.com/iam-admin/serviceaccounts | | `REDIS_SEARCH_HOST` | ex `127.0.0.1` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances | | `REDIS_GROUP_HOST` | ex `127.0.0.1` | Redis host for groups | no | https://console.cloud.google.com/memorystore/redis/instances | diff --git a/provider/indexer-gcp/src/main/resources/application-dev.properties b/provider/indexer-gcp/src/main/resources/application-dev.properties index 61da3c7139e0f8e834b2ec5b90173b2c517d075d..bc78449e46c2bd7fbb72c8289f8732eda5fb5fdf 100644 --- a/provider/indexer-gcp/src/main/resources/application-dev.properties +++ b/provider/indexer-gcp/src/main/resources/application-dev.properties @@ -3,7 +3,8 @@ GOOGLE_CLOUD_PROJECT=opendes INDEXER_HOST=os-indexer-dot-opendes.appspot.com STORAGE_HOSTNAME=os-storage-dot-opendes.appspot.com -STORAGE_SCHEMA_HOST=https://os-storage-dot-opendes.appspot.com/api/storage/v2/schemas +SCHEMA_HOST=https://os-schema-dot-opendes.appspot.com/api/schema-service/v1/schema + STORAGE_QUERY_RECORD_HOST=https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records:batch STORAGE_RECORDS_BATCH_SIZE=20 diff --git a/provider/indexer-gcp/src/main/resources/application-kuber.properties b/provider/indexer-gcp/src/main/resources/application-kuber.properties index bc2a5f9014fa1b406a008e2cc69f161a8e3a9a8b..feec48270e63c78e1f1c79ab090aaac4ac31b1b4 100644 --- a/provider/indexer-gcp/src/main/resources/application-kuber.properties +++ b/provider/indexer-gcp/src/main/resources/application-kuber.properties @@ -3,7 +3,8 @@ GOOGLE_CLOUD_PROJECT=${GOOGLE_CLOUD_PROJECT} INDEXER_HOST=os-indexer-service STORAGE_HOSTNAME=os-storage-service -STORAGE_SCHEMA_HOST=http://os-storage-service/api/storage/v2/schemas +SCHEMA_HOST=http://os-schema-service/api/schema-service/v1/schema + STORAGE_QUERY_RECORD_HOST=http://os-storage-service/api/storage/v2/query/records STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=http://os-storage-service/api/storage/v2/query/records:batch STORAGE_RECORDS_BATCH_SIZE=20 diff --git a/provider/indexer-gcp/src/main/resources/application-testing.properties b/provider/indexer-gcp/src/main/resources/application-testing.properties index fb0c9e2d6ce29c87dca6f58d1afec0b420c1e663..4394b428e1ccbe644ae5e16423b76c507370e1d6 100644 --- a/provider/indexer-gcp/src/main/resources/application-testing.properties +++ b/provider/indexer-gcp/src/main/resources/application-testing.properties @@ -3,7 +3,8 @@ GOOGLE_CLOUD_PROJECT=opendes-evt INDEXER_HOST=os-indexer-dot-opendes-evt.appspot.com STORAGE_HOSTNAME=os-storage-dot-opendes-evt.appspot.com -STORAGE_SCHEMA_HOST=https://os-storage-dot-opendes-evt.appspot.com/api/storage/v2/schemas +SCHEMA_HOST=https://os-schema-dot-opendes-evt.appspot.com/api/schema-service/v1/schema + STORAGE_QUERY_RECORD_HOST=https://os-storage-dot-opendes-evt.appspot.com/api/storage/v2/query/records STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=https://os-storage-dot-opendes-evt.appspot.com/api/storage/v2/query/records:batch STORAGE_RECORDS_BATCH_SIZE=20 diff --git a/provider/indexer-ibm/src/main/resources/application.properties b/provider/indexer-ibm/src/main/resources/application.properties index 9a7a16cc4942744aea51f9e54bc4bae855892b8d..d84c894a183ab22088b4d299e6355b1759a06a2b 100644 --- a/provider/indexer-ibm/src/main/resources/application.properties +++ b/provider/indexer-ibm/src/main/resources/application.properties @@ -26,9 +26,11 @@ KINDS_REDIS_DATABASE=1 CRON_INDEX_CLEANUP_THRESHOLD_DAYS=3 CRON_EMPTY_INDEX_CLEANUP_THRESHOLD_DAYS=7 +schema_service_url=http://localhost:8083 +SCHEMA_HOST=${schema_service_url}/api/schema-service/v1/schema + storage_service_url=http://localhost:8082 #storage_service_url=https://os-storage-ibm-osdu-r2.osduadev-a1c3eaf78a86806e299f5f3f207556f0-0000.us-south.containers.appdomain.cloud -STORAGE_SCHEMA_HOST=${storage_service_url}/api/storage/v2/schemas STORAGE_QUERY_RECORD_HOST=${storage_service_url}/api/storage/v2/query/records STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=${storage_service_url}/api/storage/v2/query/records:batch STORAGE_RECORDS_BATCH_SIZE=20