switch to the Schema Service calls

.gitlab-ci.yml

@@ -19,7 +19,7 @@ variables:
   OSDU_GCP_APPLICATION_NAME: os-indexer
   OSDU_GCP_PROJECT: nice-etching-277309
   OSDU_GCP_TENANT_NAME: osdu
-  OSDU_GCP_STORAGE_SCHEMA_HOST: https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/schemas
+  OSDU_GCP_SCHEMA_HOST: https://os-schema-dot-nice-etching-277309.uc.r.appspot.com/api/schema-service/v1/schema
   OSDU_SECURITY_HTTPS_CERTIFICATE_TRUST: 'true'
 
   IBM_BUILD_SUBDIR: provider/indexer-ibm

devops/azure/chart/templates/deployment.yaml

@@ -96,10 +96,12 @@ spec:
           value: http://entitlements-azure/entitlements/v1
         - name: entitlements_service_api_key
           value: "OBSOLETE"
+        - name: schema_service_url
+          value: http://storage/api/schema-service/v1
+        - name: SCHEMA_HOST
+          value: http://schema/api/schema-service/v1/schema
         - name: storage_service_url
           value: http://storage/api/storage/v2
-        - name: STORAGE_SCHEMA_HOST
-          value: http://storage/api/storage/v2/schemas
         - name: STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST
           value: http://storage/api/storage/v2/query/records:batch
         - name: STORAGE_QUERY_RECORD_HOST

devops/azure/release.yaml

@@ -154,8 +154,8 @@ spec:
           value: "OBSOLETE"
         - name: storage_service_url
           value: http://storage/api/storage/v2
-        - name: STORAGE_SCHEMA_HOST
-          value: http://storage/api/storage/v2/schemas
+        - name: SCHEMA_HOST
+          value: http://schema/api/schema-service/v1/schema
         - name: STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST
           value: http://storage/api/storage/v2/query/records:batch
         - name: STORAGE_QUERY_RECORD_HOST

indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java

@@ -18,6 +18,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import lombok.AccessLevel;
 import lombok.experimental.FieldDefaults;
+import org.opengroup.osdu.indexer.schema.converter.interfaces.SchemaToStorageFormat;
 import org.opengroup.osdu.indexer.schema.converter.tags.*;
 import org.springframework.stereotype.Component;
 
@@ -30,7 +31,7 @@ import java.util.stream.Collectors;
  */
 @Component
 @FieldDefaults(makeFinal=true, level= AccessLevel.PRIVATE)
-public class SchemaToStorageFormatImpl {
+public class SchemaToStorageFormatImpl implements SchemaToStorageFormat {
 
     ObjectMapper objectMapper;
 
@@ -41,6 +42,7 @@ public class SchemaToStorageFormatImpl {
         this.objectMapper = objectMapper;
     }
 
+    @Override
     public String convertToString(final String schemaServiceFormat, String kind) {
         assert schemaServiceFormat!= null;
         assert kind!= null;

indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/interfaces/SchemaToStorageFormat.java

+package org.opengroup.osdu.indexer.schema.converter.interfaces;
+
+public interface SchemaToStorageFormat {
+    String convertToString(String schemaServiceFormat, String kind);
+}

indexer-core/src/main/java/org/opengroup/osdu/indexer/service/impl/SchemaServiceImpl.java

@@ -20,6 +20,7 @@ import org.opengroup.osdu.core.common.http.FetchServiceHttpRequest;
 import org.opengroup.osdu.core.common.http.IUrlFetchService;
 import org.opengroup.osdu.core.common.model.http.HttpResponse;
 import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.indexer.schema.converter.interfaces.SchemaToStorageFormat;
 import org.opengroup.osdu.indexer.service.SchemaService;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
@@ -31,29 +32,32 @@ import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
 
 /**
- * Provides implementation of the Service that retrieves schemas from the Schema Service
+ * Provides implementation of the client service that retrieves schemas from the Schema Service
  */
 @Component
 public class SchemaServiceImpl implements SchemaService {
     @Inject
     private IUrlFetchService urlFetchService;
 
-    @Value("${STORAGE_SCHEMA_HOST}")
-    private String STORAGE_SCHEMA_HOST;
+    @Value("${SCHEMA_HOST}")
+    private String SCHEMA_HOST;
 
     @Inject
     private IRequestInfo requestInfo;
 
+    @Inject
+    private SchemaToStorageFormat schemaToStorageFormat;
+
     @Override
     public String getSchema(String kind) throws URISyntaxException, UnsupportedEncodingException {
-        // this is temporary implementation that still uses storage service
-        String url = String.format("%s/%s", STORAGE_SCHEMA_HOST, URLEncoder.encode(kind, StandardCharsets.UTF_8.toString()));
+        String url = String.format("%s/%s", SCHEMA_HOST, URLEncoder.encode(kind, StandardCharsets.UTF_8.toString()));
         FetchServiceHttpRequest request = FetchServiceHttpRequest.builder()
                 .httpMethod(HttpMethods.GET)
                 .headers(this.requestInfo.getHeadersMap())
                 .url(url)
                 .build();
         HttpResponse response = this.urlFetchService.sendRequest(request);
-        return response.getResponseCode() != HttpStatus.SC_OK ? null : response.getBody();
+        return response.getResponseCode() != HttpStatus.SC_OK ? null :
+                schemaToStorageFormat.convertToString(response.getBody(), kind);
     }
 }

provider/indexer-aws/src/main/resources/application.properties

@@ -22,7 +22,7 @@ aws.es.serviceName=es
 
 GAE_SERVICE=indexer
 
-STORAGE_SCHEMA_HOST=${STORAGE_HOST}/api/storage/v2/schemas
+SCHEMA_HOST=${SCHEMA_HOST}/api/schema-service/v1/schema
 STORAGE_QUERY_RECORD_HOST=${STORAGE_HOST}/api/storage/v2/query/records
 STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=${STORAGE_HOST}/api/storage/v2/query/records:batch
 STORAGE_RECORDS_BATCH_SIZE=20

provider/indexer-azure/.envrc.template

 ##
 # Needed to run the service
 ##
+export schema_service_url=
+export SCHEMA_HOST=
 export storage_service_url=
-export STORAGE_SCHEMA_HOST=
 export STORAGE_QUERY_RECORD_HOST=
 export STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=
 export servicebus_namespace_name=

provider/indexer-azure/README.md

@@ -40,8 +40,9 @@ az keyvault secret show --vault-name $KEY_VAULT_NAME --name $KEY_VAULT_SECRET_NA
 | name | value | description | sensitive? | source |
 | ---  | ---   | ---         | ---        | ---    |
 | `server.servlet.contextPath` | `/api/indexer/v2/` | Servlet context path | no | - |
+| `schema_service_url` | ex `https://schema.azurewebsites.net` | Endpoint of schema service | no | output of infrastructure deployments |
+| `SCHEMA_HOST` | `${schema_service_url}/api/schema-service/v1/schema` | Endpoint of schema API | no | - |
 | `storage_service_url` | ex `https://storage.azurewebsites.net` | Endpoint of storage service | no | output of infrastructure deployments |
-| `STORAGE_SCHEMA_HOST` | `${storage_service_url}/schemas` | Endpoint of schema API | no | - |
 | `STORAGE_QUERY_RECORD_HOST` | `${storage_service_url}/query/records` | Endpoint of records API | no | - |
 | `STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST` | `${storage_service_url}/query/records:batch` | Endpoint of records batch API | no | - |
 | `KEYVAULT_URI` | ex `https://foo-kv.vault.azure.net/` | . | . | . |

provider/indexer-azure/src/main/resources/application.properties

@@ -34,8 +34,10 @@ KINDS_REDIS_DATABASE=1
 CRON_INDEX_CLEANUP_THRESHOLD_DAYS=3
 CRON_EMPTY_INDEX_CLEANUP_THRESHOLD_DAYS=7
 
+schema_service_url=${schema_service_endpoint}
+SCHEMA_HOST=${schema_service_url}/schema
+
 storage_service_url=${storage_service_endpoint}
-STORAGE_SCHEMA_HOST=${storage_service_url}/schemas
 STORAGE_QUERY_RECORD_HOST=${storage_service_url}/query/records
 STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=${storage_service_url}/query/records:batch
 STORAGE_RECORDS_BATCH_SIZE=20

provider/indexer-gcp/README.md

@@ -26,7 +26,7 @@ In order to run the service locally or remotely, you will need to have the follo
 | `INDEXER_QUEUE_HOST` | ex `https://os-indexer-queue-dot-opendes.appspot.com/_dps/task-handlers/enqueue` | Indexer-Queue API endpoint | no | output of infrastructure deployment |
 | `CRS_API` | ex `https://crs-converter-gae-dot-opendes.appspot.com/api/crs/v1` | CRS API endpoint | no | https://console.cloud.google.com/memorystore/redis/instances |
 | `STORAGE_HOSTNAME` | ex `os-storage-dot-opendes.appspot.com` | Storage Host | no | output of infrastructure deployment |
-| `STORAGE_SCHEMA_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/schemas` | Storage API endpoint 'schemas' | no | https://console.cloud.google.com/apis/credentials |
+| `SCHEMA_HOST` | ex `https://os-schema-dot-opendes.appspot.com/api/schema-service/v1/schema` | Storage API endpoint 'schemas' | no | https://console.cloud.google.com/apis/credentials |
 | `STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records:batch` | Storage API endpoint 'records'  | no | https://console.cloud.google.com/iam-admin/serviceaccounts |
 | `REDIS_SEARCH_HOST` | ex `127.0.0.1` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances |
 | `REDIS_GROUP_HOST` | ex `127.0.0.1` | Redis host for groups | no | https://console.cloud.google.com/memorystore/redis/instances |

provider/indexer-gcp/src/main/resources/application-dev.properties

@@ -3,7 +3,8 @@ GOOGLE_CLOUD_PROJECT=opendes
 INDEXER_HOST=os-indexer-dot-opendes.appspot.com
 STORAGE_HOSTNAME=os-storage-dot-opendes.appspot.com
 
-STORAGE_SCHEMA_HOST=https://os-storage-dot-opendes.appspot.com/api/storage/v2/schemas
+SCHEMA_HOST=https://os-schema-dot-opendes.appspot.com/api/schema-service/v1/schema
+
 STORAGE_QUERY_RECORD_HOST=https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records
 STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records:batch
 STORAGE_RECORDS_BATCH_SIZE=20

provider/indexer-gcp/src/main/resources/application-kuber.properties

@@ -3,7 +3,8 @@ GOOGLE_CLOUD_PROJECT=${GOOGLE_CLOUD_PROJECT}
 INDEXER_HOST=os-indexer-service
 STORAGE_HOSTNAME=os-storage-service
 
-STORAGE_SCHEMA_HOST=http://os-storage-service/api/storage/v2/schemas
+SCHEMA_HOST=http://os-schema-service/api/schema-service/v1/schema
+
 STORAGE_QUERY_RECORD_HOST=http://os-storage-service/api/storage/v2/query/records
 STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=http://os-storage-service/api/storage/v2/query/records:batch
 STORAGE_RECORDS_BATCH_SIZE=20

provider/indexer-gcp/src/main/resources/application-testing.properties

@@ -3,7 +3,8 @@ GOOGLE_CLOUD_PROJECT=opendes-evt
 INDEXER_HOST=os-indexer-dot-opendes-evt.appspot.com
 STORAGE_HOSTNAME=os-storage-dot-opendes-evt.appspot.com
 
-STORAGE_SCHEMA_HOST=https://os-storage-dot-opendes-evt.appspot.com/api/storage/v2/schemas
+SCHEMA_HOST=https://os-schema-dot-opendes-evt.appspot.com/api/schema-service/v1/schema
+
 STORAGE_QUERY_RECORD_HOST=https://os-storage-dot-opendes-evt.appspot.com/api/storage/v2/query/records
 STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=https://os-storage-dot-opendes-evt.appspot.com/api/storage/v2/query/records:batch
 STORAGE_RECORDS_BATCH_SIZE=20

provider/indexer-ibm/src/main/resources/application.properties

@@ -26,9 +26,11 @@ KINDS_REDIS_DATABASE=1
 CRON_INDEX_CLEANUP_THRESHOLD_DAYS=3
 CRON_EMPTY_INDEX_CLEANUP_THRESHOLD_DAYS=7
 
+schema_service_url=http://localhost:8083
+SCHEMA_HOST=${schema_service_url}/api/schema-service/v1/schema
+
 storage_service_url=http://localhost:8082
 #storage_service_url=https://os-storage-ibm-osdu-r2.osduadev-a1c3eaf78a86806e299f5f3f207556f0-0000.us-south.containers.appdomain.cloud
-STORAGE_SCHEMA_HOST=${storage_service_url}/api/storage/v2/schemas
 STORAGE_QUERY_RECORD_HOST=${storage_service_url}/api/storage/v2/query/records
 STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=${storage_service_url}/api/storage/v2/query/records:batch
 STORAGE_RECORDS_BATCH_SIZE=20