diff --git a/devops/aws/chart/Chart.yaml b/devops/aws/chart/Chart.yaml
index 34992d442f3daa70968328781f6598534c06817f..afc05ca9d2859421b0e55fd1a389e605cd355310 100644
--- a/devops/aws/chart/Chart.yaml
+++ b/devops/aws/chart/Chart.yaml
@@ -1,12 +1,11 @@
apiVersion: v2
name: "os-storage"
version: __CHART_VERSION__
-kubeVersion: ">= 1.21.x-x-x < 1.24.x-x-x"
description: Storage Helm Chart for Kubernetes
type: application
appVersion: __VERSION__
dependencies:
- name: osdu-aws-lib
- version: 0.2.0
- repository: __HELM_REPO__/osdu-aws-lib/
+ version: ~0
+ repository: __HELM_REPO__
deprecated: false
diff --git a/devops/aws/chart/values.yaml b/devops/aws/chart/values.yaml
index 0a3fec083fedf022e9ed4ad836a00aaed0121aa8..85933a2e3ae434ab556cb9c1a64164c6cfe3af9f 100644
--- a/devops/aws/chart/values.yaml
+++ b/devops/aws/chart/values.yaml
@@ -58,6 +58,8 @@ environmentVariables:
value: "true"
- name: PARAMETER_MOUNT_PATH
value: "/mnt/params"
+ - name: STORAGE_SERVICE_REPOSITORY_IMPLEMENTATION
+ value: "{{ default `dynamodb` .Values.global.coreDbProvider }}"
# Resource Config
maxConnections: 200
diff --git a/devops/gc/deploy/README.md b/devops/gc/deploy/README.md
index dba30bc19344fdf06a6980e79f2f1e831683dc9f..98ce0c39f088be5abac2163b7a75e1cd669a36dd 100644
--- a/devops/gc/deploy/README.md
+++ b/devops/gc/deploy/README.md
@@ -26,6 +26,13 @@ Packages are only needed for installation from a local computer.
First you need to set variables in **values.yaml** file using any code editor. Some of the values are prefilled, but you need to specify some values as well. You can find more information about them below.
+### Global variables
+
+| Name | Description | Type | Default |Required |
+|------|-------------|------|---------|---------|
+**global.domain** | your domain for the external endpoint, ex `example.com` | string | - | yes
+**global.onPremEnabled** | whether on-prem is enabled | boolean | false | yes
+
### Configmap variables
| Name | Description | Type | Default |Required |
@@ -73,9 +80,7 @@ First you need to set variables in **values.yaml** file using any code editor. S
**conf.storageRedisSecretName** | secret for redis that contains redis password with REDIS_PASSWORD key | string | `storage-redis-secret` | yes
**conf.bootstrapSecretName** | secret for bootstrap to access openid provider | string | `datafier-secret` | only if `conf.bootstrapEnabled` is true
**conf.replicas** | Number of replicas | integer | 3 | yes
-**conf.onPremEnabled** | whether on-prem is enabled | boolean | false | yes
**conf.bootstrapEnabled** | whether storage bootstrap is enabled | boolean | false | yes
-**conf.domain** | your domain, ex `example.com` | string | - | yes
### Istio variables
diff --git a/devops/gc/deploy/templates/bootstrap-configmap.yaml b/devops/gc/deploy/templates/bootstrap-configmap.yaml
index f4cf7f0a305c0634ec02e0b3ca7bcedf9c949933..337f5c3454891175603872d1d8e19f8a12d742e7 100644
--- a/devops/gc/deploy/templates/bootstrap-configmap.yaml
+++ b/devops/gc/deploy/templates/bootstrap-configmap.yaml
@@ -10,5 +10,5 @@ data:
STORAGE_HOST: {{ .Values.data.storageHost | quote }}
DEFAULT_LEGAL_TAG: {{ .Values.data.defaultLegalTag | quote }}
DATA_PARTITION_ID: {{ .Values.data.dataPartitionId | quote }}
- ONPREM_ENABLED: {{ .Values.conf.onPremEnabled | quote }}
+ ONPREM_ENABLED: {{ .Values.global.onPremEnabled | quote }}
{{- end }}
diff --git a/devops/gc/deploy/templates/bootstrap-deployment.yaml b/devops/gc/deploy/templates/bootstrap-deployment.yaml
index 31999cee993ba1cf911e8c3d54edcfc940e5d461..509f93d3dfa0018d82ea12e57cce5a64c13d9d5f 100644
--- a/devops/gc/deploy/templates/bootstrap-deployment.yaml
+++ b/devops/gc/deploy/templates/bootstrap-deployment.yaml
@@ -34,7 +34,7 @@ spec:
envFrom:
- configMapRef:
name: {{ printf "%s-config-bootstrap" .Values.conf.appName | quote }}
- {{- if .Values.conf.onPremEnabled }}
+ {{- if .Values.global.onPremEnabled }}
- secretRef:
name: {{ .Values.conf.bootstrapSecretName | quote }}
{{- end }}
diff --git a/devops/gc/deploy/templates/deployment.yaml b/devops/gc/deploy/templates/deployment.yaml
index 48e96b1cead930313402c7351d7c0723b73f6c89..2f746f6ca11638130679d64378a1d0f24876bafe 100644
--- a/devops/gc/deploy/templates/deployment.yaml
+++ b/devops/gc/deploy/templates/deployment.yaml
@@ -39,7 +39,7 @@ spec:
envFrom:
- configMapRef:
name: {{ printf "%s-config" .Values.conf.appName | quote }}
- {{- if .Values.conf.onPremEnabled }}
+ {{- if .Values.global.onPremEnabled }}
- secretRef:
name: {{ .Values.conf.keycloakSecretName | quote }}
- secretRef:
diff --git a/devops/gc/deploy/templates/service-account.yaml b/devops/gc/deploy/templates/service-account.yaml
index f0771985cd5da9db77781dc3ab635e52378b3fb6..eec72f12889eec74d3de43ae6527a860bcd876ea 100644
--- a/devops/gc/deploy/templates/service-account.yaml
+++ b/devops/gc/deploy/templates/service-account.yaml
@@ -1,4 +1,4 @@
-{{- if .Values.conf.onPremEnabled }}
+{{- if .Values.global.onPremEnabled }}
apiVersion: v1
kind: ServiceAccount
metadata:
diff --git a/devops/gc/deploy/templates/service.yaml b/devops/gc/deploy/templates/service.yaml
index 45bd505383e8204e40ebf5188814bc4a707b6c34..f985dd5f7a65eff8c6c0475c13d0298b9770ec00 100644
--- a/devops/gc/deploy/templates/service.yaml
+++ b/devops/gc/deploy/templates/service.yaml
@@ -1,7 +1,7 @@
apiVersion: v1
kind: Service
metadata:
- {{- if not .Values.conf.onPremEnabled }}
+ {{- if not .Values.global.onPremEnabled }}
annotations:
cloud.google.com/neg: '{"ingress": true}'
{{- end }}
diff --git a/devops/gc/deploy/templates/virtual-service.yaml b/devops/gc/deploy/templates/virtual-service.yaml
index 4bf9d0e65def75bd50ce0285bfeb35621363f06b..0e589c820a8a9392f5e3232f3015093a37c922c8 100644
--- a/devops/gc/deploy/templates/virtual-service.yaml
+++ b/devops/gc/deploy/templates/virtual-service.yaml
@@ -5,10 +5,10 @@ metadata:
namespace: {{ .Release.Namespace | quote }}
spec:
hosts:
- {{- if and .Values.conf.domain .Values.conf.onPremEnabled }}
- - {{ printf "osdu.%s" .Values.conf.domain | quote }}
- {{- else if .Values.conf.domain }}
- - {{ .Values.conf.domain | quote }}
+ {{- if and .Values.global.domain .Values.global.onPremEnabled }}
+ - {{ printf "osdu.%s" .Values.global.domain | quote }}
+ {{- else if .Values.global.domain }}
+ - {{ .Values.global.domain | quote }}
{{- else }}
- "*"
{{- end }}
diff --git a/devops/gc/deploy/values.yaml b/devops/gc/deploy/values.yaml
index 6b105c7d28c239944d011f95e9e02783da204954..2d78e36ed9466c01543d4f60cf305d2d07f18ed6 100644
--- a/devops/gc/deploy/values.yaml
+++ b/devops/gc/deploy/values.yaml
@@ -2,6 +2,10 @@
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.
+global:
+ domain: ""
+ onPremEnabled: false
+
data:
# configmaps
logLevel: "ERROR"
@@ -40,9 +44,7 @@ conf:
rabbitmqSecretName: "rabbitmq-secret"
bootstrapSecretName: "datafier-secret"
replicas: 3
- onPremEnabled: false
bootstrapEnabled: false
- domain: ""
storageRedisSecretName: "storage-redis-secret"
istio:
diff --git a/provider/storage-aws/pom.xml b/provider/storage-aws/pom.xml
index a630d95bbb15c26424a4b29869ee453586b7d9cf..4cc952e9fb12028f1d059d9d5499199bc42191e3 100644
--- a/provider/storage-aws/pom.xml
+++ b/provider/storage-aws/pom.xml
@@ -66,7 +66,7 @@
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
<artifactId>os-core-lib-aws</artifactId>
- <version>0.17.0</version>
+ <version>0.20.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
diff --git a/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/QueryRepositoryImpl.java b/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/QueryRepositoryImpl.java
index 4e472b2257137f3a3d5f8e4a53c8af848be73cc1..32b2c55b2de7579741c480570d1bda59b30585a7 100644
--- a/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/QueryRepositoryImpl.java
+++ b/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/QueryRepositoryImpl.java
@@ -15,6 +15,7 @@
package org.opengroup.osdu.storage.provider.aws;
import com.amazonaws.services.dynamodbv2.model.AttributeValue;
+import com.amazonaws.services.dynamodbv2.model.ComparisonOperator;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelperFactory;
import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelperV2;
@@ -121,7 +122,16 @@ public class QueryRepositoryImpl implements IQueryRepository {
QueryPageResult<RecordMetadataDoc> scanPageResults;
try {
- scanPageResults = recordMetadataQueryHelper.queryPage(RecordMetadataDoc.class, recordMetadataKey, "Status", "active", numRecords, cursor);
+ scanPageResults = recordMetadataQueryHelper.queryPage(
+ RecordMetadataDoc.class,
+ recordMetadataKey,
+ "Status",
+ "active",
+ "Id",
+ ComparisonOperator.BEGINS_WITH,
+ String.format("%s:", this.headers.getPartitionId()),
+ numRecords,
+ cursor);
} catch (UnsupportedEncodingException e) {
throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error parsing results",
e.getMessage(), e);
diff --git a/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/mongo/mongodb/MultiClusteredConfigReaderStorage.java b/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/mongo/mongodb/MultiClusteredConfigReaderStorage.java
index b8c1ab0738a9011623e9d2c5f218fae69dd03d02..23e52623404d73159378df4cc4f9cc8530c3318d 100644
--- a/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/mongo/mongodb/MultiClusteredConfigReaderStorage.java
+++ b/provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/mongo/mongodb/MultiClusteredConfigReaderStorage.java
@@ -9,7 +9,6 @@ import org.springframework.stereotype.Component;
@Lazy
@Component
public class MultiClusteredConfigReaderStorage extends AbstractMultiClusteredConfigReader {
- String serviceName = "storage";
@Autowired
public MultiClusteredConfigReaderStorage(SSMManagerUtil ssmManagerUtil) {
@@ -17,7 +16,7 @@ public class MultiClusteredConfigReaderStorage extends AbstractMultiClusteredCon
}
@Override
- protected String applyServiceName(String originalName) {
- return originalName.replace(serviceNamePlaceHolder, serviceName);
+ protected String getDatabaseName(String environment) {
+ return environment + "_osdu_storage";
}
}
diff --git a/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/QueryRepositoryTest.java b/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/QueryRepositoryTest.java
index 7ff11d913a3cd515a609a3c97e5e59170f03f1e9..ab2adb6c7caf3e8f44217d235784dd7ecc2a01ad 100644
--- a/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/QueryRepositoryTest.java
+++ b/provider/storage-aws/src/test/java/org/opengroup/osdu/storage/provider/aws/api/QueryRepositoryTest.java
@@ -14,6 +14,7 @@
package org.opengroup.osdu.storage.provider.aws.api;
+import com.amazonaws.services.dynamodbv2.model.ComparisonOperator;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -134,15 +135,85 @@ public class QueryRepositoryTest {
RecordMetadataDoc recordMetadataKey = new RecordMetadataDoc();
recordMetadataKey.setKind(kind);
- Mockito.when(queryHelper.queryPage(Mockito.eq(RecordMetadataDoc.class), Mockito.anyObject(), Mockito.anyString(), Mockito.anyString(), Mockito.anyInt(), Mockito.eq(cursor)))
- .thenReturn(expectedQueryPageResult);
+ Mockito.when(queryHelper.queryPage(
+ Mockito.eq(RecordMetadataDoc.class),
+ Mockito.anyObject(),
+ Mockito.anyString(),
+ Mockito.anyString(),
+ Mockito.anyString(),
+ Mockito.eq(ComparisonOperator.BEGINS_WITH),
+ Mockito.anyString(),
+ Mockito.anyInt(),
+ Mockito.anyString()))
+ .thenReturn(expectedQueryPageResult);
+
+ // Act
+ DatastoreQueryResult datastoreQueryResult = repo.getAllRecordIdsFromKind(kind, 50, cursor, Optional.empty());
+
+ // Assert
+ Mockito.verify(queryHelper, Mockito.times(1)).queryPage(
+ Mockito.eq(RecordMetadataDoc.class),
+ Mockito.anyObject(),
+ Mockito.anyString(),
+ Mockito.anyString(),
+ Mockito.anyString(),
+ Mockito.eq(ComparisonOperator.BEGINS_WITH),
+ Mockito.anyString(),
+ Mockito.eq(50),
+ Mockito.eq(cursor)
+ );
+ Assert.assertEquals(expectedDatastoreQueryResult, datastoreQueryResult);
+ }
+
+ @Test
+ public void getAllRecordIdsFromDifferentKind() throws UnsupportedEncodingException {
+ // Arrange
+ String kind = "osdu:source:type:1.0.0";
+ String cursor = "abc123";
+ String recordId = "tenant:source:type:1.0.0.1212";
+ List<String> resultsIds = new ArrayList<>();
+ resultsIds.add(recordId);
+ DatastoreQueryResult expectedDatastoreQueryResult = new DatastoreQueryResult(cursor, resultsIds);
+ String user = "test-user";
+ RecordMetadataDoc expectedRecordMetadataDoc = new RecordMetadataDoc();
+ expectedRecordMetadataDoc.setId(recordId);
+ expectedRecordMetadataDoc.setKind(kind);
+ expectedRecordMetadataDoc.setUser(user);
+ expectedRecordMetadataDoc.setStatus("active");
+ List<RecordMetadataDoc> expectedRecordMetadataDocList = new ArrayList<>();
+ expectedRecordMetadataDocList.add(expectedRecordMetadataDoc);
+ QueryPageResult<RecordMetadataDoc> expectedQueryPageResult = new QueryPageResult<>(cursor, expectedRecordMetadataDocList);
+ // Set GSI hash key
+ RecordMetadataDoc recordMetadataKey = new RecordMetadataDoc();
+ recordMetadataKey.setKind(kind);
+
+ Mockito.when(queryHelper.queryPage(
+ Mockito.eq(RecordMetadataDoc.class),
+ Mockito.anyObject(),
+ Mockito.anyString(),
+ Mockito.anyString(),
+ Mockito.anyString(),
+ Mockito.eq(ComparisonOperator.BEGINS_WITH),
+ Mockito.anyString(),
+ Mockito.anyInt(),
+ Mockito.anyString()))
+ .thenReturn(expectedQueryPageResult);
// Act
DatastoreQueryResult datastoreQueryResult = repo.getAllRecordIdsFromKind(kind, 50, cursor, Optional.empty());
// Assert
- Mockito.verify(queryHelper, Mockito.times(1)).queryPage(Mockito.eq(RecordMetadataDoc.class), Mockito.anyObject(), Mockito.anyString(),
- Mockito.anyString(), Mockito.anyInt(), Mockito.eq(cursor));
+ Mockito.verify(queryHelper, Mockito.times(1)).queryPage(
+ Mockito.eq(RecordMetadataDoc.class),
+ Mockito.anyObject(),
+ Mockito.anyString(),
+ Mockito.anyString(),
+ Mockito.anyString(),
+ Mockito.eq(ComparisonOperator.BEGINS_WITH),
+ Mockito.anyString(),
+ Mockito.eq(50),
+ Mockito.eq(cursor)
+ );
Assert.assertEquals(expectedDatastoreQueryResult, datastoreQueryResult);
}
}
diff --git a/provider/storage-gc/docs/anthos/README.md b/provider/storage-gc/docs/anthos/README.md
index 63a3baa7337d00b1be8242f95aedfb78793f293c..b0b1b4ed7464e85654f539aed0d52b2ebe1196fd 100644
--- a/provider/storage-gc/docs/anthos/README.md
+++ b/provider/storage-gc/docs/anthos/README.md
@@ -14,6 +14,8 @@
* [Exchanges and queues configuration](#Exchanges-and-queues-configuration)
* [Interaction with message brokers](#Interaction-with-message-brokers)
* [Keycloak configuration](#Keycloak-configuration)
+* [Running E2E Tests](#Running-E2E-Tests)
+* [Running locally](#Running-locally)
## Environment variables
@@ -421,7 +423,7 @@ Add `partition-and-entitlements` scope to `Default Client Scopes` and generate K
Give `client-id` and `client-secret` to services, which should be authorized within the platform.
-### Running E2E Tests
+## Running E2E Tests
You will need to have the following environment variables defined.
@@ -458,3 +460,67 @@ Execute following command to build code and run all the integration tests:
# build + run Google Cloud integration tests.
$ (cd testing/storage-test-anthos/ && mvn clean test)
```
+
+
+## Running locally
+To run storage service locally connected with anthos environment:
+#### Specify mappers drivers property or run `SPRING_PROFILES_ACTIVE=anthos`
+```properties
+obmDriver=minio
+osmDriver=postgres
+oqmDriver=rabbitmq
+```
+#### Specify osdu services urls:
+```properties
+DOMAIN=https://osdu.ref.gcp.gnrg-osdu.projects.epam.com
+AUTHORIZE_API=${DOMAIN}/api/entitlements/v2
+CRS_API=${DOMAIN}/api/crs/v2
+LEGALTAG_API=${DOMAIN}/api/legal/v1
+PARTITION_API=${DOMAIN}/api/partition/v1/
+```
+#### Auth variables:
+```properties
+opa.enabled=false;
+partition-auth-enabled=false
+service.token.provider=OPENID
+OPENID_PROVIDER_CLIENT_ID=${CLIENT_ID};
+OPENID_PROVIDER_CLIENT_SECRET=${CLIENT_SECRET};
+OPENID_PROVIDER_URL=${OPENID_PROVIDER_URL};
+```
+#### Redis variables:
+```properties
+REDIS_STORAGE_HOST=127.0.0.1
+REDIS_GROUP_HOST=127.0.0.1
+```
+#### Partition service should contain non-production sensitive properties to override them with localhost and use through a custom prefixes.
+##### OSM:
+```properties
+POSTGRES_DATASOURCE_URL_OSDU=jdbc:postgresql://localhost:5432/storage;
+POSTGRES_DB_USERNAME_OSDU=${POSTGRES_USERNAME};
+POSTGRES_DB_PASSWORD_OSDU=${POSTGRES_PASSWORD};
+```
+##### OBM:
+```properties
+OBM_MINIO_PARTITION_PROPERTIES_PREFIX=obm.minio.localDebug;
+MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY};
+MINIO_SECRET_KEY=${MINIO_SECRET_KEY};
+```
+##### OQM:
+```properties
+OQM_RABBITMQ_PARTITION_PROPERTIES_PREFIX=oqm.rabbitmq.localDebug;
+RABBIT_MQ_HOST=localhost;
+RABBITMQ_ADMIN_HOST=localhost;
+RABBITMQ_ADMIN_USERNAME=${RABBITMQ_ADMIN_USERNAME};
+RABBITMQ_ADMIN_PASSWORD=${RABBITMQ_ADMIN_PASSWORD};
+```
+#### To connect postgres or minio or rabbitmq port-forwarding should be used:
+```shell
+gcloud auth list
+gcloud config set account <account_name>
+gcloud config set project <project_name>
+kubectl port-forward <rabbit_pod_name> 15672:15672
+kubectl port-forward <rabbit_pod_name> 5672:5672
+kubectl port-forward <minio_pod_name> 9000:9000
+gcloud components install cloud_sql_proxy
+cloud_sql_proxy -instances=<instance_connection_string> -credential_file=<anthos_service_account_json_file>
+```
diff --git a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/repository/ObmStorage.java b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/repository/ObmStorage.java
index ac43ff3551cad4738bce76eb50125acc9d41275c..5974e7df4adb28d9e409641e0c771147aa51aeff 100644
--- a/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/repository/ObmStorage.java
+++ b/provider/storage-gc/src/main/java/org/opengroup/osdu/storage/provider/gcp/web/repository/ObmStorage.java
@@ -185,7 +185,7 @@ public class ObmStorage implements ICloudStorage {
throw new ObmDriverRuntimeException(S3CompatibleErrors.NO_SUCH_KEY_CODE, new RuntimeException(String.format("'%s' not found", path)));
}
} catch (ObmDriverRuntimeException exception) {
- throw new AppException(HttpStatus.SC_FORBIDDEN, ACCESS_DENIED_ERROR_REASON, ACCESS_DENIED_ERROR_MSG, exception);
+ throw new AppException(exception.getError().getHttpStatusCode(), exception.getCause().getMessage(), exception.getMessage(), exception);
}
}
diff --git a/storage-core/src/main/java/org/opengroup/osdu/storage/conversion/CrsConversionService.java b/storage-core/src/main/java/org/opengroup/osdu/storage/conversion/CrsConversionService.java
index 55a2cc5f39f4f40ae690d493ee4c37cd30875b40..ba79800e689a3602067fa00aa1fee54716517be0 100644
--- a/storage-core/src/main/java/org/opengroup/osdu/storage/conversion/CrsConversionService.java
+++ b/storage-core/src/main/java/org/opengroup/osdu/storage/conversion/CrsConversionService.java
@@ -119,6 +119,9 @@ public class CrsConversionService {
JsonObject recordJsonObject = originalRecords.get(i);
String recordId = this.getRecordId(recordJsonObject);
ConversionStatus.ConversionStatusBuilder statusBuilder = this.getConversionStatusBuilderFromList(recordId, conversionStatuses);
+ if(statusBuilder == null){
+ continue;
+ }
JsonObject dataBlock = recordJsonObject.getAsJsonObject(Constants.DATA);
if (dataBlock == null) {
statusBuilder.addError(CrsConversionServiceErrorMessages.MISSING_DATA_BLOCK);
@@ -142,6 +145,9 @@ public class CrsConversionService {
JsonObject recordJsonObject = originalRecords.get(i);
String recordId = this.getRecordId(recordJsonObject);
ConversionStatus.ConversionStatusBuilder statusBuilder = this.getConversionStatusBuilderFromList(recordId, conversionStatuses);
+ if(statusBuilder == null){
+ continue;
+ }
List<String> validationErrors = new ArrayList<>();
JsonObject filteredObjects = this.dpsConversionService.filterDataFields(recordJsonObject, validationErrors);
Iterator<String> keys = filteredObjects.keySet().iterator();
@@ -189,7 +195,6 @@ public class CrsConversionService {
}
} else {
statusBuilder.addError(CrsConversionServiceErrorMessages.MISSING_AS_INGESTED_COORDINATES);
- continue;
}
}
}
diff --git a/storage-core/src/main/java/org/opengroup/osdu/storage/service/BatchServiceImpl.java b/storage-core/src/main/java/org/opengroup/osdu/storage/service/BatchServiceImpl.java
index e7616b783641daf5798f80360876087db17cd14f..717f18f397fef48b36a34e7de94e168de577eee4 100644
--- a/storage-core/src/main/java/org/opengroup/osdu/storage/service/BatchServiceImpl.java
+++ b/storage-core/src/main/java/org/opengroup/osdu/storage/service/BatchServiceImpl.java
@@ -194,6 +194,9 @@ public abstract class BatchServiceImpl implements BatchService {
}
validRecords.put(recordId, recordMetadata.getVersionPath(recordMetadata.getLatestVersion()));
}
+ if (!recordsNotFound.isEmpty()) {
+ logger.error("Records were not found due to missed record metadata: " + recordsNotFound);
+ }
List<String> validRecordObjects = new ArrayList<>(validRecords.values());
if (validRecordObjects.isEmpty()) {
@@ -203,6 +206,7 @@ public abstract class BatchServiceImpl implements BatchService {
return response;
}
+ List<String> recordsNotFoundInCloudStorage = new ArrayList<>();
Map<String, String> recordsPreAclMap = this.cloudStorage.read(validRecords, collaborationContext);
this.logUnauthorizedGCSRecords(validRecords, recordsPreAclMap);
Map<String, String> recordsFromCloudStorage = this.postCheckRecordsAcl(recordsPreAclMap, recordsMetadata, collaborationContext);
@@ -214,7 +218,7 @@ public abstract class BatchServiceImpl implements BatchService {
recordsFromCloudStorage.keySet().forEach(recordId -> {
String recordData = recordsFromCloudStorage.get(recordId);
if (Strings.isNullOrEmpty(recordData)) {
- recordsNotFound.add(recordId);
+ recordsNotFoundInCloudStorage.add(recordId);
} else {
JsonElement jsonRecord = jsonParser.parse(recordData);
RecordMetadata recordMetadata = recordsMetadata.get(CollaborationUtil.getIdWithNamespace(recordId, collaborationContext));
@@ -224,21 +228,29 @@ public abstract class BatchServiceImpl implements BatchService {
}
});
+ if (!recordsNotFoundInCloudStorage.isEmpty()) {
+ logger.error("Records were not found in cloud storage: " + recordsNotFoundInCloudStorage);
+ recordsNotFound.addAll(recordsNotFoundInCloudStorage);
+ }
+
if (isConversionNeeded && !validRecords.isEmpty()) {
RecordsAndStatuses recordsAndStatuses = this.conversionService.doConversion(jsonObjectRecords);
this.checkMismatchAndAddToNotFound(recordIds, recordsNotFound, recordsAndStatuses.getRecords());
response.setConversionStatuses(recordsAndStatuses.getConversionStatuses());
response.setRecords(this.convertFromJsonObjectListToStringList(recordsAndStatuses.getRecords()));
response.setNotFound(recordsNotFound);
- return response;
+ } else {
+ this.checkMismatchAndAddToNotFound(recordIds, recordsNotFound, jsonObjectRecords);
+ response.setConversionStatuses(conversionStatuses);
+ response.setRecords(this.convertFromJsonObjectListToStringList(jsonObjectRecords));
+ response.setNotFound(recordsNotFound);
+ this.auditLog(validRecordObjects, this.auditLogger::readMultipleRecordsWithOptionalConversionSuccess,
+ recordsNotFound, this.auditLogger::readMultipleRecordsWithOptionalConversionFail);
}
- this.checkMismatchAndAddToNotFound(recordIds, recordsNotFound, jsonObjectRecords);
- response.setConversionStatuses(conversionStatuses);
- response.setRecords(this.convertFromJsonObjectListToStringList(jsonObjectRecords));
- response.setNotFound(recordsNotFound);
- this.auditLog(validRecordObjects, this.auditLogger::readMultipleRecordsWithOptionalConversionSuccess,
- recordsNotFound, this.auditLogger::readMultipleRecordsWithOptionalConversionFail);
+ if (!recordsNotFound.isEmpty()) {
+ logger.error("Records were not found in total: " + recordsNotFound);
+ }
return response;
}
@@ -259,20 +271,19 @@ public abstract class BatchServiceImpl implements BatchService {
}
if (isOpaEnabled) {
- List<ValidationOutputRecord> dataAuthzResult = this.opaService.validateUserAccessToRecords(recordMetadataList, OperationType.view);
- for (ValidationOutputRecord outputRecord : dataAuthzResult) {
+ List<ValidationOutputRecord> dataAuthResult = this.opaService.validateUserAccessToRecords(recordMetadataList, OperationType.view);
+ for (ValidationOutputRecord outputRecord : dataAuthResult) {
if (outputRecord.getErrors().isEmpty()) {
String recordId = outputRecord.getId();
String recordData = recordsPreAclMap.get(recordId);
- recordsMap.put(recordId, recordData); }
+ recordsMap.put(recordId, recordData);
+ }
}
} else {
-
List<RecordMetadata> passAclCheckRecordsMetadata = this.entitlementsAndCacheService.hasValidAccess(recordMetadataList, this.headers);
for (RecordMetadata metadata : passAclCheckRecordsMetadata) {
String recordId = metadata.getId();
String recordData = recordsPreAclMap.get(recordId);
-
recordsMap.put(recordId, recordData);
}
}
@@ -291,19 +302,18 @@ public abstract class BatchServiceImpl implements BatchService {
}
private void checkMismatchAndAddToNotFound(List<String> requestIds, List<String> notFoundIds, List<JsonObject> fetchedRecords) {
- if ((notFoundIds.size() + fetchedRecords.size()) == requestIds.size()) {
+ if (notFoundIds.size() + fetchedRecords.size() == requestIds.size()) {
return;
}
- List<String> fetchedIds = fetchedRecords.stream().map(e -> this.getRecordId(e)).collect(Collectors.toList());
+ List<String> fetchedIds = fetchedRecords.stream().map(this::getRecordId).collect(Collectors.toList());
for (String requestId : requestIds) {
if (!notFoundIds.contains(requestId) && !fetchedIds.contains(requestId)) {
- this.logger.warning("Missing record when fetch records, adding to not found: " + requestId);
+ this.logger.error("Missing record when fetch records, adding to not found: " + requestId);
notFoundIds.add(requestId);
}
}
- return;
}
private String getRecordId(JsonObject record) {
diff --git a/storage-core/src/test/java/org/opengroup/osdu/storage/service/RecordServiceImplTest.java b/storage-core/src/test/java/org/opengroup/osdu/storage/service/RecordServiceImplTest.java
index 49619fc80e6087404efae0a7f3cdf8b380a2b382..927f10094de124c97f668801ac31cc2880f39568 100644
--- a/storage-core/src/test/java/org/opengroup/osdu/storage/service/RecordServiceImplTest.java
+++ b/storage-core/src/test/java/org/opengroup/osdu/storage/service/RecordServiceImplTest.java
@@ -313,7 +313,7 @@ public class RecordServiceImplTest {
assertEquals("any kind", capturedRecord.getKind());
assertEquals(RECORD_ID, capturedRecord.getId());
assertEquals(RecordState.deleted, capturedRecord.getStatus());
- assertNotNull(capturedRecord.getModifyTime());
+ assertTrue(record.getModifyTime() != 0);
assertEquals("anyUserName", capturedRecord.getModifyUser());
ArgumentCaptor<PubSubDeleteInfo> pubsubMessageCaptor = ArgumentCaptor.forClass(PubSubDeleteInfo.class);
@@ -398,7 +398,6 @@ public class RecordServiceImplTest {
assertEquals(RecordState.deleted, record.getStatus());
assertEquals(USER_NAME, record.getModifyUser());
- assertNotNull(record.getModifyTime());
assertTrue(record.getModifyTime() != 0);
}
@@ -448,7 +447,6 @@ public class RecordServiceImplTest {
assertEquals(RecordState.deleted, record.getStatus());
assertEquals(USER_NAME, record.getModifyUser());
- assertNotNull(record.getModifyTime());
assertTrue(record.getModifyTime() != 0);
}
@@ -527,8 +525,7 @@ public class RecordServiceImplTest {
assertEquals(RecordState.deleted, record.getStatus());
assertEquals(USER_NAME, record.getModifyUser());
- assertNotNull(record.getModifyTime());
-
+ assertTrue(record.getModifyTime() != 0);
assertEquals(1, e.getNotDeletedRecords().size());
assertEquals(RECORD_ID_1, e.getNotDeletedRecords().get(0).getKey());
assertEquals(expectedErrorMessage, e.getNotDeletedRecords().get(0).getValue());
diff --git a/testing/storage-test-aws/pom.xml b/testing/storage-test-aws/pom.xml
index 7f7a537df8749ce6e3e24a5458388bcddef26bcd..8681b5c3a867e7411592ae9cf1137e7f22871a85 100644
--- a/testing/storage-test-aws/pom.xml
+++ b/testing/storage-test-aws/pom.xml
@@ -40,7 +40,7 @@
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
<artifactId>os-core-lib-aws</artifactId>
- <version>0.14.0-rc2</version>
+ <version>0.19.0-rc3</version>
</dependency>
<dependency>