Merge branch 'feature/fix-datetime-validation' into 'develop'

Feature/fix datetime validation

See merge request !109

.run/App.run.xml

@@ -4,11 +4,15 @@
       <env name="APPLICATIONINSIGHTS_CONNECTION_STRING" value="InstrumentationKey\=351ad1c3-bb85-4aeb-9a35-b8f8be588849;IngestionEndpoint\=https://southeastasia-1.in.applicationinsights.azure.com/;LiveEndpoint\=https://southeastasia.livediagnostics.monitor.azure.com/;ApplicationId\=86e320e5-8214-4ce0-9610-5d417c8ee70f" />
       <env name="APPLICATIONINSIGHTS_ROLE_NAME" value="pddms-entity" />
       <env name="AZURE" value="1" />
+      <env name="AZURE_CLIENT_ID" value="e4d6ad82-7bd8-46be-baf6-3c7c790b399b" />
+      <env name="AZURE_CLIENT_SECRET" value="jxJ8Q~Apiwqc7DzxVk_PYe-GHcFcAdYEo_vcRck5" />
       <env name="AZURE_DNS_NAME" value="osdu-ship.msft-osdu-test.org" />
       <env name="AZURE_SERVICEBUS_FULL_NAMESPACE" value="osduservicebus.servicebus.windows.net" />
       <env name="AZURE_SERVICEBUS_MANAGED_CLIENT_ID" value="f9bb3b94-9683-4161-951c-5ca343496d2c" />
       <env name="AZURE_SERVICEBUS_SHORT_NAMESPACE" value="osduservicebus" />
+      <env name="AZURE_TENANT_ID" value="58975fd3-4977-44d0-bea8-37af0baac100" />
       <env name="CLOUD_PROVIDER" value="microsoft" />
+      <env name="CLOUD_SERVICE_MESSAGING" value="servicebus" />
       <env name="CLOUD_SERVICE_PROVIDER" value="azure" />
       <env name="JWT_AUTHENTICATION_DISABLED" value="false" />
       <env name="OSDU_CORE_ENTITLEMENT_APIKEY" value="OBSOLETE" />
@@ -28,7 +32,6 @@
       <env name="PRESHIPPING_STORAGE_PATH" value="/api/storage/v2" />
       <env name="REDIS_HOSTNAME" value="localhost" />
       <env name="REDIS_PORT" value="6379" />
-      <env name="CLOUD_SERVICE_MESSAGING" value="servicebus" />
     </envs>
     <option name="MAIN_CLASS_NAME" value="org.opengroup.osdu.production.App" />
     <module name="pddms-timeseries-ingestion" />

src/main/scala/org/opengroup/osdu/production/WritebackApp.scala

@@ -17,7 +17,7 @@
 
 package org.opengroup.osdu.production
 
-import java.util.concurrent.{Executors, TimeUnit}
+import java.util.concurrent.Executors
 import org.apache.pekko.Done
 import org.apache.pekko.actor.ActorSystem
 import org.apache.pekko.http.scaladsl.model.{HttpRequest, HttpResponse}
@@ -35,8 +35,8 @@ import org.opengroup.osdu.production.entities.others.{EntityIdCache, EntityIdSto
 import org.opengroup.osdu.production.flow.UnSecureHttpFlow
 import org.slf4j.MDC
 
-import scala.concurrent.duration.Duration
-import scala.concurrent.{Await, ExecutionContext, Future, Promise}
+
+import scala.concurrent.{ExecutionContext, Future, Promise}
 import scala.util.{Failure, Success, Try}
 
 class WritebackApp extends LogF {

src/main/scala/org/opengroup/osdu/production/util/validation/SingleWriteBackRequestValidator.scala

@@ -54,9 +54,6 @@ object SingleWriteBackRequestValidator extends LogF {
                           elementDetailsMap.put(element.id, element)
                       }
                       loggerF.logInfo(_ => StructuredLog(s"validate the Request Details And TimeSeries Data", EventTypes.DataUpdate, "", LogCategories.Asynchronous, "", Map("requestId" -> requestId)))
-                      loggerF.info(s"timeSeries1: ${timeSeries.timeSeriesDateTime}")
-                      loggerF.info(s"timeSeries2: ${timeSeries.timeSeriesDateTime.get}")
-                      loggerF.info(s"timeSeries3: ${timeSeries.timeSeriesDateTime.get.points}")
                       validateRequestDetailsAndTimeSeriesData(requestId, requestStartTime, source, entityKey, propertyDescriptor, timeSeries, elementDetailsMap).map {
                         case Good(derivedRequestDetails: DerivedRequestDetails) =>
                           Good(WriteBackDetails(subscriberDetails, derivedRequestDetails, RequestDetailsWithTimeSeries(requestId, RequestDetails(source, entityKey, propertyDescriptor, requestStartTime), timeSeries)))
@@ -79,12 +76,13 @@ object SingleWriteBackRequestValidator extends LogF {
             }
         }
       case Good(None) => Future(Bad(ErrorMessageResponse(StatusCodes.NotFound.intValue, StatusCodes.NotFound.reason, WriteBackServiceMessages.unsupportedSource)))
-      case Bad(None) => Future(Bad(ErrorMessageResponse(StatusCodes.InternalServerError.intValue, StatusCodes.InternalServerError.reason, "Error while processing request")))
+      case Bad(None) => Future(Bad(ErrorMessageResponse(StatusCodes.InternalServerError.intValue, StatusCodes.InternalServerError.reason, "Error while processing request1")))
     }.recover {
       case exception: Exception =>
-        loggerF.logError(_ => StructuredLog(s"Error while processing request ${exception.getCause}", EventTypes.DataUpdate, "", LogCategories.Asynchronous, "",
+        logger.error("Error while processing request", exception)
+        loggerF.logError(_ => StructuredLog(s"Error while processing request2 ${exception.getCause}", EventTypes.DataUpdate, "", LogCategories.Asynchronous, "",
           Map("requestId" -> requestId)))
-        Bad(ErrorMessageResponse(StatusCodes.InternalServerError.intValue, StatusCodes.InternalServerError.reason, "Error while processing request"))
+        Bad(ErrorMessageResponse(StatusCodes.InternalServerError.intValue, StatusCodes.InternalServerError.reason, "Error while processing request3"))
     }
   }
 }

src/main/scala/org/opengroup/osdu/production/util/validation/ValidationUtil.scala

@@ -129,13 +129,13 @@ object ValidationUtil extends LogF {
         } else {
           logger.info(s"received data points1: ${timeSeries.timeSeriesDateTime}")
           logger.info(s"received data points2: ${timeSeries.timeSeriesDateTime.get.points}")
-          val isIsoDateTime = timeSeries.timeSeriesDateTime.get.points.foldLeft(true) { (valid, point) =>
-            val p = point.value.asInstanceOf[String]
-            valid & Try(ZonedDateTime.parse(p)).isSuccess
+          val inValidDateTimes = timeSeries.timeSeriesDateTime.get.points.foldLeft(List.empty[String]) { (inValid, point) =>
+            val pValue = point.value.asInstanceOf[String]
+            if Try(ZonedDateTime.parse(pValue)).isSuccess then inValid else pValue :: inValid
           }
 
-          if (!isIsoDateTime) {
-            Future(Bad(ErrorMessageResponse(StatusCodes.BadRequest.intValue, StatusCodes.BadRequest.reason, WriteBackServiceMessages.createMessage(WriteBackServiceMessages.invalidDataTypeISODateTime, timeSeries.timeSeriesSetString.get.points.map(_.value).mkString(", "), "timeSeriesDateTime"))))
+          if (!inValidDateTimes.isEmpty) {
+            Future(Bad(ErrorMessageResponse(StatusCodes.BadRequest.intValue, StatusCodes.BadRequest.reason, WriteBackServiceMessages.createMessage(WriteBackServiceMessages.invalidDataTypeISODateTime, inValidDateTimes.mkString(", "), "timeSeriesDateTime"))))
           } else {
 
 //            val invalidDataPoints = validateDataFormat("timeSeriesDateTime", timeSeries.timeSeriesDateTime.get.points)