Merge branch feature/morten.ofstad/ImportInformation with refs/heads/master...

Merge branch feature/morten.ofstad/ImportInformation with refs/heads/master into refs/merge-requests/81/train

src/OpenVDS/IO/File.h

@@ -97,6 +97,7 @@ public:
   OPENVDS_EXPORT void Close();
 
   OPENVDS_EXPORT int64_t Size(Error& error) const;
+  OPENVDS_EXPORT std::string LastWriteTime(Error& error) const;
 
   OPENVDS_EXPORT bool Read(void* pxData, int64_t nOffset, int32_t nLength, Error& error) const;
   OPENVDS_EXPORT bool Write(const void* pxData, int64_t nOffset, int32_t nLength, Error& error);

src/OpenVDS/IO/IOManager.h

@@ -31,6 +31,7 @@ namespace OpenVDS
   public:
     virtual ~TransferDownloadHandler() {};
     virtual void HandleObjectSize(int64_t size) = 0;
+    virtual void HandleObjectLastWriteTime(const std::string &lastWriteTimeISO8601) = 0;
     virtual void HandleMetadata(const std::string &key, const std::string &header) = 0;
     virtual void HandleData(std::vector<uint8_t> &&data) = 0;
     virtual void Completed(const Request &request, const Error &error) = 0;

src/OpenVDS/IO/IOManagerAWS.cpp

@@ -150,6 +150,10 @@ namespace OpenVDS
 
       int64_t content_length = int64_t(result.GetContentLength());
       objReq->m_handler->HandleObjectSize(content_length);
+
+      auto lastModified = result.GetLastModified();
+      objReq->m_handler->HandleObjectLastWriteTime(convertAwsString(lastModified.ToGmtString(Aws::Utils::DateFormat::ISO_8601)));
+
       for (auto it : result.GetMetadata())
       {
         objReq->m_handler->HandleMetadata(convertAwsString(it.first), convertAwsString(it.second));
@@ -174,7 +178,6 @@ namespace OpenVDS
   ReadObjectInfoRequestAWS::ReadObjectInfoRequestAWS(const std::string &id, const std::shared_ptr<TransferDownloadHandler>& handler)
     : GetOrHeadRequestAWS(id, handler)
   {
-
   }
 
   void ReadObjectInfoRequestAWS::run(Aws::S3::S3Client& client, const std::string& bucket, std::weak_ptr<ReadObjectInfoRequestAWS> readObjectInfoRequest)
@@ -201,12 +204,18 @@ namespace OpenVDS
     if (getObjectOutcome.IsSuccess())
     {
       Aws::S3::Model::GetObjectResult result = const_cast<Aws::S3::Model::GetObjectOutcome&>(getObjectOutcome).GetResultWithOwnership();
+
       int64_t content_length = int64_t(result.GetContentLength());
       objReq->m_handler->HandleObjectSize(content_length);
+
+      auto lastModified = result.GetLastModified();
+      objReq->m_handler->HandleObjectLastWriteTime(convertAwsString(lastModified.ToGmtString(Aws::Utils::DateFormat::ISO_8601)));
+
       for (auto it : result.GetMetadata())
       {
         objReq->m_handler->HandleMetadata(convertAwsString(it.first), convertAwsString(it.second));
       }
+
       auto& retrieved_object = result.GetBody();
       std::vector<uint8_t> data;
 
@@ -236,6 +245,7 @@ namespace OpenVDS
     : GetOrHeadRequestAWS(id, handler)
   {
   }
+
   void DownloadRequestAWS::run(Aws::S3::S3Client& client, const std::string& bucket, const IORange& range, std::weak_ptr<DownloadRequestAWS> downloadRequest)
   {
     Aws::S3::Model::GetObjectRequest object_request;

src/OpenVDS/IO/IOManagerAzure.cpp

@@ -149,8 +149,11 @@ void ReadObjectInfoRequestAzure::run(azure::storage::cloud_blob_container& conta
 
         if (auto tmp = request.lock())
         {
-          // send metadata one at a time to the metadata handler
           m_handler->HandleObjectSize(m_blob.properties().size());
+
+          m_handler->HandleObjectLastWriteTime(convertFromUtilString(m_blob.properties().last_modified().to_string(utility::datetime::ISO_8601)));
+
+          // send metadata one at a time to the metadata handler
           for (auto it : m_blob.metadata())
           {
             m_handler->HandleMetadata(convertFromUtilString(it.first), convertFromUtilString(it.second));
@@ -214,9 +217,11 @@ void DownloadRequestAzure::run(azure::storage::cloud_blob_container& container,
 
         if (auto tmp = request.lock())
         {
-          // send metadata one at a time to the metadata handler
           m_handler->HandleObjectSize(m_blob.properties().size());
 
+          m_handler->HandleObjectLastWriteTime(convertFromUtilString(m_blob.properties().last_modified().to_string(utility::datetime::ISO_8601)));
+
+          // send metadata one at a time to the metadata handler
           for (auto it : m_blob.metadata())
           {
             m_handler->HandleMetadata(convertFromUtilString(it.first), convertFromUtilString(it.second));

src/OpenVDS/IO/Linux_File.cpp

@@ -31,6 +31,8 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <fcntl.h>
+#include <fmt/format.h>
+#include <time.h>
 
 namespace OpenVDS
 {
@@ -71,7 +73,7 @@ static __thread sigjmp_buf * SystemFileView_pSigjmpEnv;
 class SystemFileView : public FileView
 {
 private:
-	static void SIGBUSHandler(int sig)
+  static void SIGBUSHandler(int sig)
   {
     if(SystemFileView_pSigjmpEnv)
     {
@@ -113,7 +115,7 @@ private:
 
 public:
   SystemFileView(SystemFileMappingObject *pFileMappingObject, int64_t nPos, int64_t nSize, bool isPopulate, Error &error)
-		: m_pxBaseAddress(NULL)
+    : m_pxBaseAddress(NULL)
   {
     (void)&m_cSignalHandlerInstaller;
 
@@ -148,7 +150,7 @@ public:
 
     if (isPopulate)
     {
-			int32_t iRetval = madvise((void*)m_pxBaseAddress, m_nNumberOfBytes, MADV_WILLNEED);
+      int32_t iRetval = madvise((void*)m_pxBaseAddress, m_nNumberOfBytes, MADV_WILLNEED);
 
       if (iRetval != 0)
       {
@@ -180,9 +182,9 @@ public:
     if (iRetval != 0)
     {
       SetIoError(errno, "FileView::prefetch ", error);
-			return false;
+      return false;
     }
-		return true;
+    return true;
   }
 };
 
@@ -245,34 +247,51 @@ int64_t File::Size(Error& error) const
   return nLength;
 }
 
+std::string File::LastWriteTime(Error& error) const
+{
+  struct stat result;
+  if (lstat(_cFileName.c_str(), &result) < 0)
+  {
+    SetIoError(errno, "File::size ", error);
+    return std::string();
+  }
+
+  struct tm lastWriteTime;
+  gmtime_r(&result.st_mtime, &lastWriteTime);
+
+  const long millis = result.st_mtim.tv_nsec / 1000;
+
+  return fmt::format("{:04d}-{:02d}-{:02d}T{:02d}:{:02d}:{:02d}.{:03d}Z", lastWriteTime.tm_year + 1900, lastWriteTime.tm_mon + 1, lastWriteTime.tm_mday, lastWriteTime.tm_hour, lastWriteTime.tm_min, lastWriteTime.tm_sec, millis);
+}
+
 bool File::Read(void* pxData, int64_t nOffset, int32_t nLength, Error& error) const
 {
-	assert(nOffset >= 0);
-	int fd    = (int)(intptr_t)_pxPlatformHandle;
-	ssize_t nread;
+  assert(nOffset >= 0);
+  int fd = (int)(intptr_t)_pxPlatformHandle;
+  ssize_t nread;
 
-	while (nLength > 0)
-	{
-		nread = pread(fd, pxData, nLength, nOffset);
-		assert(nread <= nLength);
+  while (nLength > 0)
+  {
+    nread = pread(fd, pxData, nLength, nOffset);
+    assert(nread <= nLength);
 
-		if (nread < 0)
-		{
+    if (nread < 0)
+    {
       SetIoError(errno, "File::read ", error);
-			return false;
-		}
+      return false;
+    }
 
-		if (nread == 0)
-		{
+    if (nread == 0)
+    {
       SetIoError(errno, "File::read (zero-length read) ", error);
-			return false;
-		}
-
-		nLength -= nread;
-		pxData = (char*)pxData + nread;
-		nOffset += nread;
-	}
-	return true;
+      return false;
+    }
+
+    nLength -= nread;
+    pxData = (char*)pxData + nread;
+    nOffset += nread;
+  }
+  return true;
 }
 
 bool File::Write(const void* pxData, int64_t nOffset, int32_t nLength, Error & error)
@@ -280,10 +299,10 @@ bool File::Write(const void* pxData, int64_t nOffset, int32_t nLength, Error & e
   assert(nOffset >= 0);
 
   if (!_isWriteable)
-	{
+  {
     SetIoError("File::write: file not writeable", error);
-		return false;
-	}
+    return false;
+  }
 
   int fd  = (int)(intptr_t)_pxPlatformHandle;
   ssize_t nwritten;
@@ -314,11 +333,11 @@ bool File::Write(const void* pxData, int64_t nOffset, int32_t nLength, Error & e
 bool File::Flush()
 {
 #ifdef LINUX
-	int fd  = (int)(intptr_t)_pxPlatformHandle;
-	return syncfs(fd) == 0;
+  int fd  = (int)(intptr_t)_pxPlatformHandle;
+  return syncfs(fd) == 0;
 #else
-        sync();
-        return true;
+  sync();
+  return true;
 #endif
 }
 

src/OpenVDS/IO/Win_File.cpp

@@ -23,6 +23,12 @@
 
 #include <io.h>
 #include <windows.h>
+#include <fmt/format.h>
+
+#include <sys/stat.h>
+#include <time.h>
+#define stat _stat
+#define gmtime_r(a,b) _gmtime64_s(b,a)
 
 namespace OpenVDS
 {
@@ -277,6 +283,27 @@ int64_t File::Size(Error& error) const
   return int64_t(li.QuadPart);
 }
 
+std::string File::LastWriteTime(Error& error) const
+{
+  FILETIME lastWriteTime;
+
+  if(!GetFileTime(_pxPlatformHandle, NULL, NULL, &lastWriteTime))
+  {
+    SetIoError(GetLastError(), "GetFileTime: ", error);
+    return std::string();
+  }
+
+  SYSTEMTIME systemTime;
+
+  if(!FileTimeToSystemTime(&lastWriteTime, &systemTime))
+  {
+    SetIoError(GetLastError(), "FileTimeToSystemTime: ", error);
+    return std::string();
+  }
+
+  return fmt::format("{:04d}-{:02d}-{:02d}T{:02d}:{:02d}:{:02d}.{:03d}Z", systemTime.wYear, systemTime.wMonth, systemTime.wDay, systemTime.wHour, systemTime.wMinute, systemTime.wSecond, systemTime.wMilliseconds);
+}
+
 bool File::Read(void* pxData, int64_t nOffset, int32_t nLength, Error& error) const
 {
   assert(nOffset >= 0);

src/OpenVDS/VDS/MetadataManager.cpp

@@ -36,6 +36,7 @@ MetadataPageTransfer(MetadataManager *manager, VolumeDataAccessManagerImpl *acce
 { }
 
 void HandleObjectSize(int64_t size) override {}
+void HandleObjectLastWriteTime(const std::string &lastWriteTimeISO8601) override {}
 void HandleMetadata(const std::string &key, const std::string &header) {};
 void HandleData(std::vector<uint8_t> &&data) override
 {

src/OpenVDS/VDS/ParseVDSJson.cpp

@@ -1150,6 +1150,9 @@ public:
   void HandleObjectSize(int64_t size) override
   {
   }
+  void HandleObjectLastWriteTime(const std::string &lastWriteTimeISO8601) override
+  {
+  }
   void HandleMetadata(const std::string &key, const std::string &header) override
   {
   }

src/OpenVDS/VDS/VolumeDataAccessManagerImpl.h

@@ -51,6 +51,10 @@ public:
   {
   }
 
+  void HandleObjectLastWriteTime(const std::string &lastWriteTimeISO8601) override
+  {
+  }
+
   void HandleMetadata(const std::string& key, const std::string& header) override
   {
     if (key == "vdschunkmetadata")

src/SEGYUtils/SEGYUtils/DataProvider.h

@@ -31,6 +31,12 @@ struct DataTransfer : public OpenVDS::TransferDownloadHandler
   {
     this->size = size;
   }
+
+  void HandleObjectLastWriteTime(const std::string &lastWriteTimeISO8601) override
+  {
+    this->lastWriteTime = lastWriteTimeISO8601;
+  }
+
   void HandleMetadata(const std::string &key, const std::string &header) override
   {
   }
@@ -46,6 +52,7 @@ struct DataTransfer : public OpenVDS::TransferDownloadHandler
   }
 
   int64_t size = 0;
+  std::string lastWriteTime;
   std::vector<uint8_t> data;
   OpenVDS::Error error;
 };
@@ -71,6 +78,7 @@ struct DataProvider
       if (syncRequest->IsSuccess(error))
       {
         m_size = syncTransfer->size;
+        m_lastWriteTime = syncTransfer->lastWriteTime;
       }
     }
   }
@@ -81,7 +89,8 @@ struct DataProvider
     {
       return m_file->Read(data, offset, length, error);
     }
-    else if (m_ioManager)
+    
+    if (m_ioManager)
     {
       auto dataTransfer = std::make_shared<DataTransfer>();
       auto request = m_ioManager->ReadObject(m_objectName, dataTransfer, { offset, offset + length});
@@ -94,6 +103,7 @@ struct DataProvider
       memcpy(data, dataTransfer->data.data(), std::min(size_t(length), dataTransfer->data.size()));
       return true;
     }
+
     error.code = -1;
     error.string = "Invalid dataprovider, no file nor ioManager provided";
     return false;
@@ -106,15 +116,35 @@ struct DataProvider
 
     if (m_ioManager)
       return m_size;
+
     error.code = -1;
     error.string = "Invalid dataprovider, no file nor ioManager provided";
     return 0;
   }
 
+  std::string LastWriteTime(OpenVDS::Error &error) const
+  {
+    if (m_file)
+      return m_file->LastWriteTime(error);
+
+    if (m_ioManager)
+      return m_lastWriteTime;
+
+    error.code = -1;
+    error.string = "Invalid dataprovider, no file nor ioManager provided";
+    return 0;
+  }
+
+  std::string FileOrObjectName() const
+  {
+    return m_file ? m_file->FileName() : m_objectName;
+  }
+
   std::unique_ptr<OpenVDS::File> m_file;
   std::unique_ptr<OpenVDS::IOManager> m_ioManager;
   const std::string m_objectName;
-  int64_t m_size;
+  int64_t m_size = 0;
+  std::string m_lastWriteTime;
 };
 
 struct DataView

tests/io/AzureIo.cpp

@@ -24,6 +24,10 @@ public:
   {
   }
 
+  void HandleObjectLastWriteTime(const std::string &lastWriteTimeISO8601) override
+  {
+  }
+
   void HandleMetadata(const std::string& key, const std::string& header) override
   {
     if (key == "vdschunkmetadata")

tests/io/IoPerformance.cpp

@@ -24,6 +24,11 @@ public:
   void HandleObjectSize(int64_t size) override
   {
   }
+
+  void HandleObjectLastWriteTime(const std::string &lastWriteTimeISO8601) override
+  {
+  }
+
   void HandleMetadata(const std::string& key, const std::string& header) override
   {
   }

tests/utils/FacadeIOManager.h

@@ -100,6 +100,7 @@ public:
       auto &object = (*object_it).second;
       error = object.error;
       handler->HandleObjectSize(int64_t(object.data.size()));
+
       for (auto& meta : object.metaHeader)
       {
         handler->HandleMetadata(meta.first, meta.second);

tools/SEGYImport/SEGYImport.cpp

@@ -709,6 +709,52 @@ createSurveyCoordinateSystemMetadata(SEGYFileInfo const& fileInfo, OpenVDS::Meta
   metadataContainer.SetMetadataDoubleVector2(LATTICE_CATEGORY, LATTICE_CROSSLINE_SPACING, OpenVDS::DoubleVector2(crosslineSpacing[0], crosslineSpacing[1]));
 }
 
+/////////////////////////////////////////////////////////////////////////////
+bool
+createImportInformationMetadata(DataProvider &dataProvider, OpenVDS::MetadataContainer& metadataContainer, OpenVDS::Error &error)
+{
+  auto now = std::chrono::system_clock::now();
+  std::time_t tt = std::chrono::system_clock::to_time_t(now);
+  std::tm tm = *std::gmtime(&tt);
+  auto duration = now.time_since_epoch();
+  int millis = std::chrono::duration_cast<std::chrono::milliseconds>(duration).count() % 1000;
+
+  std::string importTimeStamp = fmt::format("{:04d}-{:02d}-{:02d}T{:02d}:{:02d}:{:02d}.{:03d}Z", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, millis);
+
+  std::string inputFileName = dataProvider.FileOrObjectName();
+
+  // Strip the path from the file/object name
+  const char pathSeparators[] = { '/', '\\', ':' };
+  for(auto pathSeparator : pathSeparators)
+  {
+    size_t pos = inputFileName.rfind(pathSeparator);
+    if(pos != std::string::npos) inputFileName = inputFileName.substr(pos + 1);
+  }
+
+  // In lack of a better displayName we use the file name
+  std::string displayName = inputFileName;
+
+  std::string inputTimeStamp = dataProvider.LastWriteTime(error);
+  if (error.code != 0)
+  {
+    return false;
+  }
+
+  int64_t inputFileSize = dataProvider.Size(error);
+  if (error.code != 0)
+  {
+    return false;
+  }
+
+  // Set import information
+  metadataContainer.SetMetadataString(KNOWNMETADATA_CATEGORY_IMPORTINFORMATION, KNOWNMETADATA_IMPORTINFORMATION_DISPLAYNAME, displayName);
+  metadataContainer.SetMetadataString(KNOWNMETADATA_CATEGORY_IMPORTINFORMATION, KNOWNMETADATA_IMPORTINFORMATION_INPUTFILENAME, inputFileName);
+  metadataContainer.SetMetadataDouble(KNOWNMETADATA_CATEGORY_IMPORTINFORMATION, KNOWNMETADATA_IMPORTINFORMATION_INPUTFILESIZE, (double)inputFileSize);
+  metadataContainer.SetMetadataString(KNOWNMETADATA_CATEGORY_IMPORTINFORMATION, KNOWNMETADATA_IMPORTINFORMATION_INPUTTIMESTAMP, inputTimeStamp);
+  metadataContainer.SetMetadataString(KNOWNMETADATA_CATEGORY_IMPORTINFORMATION, KNOWNMETADATA_IMPORTINFORMATION_IMPORTTIMESTAMP, importTimeStamp);
+  return true;
+}
+
 bool
 parseSEGYFileInfoFile(OpenVDS::File const& file, SEGYFileInfo& fileInfo)
 {
@@ -1355,6 +1401,14 @@ main(int argc, char* argv[])
   OpenVDS::MetadataContainer
     metadataContainer;
 
+  createImportInformationMetadata(dataProvider, metadataContainer, error);
+
+  if (error.code != 0)
+  {
+    std::cerr << error.string;
+    return EXIT_FAILURE;
+  }
+
   createSEGYHeadersMetadata(dataProvider, metadataContainer, error);
 
   if (error.code != 0)