Commit 6dbb4a2a authored by Morten Ofstad's avatar Morten Ofstad
Browse files

Merge branch feature/morten.ofstad/ImportInformation with refs/heads/master...

Merge branch feature/morten.ofstad/ImportInformation with refs/heads/master into refs/merge-requests/81/train
parents 76cfac0d 3100191f
Pipeline #924 passed with stages
in 8 minutes and 13 seconds
......@@ -97,6 +97,7 @@ public:
OPENVDS_EXPORT void Close();
OPENVDS_EXPORT int64_t Size(Error& error) const;
OPENVDS_EXPORT std::string LastWriteTime(Error& error) const;
OPENVDS_EXPORT bool Read(void* pxData, int64_t nOffset, int32_t nLength, Error& error) const;
OPENVDS_EXPORT bool Write(const void* pxData, int64_t nOffset, int32_t nLength, Error& error);
......
......@@ -31,6 +31,7 @@ namespace OpenVDS
public:
virtual ~TransferDownloadHandler() {};
virtual void HandleObjectSize(int64_t size) = 0;
virtual void HandleObjectLastWriteTime(const std::string &lastWriteTimeISO8601) = 0;
virtual void HandleMetadata(const std::string &key, const std::string &header) = 0;
virtual void HandleData(std::vector<uint8_t> &&data) = 0;
virtual void Completed(const Request &request, const Error &error) = 0;
......
......@@ -150,6 +150,10 @@ namespace OpenVDS
int64_t content_length = int64_t(result.GetContentLength());
objReq->m_handler->HandleObjectSize(content_length);
auto lastModified = result.GetLastModified();
objReq->m_handler->HandleObjectLastWriteTime(convertAwsString(lastModified.ToGmtString(Aws::Utils::DateFormat::ISO_8601)));
for (auto it : result.GetMetadata())
{
objReq->m_handler->HandleMetadata(convertAwsString(it.first), convertAwsString(it.second));
......@@ -174,7 +178,6 @@ namespace OpenVDS
ReadObjectInfoRequestAWS::ReadObjectInfoRequestAWS(const std::string &id, const std::shared_ptr<TransferDownloadHandler>& handler)
: GetOrHeadRequestAWS(id, handler)
{
}
void ReadObjectInfoRequestAWS::run(Aws::S3::S3Client& client, const std::string& bucket, std::weak_ptr<ReadObjectInfoRequestAWS> readObjectInfoRequest)
......@@ -201,12 +204,18 @@ namespace OpenVDS
if (getObjectOutcome.IsSuccess())
{
Aws::S3::Model::GetObjectResult result = const_cast<Aws::S3::Model::GetObjectOutcome&>(getObjectOutcome).GetResultWithOwnership();
int64_t content_length = int64_t(result.GetContentLength());
objReq->m_handler->HandleObjectSize(content_length);
auto lastModified = result.GetLastModified();
objReq->m_handler->HandleObjectLastWriteTime(convertAwsString(lastModified.ToGmtString(Aws::Utils::DateFormat::ISO_8601)));
for (auto it : result.GetMetadata())
{
objReq->m_handler->HandleMetadata(convertAwsString(it.first), convertAwsString(it.second));
}
auto& retrieved_object = result.GetBody();
std::vector<uint8_t> data;
......@@ -236,6 +245,7 @@ namespace OpenVDS
: GetOrHeadRequestAWS(id, handler)
{
}
void DownloadRequestAWS::run(Aws::S3::S3Client& client, const std::string& bucket, const IORange& range, std::weak_ptr<DownloadRequestAWS> downloadRequest)
{
Aws::S3::Model::GetObjectRequest object_request;
......
......@@ -149,8 +149,11 @@ void ReadObjectInfoRequestAzure::run(azure::storage::cloud_blob_container& conta
if (auto tmp = request.lock())
{
// send metadata one at a time to the metadata handler
m_handler->HandleObjectSize(m_blob.properties().size());
m_handler->HandleObjectLastWriteTime(convertFromUtilString(m_blob.properties().last_modified().to_string(utility::datetime::ISO_8601)));
// send metadata one at a time to the metadata handler
for (auto it : m_blob.metadata())
{
m_handler->HandleMetadata(convertFromUtilString(it.first), convertFromUtilString(it.second));
......@@ -214,9 +217,11 @@ void DownloadRequestAzure::run(azure::storage::cloud_blob_container& container,
if (auto tmp = request.lock())
{
// send metadata one at a time to the metadata handler
m_handler->HandleObjectSize(m_blob.properties().size());
m_handler->HandleObjectLastWriteTime(convertFromUtilString(m_blob.properties().last_modified().to_string(utility::datetime::ISO_8601)));
// send metadata one at a time to the metadata handler
for (auto it : m_blob.metadata())
{
m_handler->HandleMetadata(convertFromUtilString(it.first), convertFromUtilString(it.second));
......
......@@ -31,6 +31,8 @@
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <fmt/format.h>
#include <time.h>
namespace OpenVDS
{
......@@ -71,7 +73,7 @@ static __thread sigjmp_buf * SystemFileView_pSigjmpEnv;
class SystemFileView : public FileView
{
private:
static void SIGBUSHandler(int sig)
static void SIGBUSHandler(int sig)
{
if(SystemFileView_pSigjmpEnv)
{
......@@ -113,7 +115,7 @@ private:
public:
SystemFileView(SystemFileMappingObject *pFileMappingObject, int64_t nPos, int64_t nSize, bool isPopulate, Error &error)
: m_pxBaseAddress(NULL)
: m_pxBaseAddress(NULL)
{
(void)&m_cSignalHandlerInstaller;
......@@ -148,7 +150,7 @@ public:
if (isPopulate)
{
int32_t iRetval = madvise((void*)m_pxBaseAddress, m_nNumberOfBytes, MADV_WILLNEED);
int32_t iRetval = madvise((void*)m_pxBaseAddress, m_nNumberOfBytes, MADV_WILLNEED);
if (iRetval != 0)
{
......@@ -180,9 +182,9 @@ public:
if (iRetval != 0)
{
SetIoError(errno, "FileView::prefetch ", error);
return false;
return false;
}
return true;
return true;
}
};
......@@ -245,34 +247,51 @@ int64_t File::Size(Error& error) const
return nLength;
}
std::string File::LastWriteTime(Error& error) const
{
struct stat result;
if (lstat(_cFileName.c_str(), &result) < 0)
{
SetIoError(errno, "File::size ", error);
return std::string();
}
struct tm lastWriteTime;
gmtime_r(&result.st_mtime, &lastWriteTime);
const long millis = result.st_mtim.tv_nsec / 1000;
return fmt::format("{:04d}-{:02d}-{:02d}T{:02d}:{:02d}:{:02d}.{:03d}Z", lastWriteTime.tm_year + 1900, lastWriteTime.tm_mon + 1, lastWriteTime.tm_mday, lastWriteTime.tm_hour, lastWriteTime.tm_min, lastWriteTime.tm_sec, millis);
}
bool File::Read(void* pxData, int64_t nOffset, int32_t nLength, Error& error) const
{
assert(nOffset >= 0);
int fd = (int)(intptr_t)_pxPlatformHandle;
ssize_t nread;
assert(nOffset >= 0);
int fd = (int)(intptr_t)_pxPlatformHandle;
ssize_t nread;
while (nLength > 0)
{
nread = pread(fd, pxData, nLength, nOffset);
assert(nread <= nLength);
while (nLength > 0)
{
nread = pread(fd, pxData, nLength, nOffset);
assert(nread <= nLength);
if (nread < 0)
{
if (nread < 0)
{
SetIoError(errno, "File::read ", error);
return false;
}
return false;
}
if (nread == 0)
{
if (nread == 0)
{
SetIoError(errno, "File::read (zero-length read) ", error);
return false;
}
nLength -= nread;
pxData = (char*)pxData + nread;
nOffset += nread;
}
return true;
return false;
}
nLength -= nread;
pxData = (char*)pxData + nread;
nOffset += nread;
}
return true;
}
bool File::Write(const void* pxData, int64_t nOffset, int32_t nLength, Error & error)
......@@ -280,10 +299,10 @@ bool File::Write(const void* pxData, int64_t nOffset, int32_t nLength, Error & e
assert(nOffset >= 0);
if (!_isWriteable)
{
{
SetIoError("File::write: file not writeable", error);
return false;
}
return false;
}
int fd = (int)(intptr_t)_pxPlatformHandle;
ssize_t nwritten;
......@@ -314,11 +333,11 @@ bool File::Write(const void* pxData, int64_t nOffset, int32_t nLength, Error & e
bool File::Flush()
{
#ifdef LINUX
int fd = (int)(intptr_t)_pxPlatformHandle;
return syncfs(fd) == 0;
int fd = (int)(intptr_t)_pxPlatformHandle;
return syncfs(fd) == 0;
#else
sync();
return true;
sync();
return true;
#endif
}
......
......@@ -23,6 +23,12 @@
#include <io.h>
#include <windows.h>
#include <fmt/format.h>
#include <sys/stat.h>
#include <time.h>
#define stat _stat
#define gmtime_r(a,b) _gmtime64_s(b,a)
namespace OpenVDS
{
......@@ -277,6 +283,27 @@ int64_t File::Size(Error& error) const
return int64_t(li.QuadPart);
}
std::string File::LastWriteTime(Error& error) const
{
FILETIME lastWriteTime;
if(!GetFileTime(_pxPlatformHandle, NULL, NULL, &lastWriteTime))
{
SetIoError(GetLastError(), "GetFileTime: ", error);
return std::string();
}
SYSTEMTIME systemTime;
if(!FileTimeToSystemTime(&lastWriteTime, &systemTime))
{
SetIoError(GetLastError(), "FileTimeToSystemTime: ", error);
return std::string();
}
return fmt::format("{:04d}-{:02d}-{:02d}T{:02d}:{:02d}:{:02d}.{:03d}Z", systemTime.wYear, systemTime.wMonth, systemTime.wDay, systemTime.wHour, systemTime.wMinute, systemTime.wSecond, systemTime.wMilliseconds);
}
bool File::Read(void* pxData, int64_t nOffset, int32_t nLength, Error& error) const
{
assert(nOffset >= 0);
......
......@@ -36,6 +36,7 @@ MetadataPageTransfer(MetadataManager *manager, VolumeDataAccessManagerImpl *acce
{ }
void HandleObjectSize(int64_t size) override {}
void HandleObjectLastWriteTime(const std::string &lastWriteTimeISO8601) override {}
void HandleMetadata(const std::string &key, const std::string &header) {};
void HandleData(std::vector<uint8_t> &&data) override
{
......
......@@ -1150,6 +1150,9 @@ public:
void HandleObjectSize(int64_t size) override
{
}
void HandleObjectLastWriteTime(const std::string &lastWriteTimeISO8601) override
{
}
void HandleMetadata(const std::string &key, const std::string &header) override
{
}
......
......@@ -51,6 +51,10 @@ public:
{
}
void HandleObjectLastWriteTime(const std::string &lastWriteTimeISO8601) override
{
}
void HandleMetadata(const std::string& key, const std::string& header) override
{
if (key == "vdschunkmetadata")
......
......@@ -31,6 +31,12 @@ struct DataTransfer : public OpenVDS::TransferDownloadHandler
{
this->size = size;
}
void HandleObjectLastWriteTime(const std::string &lastWriteTimeISO8601) override
{
this->lastWriteTime = lastWriteTimeISO8601;
}
void HandleMetadata(const std::string &key, const std::string &header) override
{
}
......@@ -46,6 +52,7 @@ struct DataTransfer : public OpenVDS::TransferDownloadHandler
}
int64_t size = 0;
std::string lastWriteTime;
std::vector<uint8_t> data;
OpenVDS::Error error;
};
......@@ -71,6 +78,7 @@ struct DataProvider
if (syncRequest->IsSuccess(error))
{
m_size = syncTransfer->size;
m_lastWriteTime = syncTransfer->lastWriteTime;
}
}
}
......@@ -81,7 +89,8 @@ struct DataProvider
{
return m_file->Read(data, offset, length, error);
}
else if (m_ioManager)
if (m_ioManager)
{
auto dataTransfer = std::make_shared<DataTransfer>();
auto request = m_ioManager->ReadObject(m_objectName, dataTransfer, { offset, offset + length});
......@@ -94,6 +103,7 @@ struct DataProvider
memcpy(data, dataTransfer->data.data(), std::min(size_t(length), dataTransfer->data.size()));
return true;
}
error.code = -1;
error.string = "Invalid dataprovider, no file nor ioManager provided";
return false;
......@@ -106,15 +116,35 @@ struct DataProvider
if (m_ioManager)
return m_size;
error.code = -1;
error.string = "Invalid dataprovider, no file nor ioManager provided";
return 0;
}
std::string LastWriteTime(OpenVDS::Error &error) const
{
if (m_file)
return m_file->LastWriteTime(error);
if (m_ioManager)
return m_lastWriteTime;
error.code = -1;
error.string = "Invalid dataprovider, no file nor ioManager provided";
return 0;
}
std::string FileOrObjectName() const
{
return m_file ? m_file->FileName() : m_objectName;
}
std::unique_ptr<OpenVDS::File> m_file;
std::unique_ptr<OpenVDS::IOManager> m_ioManager;
const std::string m_objectName;
int64_t m_size;
int64_t m_size = 0;
std::string m_lastWriteTime;
};
struct DataView
......
......@@ -24,6 +24,10 @@ public:
{
}
void HandleObjectLastWriteTime(const std::string &lastWriteTimeISO8601) override
{
}
void HandleMetadata(const std::string& key, const std::string& header) override
{
if (key == "vdschunkmetadata")
......
......@@ -24,6 +24,11 @@ public:
void HandleObjectSize(int64_t size) override
{
}
void HandleObjectLastWriteTime(const std::string &lastWriteTimeISO8601) override
{
}
void HandleMetadata(const std::string& key, const std::string& header) override
{
}
......
......@@ -100,6 +100,7 @@ public:
auto &object = (*object_it).second;
error = object.error;
handler->HandleObjectSize(int64_t(object.data.size()));
for (auto& meta : object.metaHeader)
{
handler->HandleMetadata(meta.first, meta.second);
......
......@@ -709,6 +709,52 @@ createSurveyCoordinateSystemMetadata(SEGYFileInfo const& fileInfo, OpenVDS::Meta
metadataContainer.SetMetadataDoubleVector2(LATTICE_CATEGORY, LATTICE_CROSSLINE_SPACING, OpenVDS::DoubleVector2(crosslineSpacing[0], crosslineSpacing[1]));
}
/////////////////////////////////////////////////////////////////////////////
bool
createImportInformationMetadata(DataProvider &dataProvider, OpenVDS::MetadataContainer& metadataContainer, OpenVDS::Error &error)
{
auto now = std::chrono::system_clock::now();
std::time_t tt = std::chrono::system_clock::to_time_t(now);
std::tm tm = *std::gmtime(&tt);
auto duration = now.time_since_epoch();
int millis = std::chrono::duration_cast<std::chrono::milliseconds>(duration).count() % 1000;
std::string importTimeStamp = fmt::format("{:04d}-{:02d}-{:02d}T{:02d}:{:02d}:{:02d}.{:03d}Z", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, millis);
std::string inputFileName = dataProvider.FileOrObjectName();
// Strip the path from the file/object name
const char pathSeparators[] = { '/', '\\', ':' };
for(auto pathSeparator : pathSeparators)
{
size_t pos = inputFileName.rfind(pathSeparator);
if(pos != std::string::npos) inputFileName = inputFileName.substr(pos + 1);
}
// In lack of a better displayName we use the file name
std::string displayName = inputFileName;
std::string inputTimeStamp = dataProvider.LastWriteTime(error);
if (error.code != 0)
{
return false;
}
int64_t inputFileSize = dataProvider.Size(error);
if (error.code != 0)
{
return false;
}
// Set import information
metadataContainer.SetMetadataString(KNOWNMETADATA_CATEGORY_IMPORTINFORMATION, KNOWNMETADATA_IMPORTINFORMATION_DISPLAYNAME, displayName);
metadataContainer.SetMetadataString(KNOWNMETADATA_CATEGORY_IMPORTINFORMATION, KNOWNMETADATA_IMPORTINFORMATION_INPUTFILENAME, inputFileName);
metadataContainer.SetMetadataDouble(KNOWNMETADATA_CATEGORY_IMPORTINFORMATION, KNOWNMETADATA_IMPORTINFORMATION_INPUTFILESIZE, (double)inputFileSize);
metadataContainer.SetMetadataString(KNOWNMETADATA_CATEGORY_IMPORTINFORMATION, KNOWNMETADATA_IMPORTINFORMATION_INPUTTIMESTAMP, inputTimeStamp);
metadataContainer.SetMetadataString(KNOWNMETADATA_CATEGORY_IMPORTINFORMATION, KNOWNMETADATA_IMPORTINFORMATION_IMPORTTIMESTAMP, importTimeStamp);
return true;
}
bool
parseSEGYFileInfoFile(OpenVDS::File const& file, SEGYFileInfo& fileInfo)
{
......@@ -1355,6 +1401,14 @@ main(int argc, char* argv[])
OpenVDS::MetadataContainer
metadataContainer;
createImportInformationMetadata(dataProvider, metadataContainer, error);
if (error.code != 0)
{
std::cerr << error.string;
return EXIT_FAILURE;
}
createSEGYHeadersMetadata(dataProvider, metadataContainer, error);
if (error.code != 0)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment