remove LRU cache and revert back to a cache similar to the original design,...

remove LRU cache and revert back to a cache similar to the original design, except that we explicitly tell the cache when to drop completed inlines

src/SEGYUtils/CMakeLists.txt

 set(EXPORTED_HEADER_FILES
+    SEGYUtils/DataProvider.h
     SEGYUtils/SEGY.h
     SEGYUtils/SEGYFileInfo.h
+    SEGYUtils/TraceDataManager.h
 )
 add_library(segyutils
     ${EXPORTED_HEADER_FILES}

src/SEGYUtils/SEGYUtils/DataProvider.h

@@ -291,21 +291,11 @@ struct DataRequestInfo
 class DataViewManager
 {
 public:
-  DataViewManager(DataProvider &dataProvider, int64_t prefetchLimit, std::vector<DataRequestInfo> dataRequestInfo)
+  DataViewManager(DataProvider &dataProvider, int64_t prefetchLimit)
     : m_dataProvider(dataProvider)
     , m_memoryLimit(prefetchLimit)
     , m_usage(0)
   {
-
-    if (m_requests.empty())
-    {
-      m_requests = std::move(dataRequestInfo);
-    }
-    else
-    {
-      m_requests.insert(m_requests.end(), dataRequestInfo.begin(), dataRequestInfo.end());
-    }
-    prefetchUntilMemoryLimit();
   }
 
   std::shared_ptr<DataView> acquireDataView(DataRequestInfo &dataRequestInfo, bool isPopulate, OpenVDS::Error& error)
@@ -321,57 +311,64 @@ public:
     auto it = m_dataViewMap.lower_bound(dataRequestInfo);
     if (it == m_dataViewMap.end() || it->first != dataRequestInfo)
     {
-      auto dataView = new DataView(m_dataProvider, dataRequestInfo.offset, dataRequestInfo.size, isPopulate, error);
-      dataView->ref();
-      it = m_dataViewMap.insert(it, {dataRequestInfo, dataView});
+      // if we've called acquire and it's not already in the map then we need to initiate the request and not just stick it in the queue
+      auto dataView = std::make_shared<DataView>(m_dataProvider, dataRequestInfo.offset, dataRequestInfo.size, true, m_error);
+      it = m_dataViewMap.insert(it, { dataRequestInfo, dataView });
+      m_usage += dataRequestInfo.size;
     }
-    else
+
+    return it->second;
+  }
+
+  void addDataRequests(const std::vector<DataRequestInfo>& dataRequestInfos)
+  {
+    std::unique_lock<std::mutex> lock(m_mutex);
+
+    if (m_error.code == 0)
+    {
+      m_requests.insert(m_requests.end(), dataRequestInfos.begin(), dataRequestInfos.end());
+      prefetchUntilMemoryLimit();
+    }
+  }
+
+  void retireDataViewsBefore(const DataRequestInfo& dataRequestInfo)
+  {
+    std::unique_lock<std::mutex> lock(m_mutex);
+
+    auto it = m_dataViewMap.begin();
+    while (it != m_dataViewMap.end() && it->first < dataRequestInfo)
     {
-      it->second->ref();
+      m_usage -= it->second->Size();
+      it = m_dataViewMap.erase(it);
     }
 
-    auto ptr = std::shared_ptr<DataView>(it->second, [this](DataView* dataView) { if (dataView) this->releaseDataView(dataView); });
-    return ptr;
+    prefetchUntilMemoryLimit();
   }
 
 private:
-  typedef std::map<DataRequestInfo, DataView *> DataViewMap;
+  typedef std::map<DataRequestInfo, std::shared_ptr<DataView>> DataViewMap;
 
   DataProvider &m_dataProvider;
   std::vector<DataRequestInfo> m_requests;
   DataViewMap m_dataViewMap;
   std::mutex m_mutex;
-  int64_t m_memoryLimit;
+  const int64_t m_memoryLimit;
   int64_t m_usage;
   OpenVDS::Error m_error;
 
-  void releaseDataView(DataView *dataView)
-  {
-    std::unique_lock<std::mutex> lock(m_mutex);
-
-    if (dataView->deref())
-    {
-      m_usage -= dataView->Size();
-      auto it = m_dataViewMap.find(DataViewMap::key_type{dataView->Pos(), dataView->Size()});
-      assert(it != m_dataViewMap.end());
-      delete it->second;
-      m_dataViewMap.erase(it);
-      prefetchUntilMemoryLimit();
-    }
-  }
-
   void prefetchUntilMemoryLimit()
   {
     if (m_usage >= m_memoryLimit)
       return;
-    int i;
-    for (i = 0; i < int(m_requests.size()) && m_usage < m_memoryLimit && m_error.code == 0; i++)
+
+    size_t i;
+    for (i = 0; i < m_requests.size() && m_usage < m_memoryLimit && m_error.code == 0; i++)
     {
       auto &req = m_requests[i];
       auto it = m_dataViewMap.find(req);
       if (it != m_dataViewMap.end())
         continue;
-      auto dataView = new DataView(m_dataProvider, req.offset, req.size, true, m_error);
+      auto dataView = std::make_shared<DataView>(m_dataProvider, req.offset, req.size, true, m_error);
       m_dataViewMap.insert(it, {req, dataView});
       m_usage += req.size;
     }

src/SEGYUtils/SEGYUtils/SEGYFileInfo.h

@@ -15,6 +15,9 @@
 ** limitations under the License.
 ****************************************************************************/
 
+#ifndef SEGY_FILE_INFO_H
+#define SEGY_FILE_INFO_H
+
 #include "SEGY.h"
 #include "DataProvider.h"
 
@@ -99,3 +102,5 @@ struct SEGYFileInfo
 
   OPENVDS_EXPORT bool Scan(DataProvider &dataprovider, SEGY::HeaderField const &primaryKeyHeaderField, SEGY::HeaderField const &secondaryKeyHeaderField = SEGY::HeaderField(), SEGYBinInfoHeaderFields const &binInfoHeaderFields = SEGYBinInfoHeaderFields::StandardHeaderFields());
 };
+
+#endif  // SEGY_FILE_INFO_H

src/SEGYUtils/SEGYUtils/TraceDataManager.h

+/****************************************************************************
+** Copyright 2020 The Open Group
+** Copyright 2020 Bluware, Inc.
+**
+** Licensed under the Apache License, Version 2.0 (the "License");
+** you may not use this file except in compliance with the License.
+** You may obtain a copy of the License at
+**
+**     http://www.apache.org/licenses/LICENSE-2.0
+**
+** Unless required by applicable law or agreed to in writing, software
+** distributed under the License is distributed on an "AS IS" BASIS,
+** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+** See the License for the specific language governing permissions and
+** limitations under the License.
+****************************************************************************/
+
+#ifndef TRACE_DATA_MANAGER_H
+#define TRACE_DATA_MANAGER_H
+
+#include "SEGYFileInfo.h"
+
+// Convenience layer that works with a DataViewManager to access trace data
+//
+class TraceDataManager
+{
+public:
+  TraceDataManager(DataViewManager & dvm, int64_t tracesPerPage, int64_t traceByteSize, int64_t numTraces) :
+    m_dataViewManager(dvm),
+    m_tracesPerPage(tracesPerPage),
+    m_traceByteSize(traceByteSize),
+    m_numTraces(numTraces),
+    m_currentPageTrace(-1)
+  {
+  }
+
+  void
+  addDataRequests(int requestSecondaryKeyStart, int requestSecondaryKeyEnd, const std::vector<SEGYSegmentInfo>::iterator& lower, const std::vector<SEGYSegmentInfo>::iterator& upper)
+  {
+    std::vector<DataRequestInfo> requests;
+
+    for (auto segment = lower; segment != upper; ++segment)
+    {
+      // axis factors for guessing trace numbers
+      const auto
+        secondaryKeySpanTotal = static_cast<double>(segment->m_binInfoStop.m_crosslineNumber - segment->m_binInfoStart.m_crosslineNumber);
+      const auto
+        secondaryStartSpan = requestSecondaryKeyStart - segment->m_binInfoStart.m_crosslineNumber,
+        secondaryStopSpan = requestSecondaryKeyEnd - segment->m_binInfoStart.m_crosslineNumber;
+      const double
+        traceStartFactor = secondaryStartSpan <= 0 ? 0.0 : secondaryStartSpan / secondaryKeySpanTotal,
+        traceStopFactor = secondaryStopSpan <= 0 ? 0.0 : secondaryStopSpan / secondaryKeySpanTotal;
+
+      // calculate approximate start and end traces
+      const auto
+        traceCount = segment->m_traceStop - segment->m_traceStart + 1;
+      const auto
+        startTrace = static_cast<int64_t>(segment->m_traceStart + traceCount * traceStartFactor),
+        stopTrace = static_cast<int64_t>(segment->m_traceStart + traceCount * traceStopFactor);
+
+      addTraceRequests(requests, startTrace, stopTrace);
+    }
+
+    m_dataViewManager.addDataRequests(requests);
+  }
+
+  const char *
+  getTraceData(int64_t traceNumber, OpenVDS::Error & error) const
+  {
+    // figure out which "page" is associated with the trace
+    // request the "page" from the data view manager
+    // return trace data pointer within the page
+
+    std::shared_ptr<DataView>
+      pageView;
+
+    error = {};
+
+    const auto
+      pageTrace = GetPageStartForTrace(traceNumber);
+    if (pageTrace == m_currentPageTrace)
+    {
+      pageView = m_currentPageView;
+    }
+    else
+    {
+      auto
+        pageRequestInfo = createPageRequestInfo(pageTrace);
+
+      pageView = m_dataViewManager.acquireDataView(pageRequestInfo, true, error);
+
+      if (error.code)
+      {
+        return nullptr;
+      }
+
+      if (!pageView)
+      {
+        error.code = 1;
+        error.string = "Failed to acquire DataView";
+        return nullptr;
+      }
+    }
+
+    const char
+      * basePtr = static_cast<const char *>(pageView->Pointer(error));
+    return basePtr + (traceNumber - pageTrace) * m_traceByteSize;
+  }
+
+  void
+  retirePagesBefore(int64_t traceNumber) const
+  {
+    const auto
+      pageTrace = GetPageStartForTrace(traceNumber);
+    auto
+      pageRequestInfo = createPageRequestInfo(pageTrace);
+    m_dataViewManager.retireDataViewsBefore(pageRequestInfo);
+  }
+
+private:
+  DataViewManager
+    & m_dataViewManager;
+  const int64_t
+    m_tracesPerPage,
+    m_traceByteSize,
+    m_numTraces;
+  int64_t
+    m_currentPageTrace;
+  std::shared_ptr<DataView>
+    m_currentPageView;
+
+  void
+  addTraceRequests(std::vector<DataRequestInfo>& requests, int64_t startTrace, int64_t stopTrace)
+  {
+    const int64_t
+      pageStartStart = GetPageStartForTrace(startTrace),
+      pageStartStop = GetPageStartForTrace(stopTrace);
+
+    for (auto pageTrace = pageStartStart; pageTrace <= pageStartStop; pageTrace += m_tracesPerPage)
+    {
+      requests.emplace_back(createPageRequestInfo(pageTrace));
+    }
+  }
+
+  DataRequestInfo
+  createPageRequestInfo(int64_t pageTrace) const
+  {
+    const auto
+      traceCount = std::min(m_tracesPerPage, m_numTraces - pageTrace);
+
+    DataRequestInfo
+      dataRequestInfo;
+    dataRequestInfo.offset = SEGY::TextualFileHeaderSize + SEGY::BinaryFileHeaderSize + pageTrace * m_traceByteSize;
+    dataRequestInfo.size = traceCount * m_traceByteSize;
+
+    return dataRequestInfo;
+  }
+
+  int64_t
+  GetPageStartForTrace(int64_t trace) const
+  {
+    return trace - (trace % m_tracesPerPage);
+  }
+
+};
+
+#endif  // TRACE_DATA_MANAGER_H

tools/SEGYImport/SEGYImport.cpp

@@ -20,6 +20,7 @@
 #include "IO/File.h"
 #include "VDS/Hash.h"
 #include <SEGYUtils/DataProvider.h>
+#include <SEGYUtils/TraceDataManager.h>
 
 #include <OpenVDS/OpenVDS.h>
 #include <OpenVDS/KnownMetadata.h>
@@ -925,38 +926,37 @@ createChannelDescriptors(SEGYFileInfo const& fileInfo, OpenVDS::FloatRange const
   return channelDescriptors;
 }
 
-int
-findFirstTrace(int primaryKey, int secondaryKey, SEGYFileInfo const& fileInfo, const void* traceData, int traceCount, int secondaryStart, int secondaryStop)
+int64_t
+findFirstTrace(TraceDataManager& traceDataManager, int primaryKey, int secondaryKey, SEGYFileInfo const& fileInfo, int64_t traceStart, int64_t traceStop, int secondaryStart, int secondaryStop, OpenVDS::Error& error)
 {
-  int traceStart = 0,
-    traceStop = traceCount - 1;
-
-  bool isSecondaryIncreasing = (secondaryStop >= secondaryStart);
+  const bool isSecondaryIncreasing = (secondaryStop >= secondaryStart);
 
   // Check if trace is at the start of the range or completely outside the range (this also handles cases where secondaryStart == secondaryStop which would fail to make a guess below)
   if (isSecondaryIncreasing ? (secondaryKey <= secondaryStart) :
     (secondaryKey >= secondaryStart))
   {
-    return 0;
+    return traceStart;
   }
   else if (isSecondaryIncreasing ? (secondaryKey > secondaryStop) :
     (secondaryKey < secondaryStop))
   {
-    return traceCount;
+    return traceStop + 1;
   }
 
   // Make an initial guess at which trace we start on based on linear interpolation
-  int trace = (int)((long long)(secondaryKey - secondaryStart) * (traceStop - traceStart) / (secondaryStop - secondaryStart));
-
-  assert(trace >= 0 && trace < traceCount && "The guessed trace must be in [traceStart = 0, traceStop = traceCount -1]");
+  int64_t trace = traceStart + (secondaryKey - secondaryStart) * (traceStop - traceStart) / (secondaryStop - secondaryStart);
 
   while (traceStart < traceStop - 1)
   {
-    const char* header = reinterpret_cast<const char*>(intptr_t(traceData) + ptrdiff_t(fileInfo.TraceByteSize()) * trace);
+    const char* header = traceDataManager.getTraceData(trace, error);
+    if (error.code != 0)
+    {
+      return traceStart;
+    }
 
     int primaryTest = SEGY::ReadFieldFromHeader(header, fileInfo.m_primaryKey, fileInfo.m_headerEndianness),
-      secondaryTest = SEGY::ReadFieldFromHeader(header, fileInfo.m_secondaryKey, fileInfo.m_headerEndianness),
-      traceDelta;
+      secondaryTest = SEGY::ReadFieldFromHeader(header, fileInfo.m_secondaryKey, fileInfo.m_headerEndianness);
+    int64_t traceDelta;
 
     if (primaryTest == primaryKey)
     {
@@ -976,9 +976,14 @@ findFirstTrace(int primaryKey, int secondaryKey, SEGYFileInfo const& fileInfo, c
     else
     {
       // We need to handle corrupted traces without hanging, so we scan backwards until we find a valid trace and then we update the interval to not include the corrupted traces
-      for (int scan = trace - 1; scan > traceStart; scan--)
+      for (int64_t scan = trace - 1; scan > traceStart; scan--)
       {
-        header = reinterpret_cast<const char*>(intptr_t(traceData) + ptrdiff_t(fileInfo.TraceByteSize()) * scan);
+        header = traceDataManager.getTraceData(scan, error);
+        if (error.code != 0)
+        {
+          return traceStart;
+        }
+
         primaryTest = SEGY::ReadFieldFromHeader(header, fileInfo.m_primaryKey, fileInfo.m_headerEndianness);
         if (primaryTest == primaryKey)
         {
@@ -1023,19 +1028,14 @@ findFirstTrace(int primaryKey, int secondaryKey, SEGYFileInfo const& fileInfo, c
   return traceStop;
 }
 
-int
-findFirstTrace(int primaryKey, int secondaryKey, SEGYFileInfo const& fileInfo, const void* traceData, int traceCount)
+int64_t
+findFirstTrace(TraceDataManager& traceDataManager, const SEGYSegmentInfo& segment, int secondaryKey, SEGYFileInfo const& fileInfo, OpenVDS::Error error)
 {
-  int traceStart = 0,
-    traceStop = traceCount - 1;
-
-  const char* headerStart = reinterpret_cast<const char*>(intptr_t(traceData) + ptrdiff_t(fileInfo.TraceByteSize()) * traceStart);
-  const char* headerStop = reinterpret_cast<const char*>(intptr_t(traceData) + ptrdiff_t(fileInfo.TraceByteSize()) * traceStop);
-
-  int secondaryStart = SEGY::ReadFieldFromHeader(headerStart, fileInfo.m_secondaryKey, fileInfo.m_headerEndianness);
-  int secondaryStop = SEGY::ReadFieldFromHeader(headerStop, fileInfo.m_secondaryKey, fileInfo.m_headerEndianness);
+  // TODO do the info start and stop reflect the start and stop for reverse-sorted inlines, or are they simply min and max?
+  const int secondaryStart = segment.m_binInfoStart.m_crosslineNumber;
+  const int secondaryStop = segment.m_binInfoStop.m_crosslineNumber;
 
-  return findFirstTrace(primaryKey, secondaryKey, fileInfo, traceData, traceCount, secondaryStart, secondaryStop);
+  return findFirstTrace(traceDataManager, segment.m_primaryKey, secondaryKey, fileInfo, segment.m_traceStart, segment.m_traceStop, secondaryStart, secondaryStop, error);
 }
 
 static DataProvider CreateDataProvider(const std::string &fileName,
@@ -1050,7 +1050,6 @@ static DataProvider CreateDataProvider(const std::string &fileName,
   return CreateDataProviderFromOpenOptions(url, connection, fileName, error);
 }
 
-
 int
 main(int argc, char* argv[])
 {
@@ -1492,11 +1491,18 @@ main(int argc, char* argv[])
     int primaryKeyStop;
     size_t lowerSegmentIndex;
     size_t upperSegmentIndex;
-    int64_t traceStart;
-    int64_t traceStop;
-    DataRequestInfo dataRequestInfo;
+    //int64_t traceStart;
+    //int64_t traceStop;
+    //DataRequestInfo dataRequestInfo;
   };
 
+  // limit DataViewManager's memory use to 1.5 sets of brick inlines
+  const int64_t dvmMemoryLimit = 3LL * brickSize * axisDescriptors[1].GetNumSamples() * fileInfo.TraceByteSize() / 2LL;
+
+  DataViewManager dataViewManager(dataProvider, dvmMemoryLimit);
+
+  TraceDataManager traceDataManager(dataViewManager, 128, traceByteSize, fileInfo.m_traceCount);
+
   std::vector<ChunkInfo> chunkInfos;
   chunkInfos.resize(amplitudeAccessor->GetChunkCount());
   std::vector<DataRequestInfo> dataRequests;
@@ -1520,18 +1526,17 @@ main(int argc, char* argv[])
     auto upper = std::upper_bound(fileInfo.m_segmentInfo.begin(), fileInfo.m_segmentInfo.end(), chunkInfo.primaryKeyStop, [](int primaryKey, SEGYSegmentInfo const& segmentInfo)->bool { return primaryKey < segmentInfo.m_primaryKey; });
     chunkInfo.upperSegmentIndex = std::distance(fileInfo.m_segmentInfo.begin(), upper);
 
-    chunkInfo.traceStart = lower->m_traceStart;
-    chunkInfo.traceStop = std::prev(upper)->m_traceStop;
+    //chunkInfo.traceStart = lower->m_traceStart;
+    //chunkInfo.traceStop = std::prev(upper)->m_traceStop;
 
-    assert(chunkInfo.traceStop > chunkInfo.traceStart);
+    //assert(chunkInfo.traceStop > chunkInfo.traceStart);
 
-    chunkInfo.dataRequestInfo.offset = SEGY::TextualFileHeaderSize + SEGY::BinaryFileHeaderSize + chunkInfo.traceStart * traceByteSize;
-    chunkInfo.dataRequestInfo.size = (chunkInfo.traceStop - chunkInfo.traceStart + 1) * traceByteSize;
-    dataRequests.push_back(chunkInfo.dataRequestInfo);
-  }
+    //chunkInfo.dataRequestInfo.offset = SEGY::TextualFileHeaderSize + SEGY::BinaryFileHeaderSize + chunkInfo.traceStart * traceByteSize;
+    //chunkInfo.dataRequestInfo.size = (chunkInfo.traceStop - chunkInfo.traceStart + 1) * traceByteSize;
+    //dataRequests.push_back(chunkInfo.dataRequestInfo);
 
-  int64_t memoryAvailable = GetTotalSystemMemory();
-  DataViewManager dataViewManager(dataProvider, memoryAvailable / 3, std::move(dataRequests));
+    traceDataManager.addDataRequests(chunkInfo.secondaryKeyStart, chunkInfo.secondaryKeyStop, lower, upper);
+  }
 
   for (int64_t chunk = 0; chunk < amplitudeAccessor->GetChunkCount() && error.code == 0; chunk++)
   {
@@ -1558,95 +1563,99 @@ main(int argc, char* argv[])
 
     auto &chunkInfo = chunkInfos[chunk];
 
-    // This acquires the new file view before releasing the previous so we usually end up re-using the same file view
-    dataView = dataViewManager.acquireDataView(chunkInfo.dataRequestInfo, false, error);
+    // if we've crossed to a new inline then trim the trace page cache
+    if (chunk > 0 && chunkInfo.lowerSegmentIndex > chunkInfos[chunk - 1].lowerSegmentIndex)
+    {
+      // we've progressed to a new set of inlines; remove earlier pages from the cache
+      traceDataManager.retirePagesBefore(fileInfo.m_segmentInfo[chunkInfo.lowerSegmentIndex].m_traceStart);
+    }
+
+    OpenVDS::VolumeDataPage* amplitudePage = amplitudeAccessor->CreatePage(chunk);
+    OpenVDS::VolumeDataPage* traceFlagPage = nullptr;
+    OpenVDS::VolumeDataPage* segyTraceHeaderPage = nullptr;
 
-    if (error.code == 0)
+    if (chunkInfo.min[0] == 0)
     {
-      OpenVDS::VolumeDataPage* amplitudePage = amplitudeAccessor->CreatePage(chunk);
-      OpenVDS::VolumeDataPage* traceFlagPage = nullptr;
-      OpenVDS::VolumeDataPage* segyTraceHeaderPage = nullptr;
+      traceFlagPage = traceFlagAccessor->CreatePage(traceFlagAccessor->GetChunkIndex(chunkInfo.min));
+      segyTraceHeaderPage = segyTraceHeaderAccessor->CreatePage(segyTraceHeaderAccessor->GetChunkIndex(chunkInfo.min));
+    }
 
-      if (chunkInfo.min[0] == 0)
-      {
-        traceFlagPage = traceFlagAccessor->CreatePage(traceFlagAccessor->GetChunkIndex(chunkInfo.min));
-        segyTraceHeaderPage = segyTraceHeaderAccessor->CreatePage(segyTraceHeaderAccessor->GetChunkIndex(chunkInfo.min));
-      }
+    int amplitudePitch[OpenVDS::Dimensionality_Max];
+    int traceFlagPitch[OpenVDS::Dimensionality_Max];
+    int segyTraceHeaderPitch[OpenVDS::Dimensionality_Max];
 
-      int amplitudePitch[OpenVDS::Dimensionality_Max];
-      int traceFlagPitch[OpenVDS::Dimensionality_Max];
-      int segyTraceHeaderPitch[OpenVDS::Dimensionality_Max];
+    void* amplitudeBuffer = amplitudePage->GetWritableBuffer(amplitudePitch);
+    void* traceFlagBuffer = traceFlagPage ? traceFlagPage->GetWritableBuffer(traceFlagPitch) : nullptr;
+    void* segyTraceHeaderBuffer = segyTraceHeaderPage ? segyTraceHeaderPage->GetWritableBuffer(segyTraceHeaderPitch) : nullptr;
 
-      void* amplitudeBuffer = amplitudePage->GetWritableBuffer(amplitudePitch);
-      void* traceFlagBuffer = traceFlagPage ? traceFlagPage->GetWritableBuffer(traceFlagPitch) : nullptr;
-      void* segyTraceHeaderBuffer = segyTraceHeaderPage ? segyTraceHeaderPage->GetWritableBuffer(segyTraceHeaderPitch) : nullptr;
+    assert(amplitudePitch[0] == 1);
+    assert(!traceFlagBuffer || traceFlagPitch[1] == 1);
+    assert(!segyTraceHeaderBuffer || segyTraceHeaderPitch[1] == SEGY::TraceHeaderSize);
 
-      assert(amplitudePitch[0] == 1);
-      assert(!traceFlagBuffer || traceFlagPitch[1] == 1);
-      assert(!segyTraceHeaderBuffer || segyTraceHeaderPitch[1] == SEGY::TraceHeaderSize);
+    // We loop through the segments that have primary keys inside this block and copy the traces that have secondary keys inside this block
+    auto lower = fileInfo.m_segmentInfo.begin() + chunkInfo.lowerSegmentIndex;
+    auto upper = fileInfo.m_segmentInfo.begin() + chunkInfo.upperSegmentIndex;
 
-      // We loop through the segments that have primary keys inside this block and copy the traces that have secondary keys inside this block
-      auto lower = fileInfo.m_segmentInfo.begin() + chunkInfo.lowerSegmentIndex;
-      auto upper = fileInfo.m_segmentInfo.begin() + chunkInfo.upperSegmentIndex;
+    for (auto segment = lower; segment != upper; ++segment)
+    {
+      int64_t firstTrace = findFirstTrace(traceDataManager, *segment, chunkInfo.secondaryKeyStart, fileInfo, error);
+      if (error.code)
+      {
+        fmt::print(stderr, "Failed when reading data: {} - {}", error.code, error.string);
+        break;
+      }
 
-      for (auto segment = lower; segment != upper; ++segment)
+      for (int64_t trace = firstTrace; trace <= segment->m_traceStop; trace++)
       {
-        const void* traceData = reinterpret_cast<const void*>(intptr_t(dataView->Pointer(error)) + (segment->m_traceStart - chunkInfo.traceStart) * traceByteSize);
+        const char* header = traceDataManager.getTraceData(trace, error);
         if (error.code)
         {
           fmt::print(stderr, "Failed when reading data: {} - {}\n", error.code, error.string);
           break;
         }
-        int traceCount = int(segment->m_traceStop - segment->m_traceStart + 1);
 
-        int firstTrace = findFirstTrace(segment->m_primaryKey, chunkInfo.secondaryKeyStart, fileInfo, traceData, traceCount);
+        const void* data = header + SEGY::TraceHeaderSize;
 
-        for (int trace = firstTrace; trace < traceCount; trace++)
-        {
-          const char* header = reinterpret_cast<const char*>(intptr_t(traceData) + traceByteSize * trace);
-          const void* data = header + SEGY::TraceHeaderSize;
-
-          int primaryTest = SEGY::ReadFieldFromHeader(header, fileInfo.m_primaryKey, fileInfo.m_headerEndianness),
-            secondaryTest = SEGY::ReadFieldFromHeader(header, fileInfo.m_secondaryKey, fileInfo.m_headerEndianness);
+        int primaryTest = SEGY::ReadFieldFromHeader(header, fileInfo.m_primaryKey, fileInfo.m_headerEndianness),
+          secondaryTest = SEGY::ReadFieldFromHeader(header, fileInfo.m_secondaryKey, fileInfo.m_headerEndianness);
 
-          // Check if the trace is outside the secondary range and go to the next segment if it is
-          if (primaryTest == segment->m_primaryKey && secondaryTest > chunkInfo.secondaryKeyStop)
-          {
-            break;
-          }
+        // Check if the trace is outside the secondary range and go to the next segment if it is
+        if (primaryTest == segment->m_primaryKey && secondaryTest > chunkInfo.secondaryKeyStop)
+        {
+          break;
+        }
 
-          int primaryIndex = layout->GetAxisDescriptor(2).CoordinateToSampleIndex((float)segment->m_primaryKey);
-          int secondaryIndex = layout->GetAxisDescriptor(1).CoordinateToSampleIndex((float)secondaryTest);
+        int primaryIndex = layout->GetAxisDescriptor(2).CoordinateToSampleIndex((float)segment->m_primaryKey);
+        int secondaryIndex = layout->GetAxisDescriptor(1).CoordinateToSampleIndex((float)secondaryTest);
 
-          assert(primaryIndex >= chunkInfo.min[2] && primaryIndex < chunkInfo.max[2]);
-          assert(secondaryIndex >= chunkInfo.min[1] && secondaryIndex < chunkInfo.max[1]);