Additional unit tests for updating ZGY files on Seismic Store.

native/src/Makefile

@@ -187,8 +187,10 @@ ifneq ($(strip $(ZFP_LIBRARY)),)
 	/bin/cp -a -t $(BIN_DIR) $(strip $(ZFP_LIBRARY))*
 endif
 
+# SD_LIBRARY is only needed because of test/sdutils.cpp doing direct access
+# to SDAPI and bypassing OpenZGY entirely.
 $(BIN_DIR)/test_all: $(TEST_OBJ) $(LIBDSO) $(SD_SENTINEL) $(ZFP_SENTINEL)
-	$(CXX) -o $@ $(CXXFLAGS) $(ORIGIN) $(TEST_OBJ) $(LIBDSO)
+	$(CXX) -o $@ $(CXXFLAGS) $(ORIGIN) $(TEST_OBJ) $(LIBDSO) $(SD_LIBRARY)
 
 #$(BIN_DIR)/zgycopyc: $(OBJ_DIR)/tools/zgycopyc.o $(OBJ_DIR)/test/mock.o $(LIBDSO)
 #	$(CXX) -o $@ $(CXXFLAGS) $(ORIGIN) $^ -fopenmp

native/src/test/test_reopen.cpp

@@ -21,6 +21,7 @@
 
 #include "test_all.h"
 #include "test_utils.h"
+#include "test_sdutils.h"
 #include "../api.h"
 #include "../iocontext.h"
 //#include "../exception.h"
@@ -93,6 +94,15 @@ get_testdata(const std::string& name)
 #endif
 }
 
+static std::string
+get_sdtestdata(const std::string& name)
+{
+  std::string testfolder = InternalZGY::Environment::getStringEnv("OPENZGY_SDTESTDATA", "sd://sntc/testdata");
+  if (testfolder.back() != '/')
+    testfolder += "/";
+  return testfolder + name;
+}
+
 static void
 do_write_twice(const std::string& filename, const IOContext* context = nullptr)
 {
@@ -1821,6 +1831,57 @@ test_reopen_zgypublic()
   }
 }
 
+#ifdef HAVE_SD
+
+/**
+ * Opening an empty file created by the old ZGY accessor and cloud plug-in
+ * is not allowed.
+ */
+void
+test_reopen_zgycloud()
+{
+  Test_Utils::CloudFileAutoDelete cad
+    ("reopen_zgycloud.zgy", Test_Utils::default_sd_context());
+  {
+    // Expected to fail because the header segment is not padded.
+    // Otherwise this file, containing no bricks, would have worked.
+    const std::string oldname = get_sdtestdata("EmptyOldFile.zgy");
+    Test_Utils::copy_sd_to_sd(oldname, cad.name());
+    std::shared_ptr<IZgyWriter> writer;
+    must_throw("Only files uploaded by OpenZGY", [&](){
+      writer = IZgyWriter::reopen(ZgyWriterArgs()
+                                  .filename(cad.name())
+                                  .iocontext(Test_Utils::default_sd_context()));
+    });
+  }
+  {
+    // Expected to fail because the first segment has both header and bulk.
+    const std::string oldname = get_sdtestdata("Synt2.zgy");
+    Test_Utils::copy_sd_to_sd(oldname, cad.name());
+    std::shared_ptr<IZgyWriter> writer;
+    must_throw("Only files uploaded by OpenZGY", [&](){
+      writer = IZgyWriter::reopen(ZgyWriterArgs()
+                                  .filename(cad.name())
+                                  .iocontext(Test_Utils::default_sd_context()));
+    });
+  }
+  {
+    // Expected to fail because only the current version (3/4) can be updated,
+    // and that check should be done first because that error is more clear.
+    // Without the check it would still fail the "uploaded by OpenZGY" test.
+    const std::string oldname = get_sdtestdata("Synt2-v1.zgy");
+    Test_Utils::copy_sd_to_sd(oldname, cad.name());
+    std::shared_ptr<IZgyWriter> writer;
+    must_throw("version is too old for this library", [&](){
+      writer = IZgyWriter::reopen(ZgyWriterArgs()
+                                  .filename(cad.name())
+                                  .iocontext(Test_Utils::default_sd_context()));
+    });
+  }
+}
+
+#endif
+
 static std::vector<std::int16_t>
 make_sequence(std::int64_t start, std::int64_t count, std::int64_t step)
 {
@@ -1851,10 +1912,8 @@ array_to_string(const std::vector<T>& vec, double scale)
  * Extract the segment information from FileStatistics -> xx_segments()
  * as the abbreviated 3-element vector of (first, all_middle, last)
  * and expand that to a list of all the segment sizes.
- * Yes this is way too roundabout.
- * TODO-@@@-Test: For a unit test the code should really have checked
- * each and every segment using SDAPI to see that all except the first
- * and last are the same size.
+ * Yes this is way too roundabout. Also it obviously cannot check
+ * that the segments that are expected to be the same size actually are.
  *
  * Note that if the FileStatistics come from a file opened for write
  * then the result depends on the segsplit setting. Data still in the
@@ -1891,11 +1950,22 @@ list_segments(std::shared_ptr<const OpenZGY::FileStatistics> fs, int verbose)
 static std::vector<std::int64_t>
 list_segments(const std::string& name, int verbose)
 {
+#if 0
   std::shared_ptr<OpenZGY::IZgyReader> reader =
     OpenZGY::IZgyReader::open(name, Test_Utils::default_sd_context());
   std::shared_ptr<const OpenZGY::FileStatistics> fs = reader->filestats();
   reader->close();
   return list_segments(fs, verbose);
+#else
+  // Bypass OpenZGY completely. This will get the actual size of each
+  // segment instead of assuming that all but the first and last have
+  // the same size.
+  std::vector<std::int64_t> result = Test_Utils::get_segsizes(name);
+  if (verbose)
+    std::cout << "segments " << array_to_string(result, 512*1024)
+              << std::endl;
+  return result;
+#endif
 }
 
 static void
@@ -2008,6 +2078,26 @@ test_reopen_sd()
     }
   }
 
+  {
+    // Same as above but more thorough as it bypasses OpenZGY.
+    std::vector<std::int64_t> check =
+      list_segments(cad.name(), verbose());
+    if (TEST_EQUAL(check.size(), (std::size_t)12)) {
+      TEST_EQUAL(check[0], 512*1024);
+      TEST_EQUAL(check[1], 4*512*1024);
+      TEST_EQUAL(check[2], 4*512*1024);
+      TEST_EQUAL(check[3], 4*512*1024);
+      TEST_EQUAL(check[4], 4*512*1024);
+      TEST_EQUAL(check[5], 4*512*1024);
+      TEST_EQUAL(check[6], 4*512*1024);
+      TEST_EQUAL(check[7], 4*512*1024);
+      TEST_EQUAL(check[8], 4*512*1024);
+      TEST_EQUAL(check[9], 4*512*1024);
+      TEST_EQUAL(check[10], 4*512*1024);
+      TEST_EQUAL(check[11], 3*512*1024);
+    }
+  }
+
   std::vector<std::int16_t> check1(11*64);
   std::vector<std::int16_t> check2(11*64);
   reader->read(size3i_t{0,0,0}, size3i_t{1,1,11*64}, check1.data(), 0);
@@ -2051,6 +2141,7 @@ public:
 #ifdef HAVE_SD
     register_test("reopen.plain_sd",         test_reopen_plain_sd);
     register_test("reopen.sd",               test_reopen_sd);
+    register_test("reopen.zgycloud",         test_reopen_zgycloud);
 #endif
   }
 } dummy_reopen;

native/src/test/test_sdutils.cpp

+// Copyright 2017-2021, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Based on Salmon/Shared/TestUtils/TempFileAutoDelete.cpp
+// And wrapper/test_utils.py
+
+#ifdef HAVE_SD // Rest of file
+
+#include "test_sdutils.h"
+#include "../impl/environment.h"
+#define TEST_NO_MAIN
+#define NOMINMAX // cutest includes Windows.h. Ugh!
+#include "cutest.h"
+
+#ifndef _WIN32
+#include <SDManager.h>
+#include <SDGenericDataset.h>
+#else
+#include <SDAPI/SDManager.h>
+#include <SDAPI/SDGenericDataset.h>
+#endif
+
+#include <stdexcept>
+
+namespace Test_Utils {
+#if 0
+}
+#endif
+
+static std::shared_ptr<seismicdrive::SDManager>
+get_manager()
+{
+  using InternalZGY::Environment;
+  const std::string sdurl{Environment::getStringEnv("OPENZGY_SDURL")};
+  const std::string sdapikey{Environment::getStringEnv("OPENZGY_SDAPIKEY")};
+  const std::string sdtoken{Environment::getStringEnv("OPENZGY_TOKEN")};
+  if (sdurl.empty() || sdapikey.empty() || sdtoken.empty())
+    throw std::runtime_error("Missing $OPENZGY_{SDURL,SDAPIKEY,TOKEN}");
+  std::shared_ptr<seismicdrive::SDManager> sdmanager
+    (new seismicdrive::SDManager(sdurl, sdapikey));
+  sdmanager->setAuthProviderFromString(sdtoken);
+  return sdmanager;
+}
+
+/**
+ * Copy a file on seismic store to another file on seismic store.
+ * This uses direct calls to SDAPI only; there is no OpenZGY involved.
+ * To be used for setting up unit test data. Caveat: No legaltag
+ * is provided, so the destination subproject must have a default set.
+ */
+void
+copy_sd_to_sd(const std::string& srcname, const std::string& dstname)
+{
+  std::shared_ptr<seismicdrive::SDManager> manager = get_manager();
+  seismicdrive::SDGenericDataset src(manager.get(), srcname);
+  src.open(seismicdrive::SDDatasetDisposition::READ_ONLY);
+  const std::uint64_t nblocks = src.getBlockNum();
+  std::vector<std::string> names;
+  for (std::uint64_t ii = 0; ii < nblocks; ++ii)
+    names.push_back(std::to_string(ii));
+  const std::vector<long long> sizearray = src.getBlocksSize(names);
+  for (long long segsize : sizearray)
+    if (segsize < 0)
+      throw std::runtime_error("Segment size must be > 0");
+
+  seismicdrive::SDGenericDataset dst(manager.get(), dstname);
+  dst.open(seismicdrive::SDDatasetDisposition::OVERWRITE);
+
+  for (std::uint64_t segnum = 0; segnum < nblocks; ++segnum) {
+    //std::cout << "copy " << segnum << " size " << sizearray[segnum] << "\n";
+    std::unique_ptr<char[]> data(new char[sizearray[segnum]]);
+    src.readBlock((int)segnum, data.get(), 0, (size_t)sizearray[segnum]);
+    dst.writeBlock((int)segnum, data.get(), (size_t)sizearray[segnum]);
+  }
+  dst.close();
+  src.close();
+}
+
+std::vector<std::int64_t>
+get_segsizes(const std::string& name)
+{
+  std::shared_ptr<seismicdrive::SDManager> manager = get_manager();
+  seismicdrive::SDGenericDataset src(manager.get(), name);
+  src.open(seismicdrive::SDDatasetDisposition::READ_ONLY);
+  const std::uint64_t nblocks = src.getBlockNum();
+  std::vector<std::string> names;
+  for (std::uint64_t ii = 0; ii < nblocks; ++ii)
+    names.push_back(std::to_string(ii));
+  const std::vector<long long> sizearray = src.getBlocksSize(names);
+  src.close();
+  std::vector<std::int64_t> result;
+  for (auto it : sizearray)
+    result.push_back(it);
+  return result;
+}
+
+} // namespace Test_Utils
+
+#endif // HAVE_SD

native/src/test/test_sdutils.h

+// Copyright 2017-2021, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <string>
+#include <vector>
+#include <cstdint>
+
+namespace Test_Utils {
+#if 0
+}
+#endif
+
+// Free functions
+
+#ifdef HAVE_SD
+
+void copy_sd_to_sd(const std::string& srcname, const std::string& dstname);
+std::vector<std::int64_t> get_segsizes(const std::string& name);
+
+#endif
+
+} // namespace

native/windows/OpenZGY.Test.vcxproj

@@ -84,6 +84,7 @@
     <ClInclude Include="..\src\test\cutest.h" />
     <ClInclude Include="..\src\test\mock.h" />
     <ClInclude Include="..\src\test\test_all.h" />
+    <ClInclude Include="..\src\test\test_sdutils.h" />
     <ClInclude Include="..\src\test\test_utils.h" />
   </ItemGroup>
   <ItemGroup>
@@ -103,6 +104,7 @@
     <ClCompile Include="..\src\test\test_mtguard.cpp" />
     <ClCompile Include="..\src\test\test_reopen.cpp" />
     <ClCompile Include="..\src\test\test_roundandclip.cpp" />
+    <ClCompile Include="..\src\test\test_sdutils.cpp" />
     <ClCompile Include="..\src\test\test_smallcache.cpp" />
     <ClCompile Include="..\src\test\test_structaccess.cpp" />
     <ClCompile Include="..\src\test\test_timer.cpp" />
@@ -129,4 +131,4 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-</Project>
\ No newline at end of file
+</Project>

native/windows/OpenZGY.Test.vcxproj.filters

@@ -20,6 +20,9 @@
     <ClInclude Include="..\src\test\test_all.h">
       <Filter>src\test</Filter>
     </ClInclude>
+    <ClInclude Include="..\src\test\test_sdutils.h">
+      <Filter>src\test</Filter>
+    </ClInclude>
     <ClInclude Include="..\src\test\test_utils.h">
       <Filter>src\test</Filter>
     </ClInclude>
@@ -73,6 +76,9 @@
     <ClCompile Include="..\src\test\test_roundandclip.cpp">
       <Filter>src\test</Filter>
     </ClCompile>
+    <ClCompile Include="..\src\test\test_sdutils.cpp">
+      <Filter>src\test</Filter>
+    </ClCompile>
     <ClCompile Include="..\src\test\test_smallcache.cpp">
       <Filter>src\test</Filter>
     </ClCompile>