Bump bundled libarchive version to 3.5.2

- Update bunlded libarchive version used on Windows/Mac
- Enable requested zstd support while we are at it. Closes #211

dependencies/zstd-1.5.0/contrib/pzstd/test/BUCK

@@ -0,0 +1,37 @@
+cxx_test(
+    name='options_test',
+    srcs=['OptionsTest.cpp'],
+    deps=['//contrib/pzstd:options'],
+)
+
+cxx_test(
+    name='pzstd_test',
+    srcs=['PzstdTest.cpp'],
+    deps=[
+        ':round_trip',
+        '//contrib/pzstd:libpzstd',
+        '//contrib/pzstd/utils:scope_guard',
+        '//programs:datagen',
+    ],
+)
+
+cxx_binary(
+    name='round_trip_test',
+    srcs=['RoundTripTest.cpp'],
+    deps=[
+        ':round_trip',
+        '//contrib/pzstd/utils:scope_guard',
+        '//programs:datagen',
+    ]
+)
+
+cxx_library(
+    name='round_trip',
+    header_namespace='test',
+    exported_headers=['RoundTrip.h'],
+    deps=[
+        '//contrib/pzstd:libpzstd',
+        '//contrib/pzstd:options',
+        '//contrib/pzstd/utils:scope_guard',
+    ]
+)

dependencies/zstd-1.5.0/contrib/pzstd/test/OptionsTest.cpp

@@ -0,0 +1,536 @@
+/*
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+#include "Options.h"
+
+#include <array>
+#include <gtest/gtest.h>
+
+using namespace pzstd;
+
+namespace pzstd {
+bool operator==(const Options &lhs, const Options &rhs) {
+  return lhs.numThreads == rhs.numThreads &&
+         lhs.maxWindowLog == rhs.maxWindowLog &&
+         lhs.compressionLevel == rhs.compressionLevel &&
+         lhs.decompress == rhs.decompress && lhs.inputFiles == rhs.inputFiles &&
+         lhs.outputFile == rhs.outputFile && lhs.overwrite == rhs.overwrite &&
+         lhs.keepSource == rhs.keepSource && lhs.writeMode == rhs.writeMode &&
+         lhs.checksum == rhs.checksum && lhs.verbosity == rhs.verbosity;
+}
+
+std::ostream &operator<<(std::ostream &out, const Options &opt) {
+  out << "{";
+  {
+    out << "\n\t"
+        << "numThreads: " << opt.numThreads;
+    out << ",\n\t"
+        << "maxWindowLog: " << opt.maxWindowLog;
+    out << ",\n\t"
+        << "compressionLevel: " << opt.compressionLevel;
+    out << ",\n\t"
+        << "decompress: " << opt.decompress;
+    out << ",\n\t"
+        << "inputFiles: {";
+    {
+      bool first = true;
+      for (const auto &file : opt.inputFiles) {
+        if (!first) {
+          out << ",";
+        }
+        first = false;
+        out << "\n\t\t" << file;
+      }
+    }
+    out << "\n\t}";
+    out << ",\n\t"
+        << "outputFile: " << opt.outputFile;
+    out << ",\n\t"
+        << "overwrite: " << opt.overwrite;
+    out << ",\n\t"
+        << "keepSource: " << opt.keepSource;
+    out << ",\n\t"
+        << "writeMode: " << static_cast<int>(opt.writeMode);
+    out << ",\n\t"
+        << "checksum: " << opt.checksum;
+    out << ",\n\t"
+        << "verbosity: " << opt.verbosity;
+  }
+  out << "\n}";
+  return out;
+}
+}
+
+namespace {
+#ifdef _WIN32
+const char nullOutput[] = "nul";
+#else
+const char nullOutput[] = "/dev/null";
+#endif
+
+constexpr auto autoMode = Options::WriteMode::Auto;
+} // anonymous namespace
+
+#define EXPECT_SUCCESS(...) EXPECT_EQ(Options::Status::Success, __VA_ARGS__)
+#define EXPECT_FAILURE(...) EXPECT_EQ(Options::Status::Failure, __VA_ARGS__)
+#define EXPECT_MESSAGE(...) EXPECT_EQ(Options::Status::Message, __VA_ARGS__)
+
+template <typename... Args>
+std::array<const char *, sizeof...(Args) + 1> makeArray(Args... args) {
+  return {{nullptr, args...}};
+}
+
+TEST(Options, ValidInputs) {
+  {
+    Options options;
+    auto args = makeArray("--processes", "5", "-o", "x", "y", "-f");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    Options expected = {5,    23,   3,        false, {"y"}, "x",
+                        true, true, autoMode, true,  2};
+    EXPECT_EQ(expected, options);
+  }
+  {
+    Options options;
+    auto args = makeArray("-p", "1", "input", "-19");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    Options expected = {1,     23,   19,       false, {"input"}, "",
+                        false, true, autoMode, true,  2};
+    EXPECT_EQ(expected, options);
+  }
+  {
+    Options options;
+    auto args =
+        makeArray("--ultra", "-22", "-p", "1", "-o", "x", "-d", "x.zst", "-f");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    Options expected = {1,    0,    22,       true, {"x.zst"}, "x",
+                        true, true, autoMode, true, 2};
+    EXPECT_EQ(expected, options);
+  }
+  {
+    Options options;
+    auto args = makeArray("--processes", "100", "hello.zst", "--decompress",
+                          "--force");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    Options expected = {100,  23,       3,    true, {"hello.zst"}, "", true,
+                        true, autoMode, true, 2};
+    EXPECT_EQ(expected, options);
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "-dp", "1", "-c");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    Options expected = {1,     23,   3,        true, {"x"}, "-",
+                        false, true, autoMode, true, 2};
+    EXPECT_EQ(expected, options);
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "-dp", "1", "--stdout");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    Options expected = {1,     23,   3,        true, {"x"}, "-",
+                        false, true, autoMode, true, 2};
+    EXPECT_EQ(expected, options);
+  }
+  {
+    Options options;
+    auto args = makeArray("-p", "1", "x", "-5", "-fo", "-", "--ultra", "-d");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    Options expected = {1,    0,    5,        true, {"x"}, "-",
+                        true, true, autoMode, true, 2};
+    EXPECT_EQ(expected, options);
+  }
+  {
+    Options options;
+    auto args = makeArray("silesia.tar", "-o", "silesia.tar.pzstd", "-p", "2");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    Options expected = {2,
+                        23,
+                        3,
+                        false,
+                        {"silesia.tar"},
+                        "silesia.tar.pzstd",
+                        false,
+                        true,
+                        autoMode,
+                        true,
+                        2};
+    EXPECT_EQ(expected, options);
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "-p", "1");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "-p", "1");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+  }
+}
+
+TEST(Options, GetOutputFile) {
+  {
+    Options options;
+    auto args = makeArray("x");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ("x.zst", options.getOutputFile(options.inputFiles[0]));
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "y", "-o", nullOutput);
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(nullOutput, options.getOutputFile(options.inputFiles[0]));
+  }
+  {
+    Options options;
+    auto args = makeArray("x.zst", "-do", nullOutput);
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(nullOutput, options.getOutputFile(options.inputFiles[0]));
+  }
+  {
+    Options options;
+    auto args = makeArray("x.zst", "-d");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ("x", options.getOutputFile(options.inputFiles[0]));
+  }
+  {
+    Options options;
+    auto args = makeArray("xzst", "-d");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ("", options.getOutputFile(options.inputFiles[0]));
+  }
+  {
+    Options options;
+    auto args = makeArray("xzst", "-doxx");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ("xx", options.getOutputFile(options.inputFiles[0]));
+  }
+}
+
+TEST(Options, MultipleFiles) {
+  {
+    Options options;
+    auto args = makeArray("x", "y", "z");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    Options expected;
+    expected.inputFiles = {"x", "y", "z"};
+    expected.verbosity = 1;
+    EXPECT_EQ(expected, options);
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "y", "z", "-o", nullOutput);
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    Options expected;
+    expected.inputFiles = {"x", "y", "z"};
+    expected.outputFile = nullOutput;
+    expected.verbosity = 1;
+    EXPECT_EQ(expected, options);
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "y", "-o-");
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "y", "-o", "file");
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    auto args = makeArray("-qqvd12qp4", "-f", "x", "--", "--rm", "-c");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    Options expected = {4,  23,   12,   true,     {"x", "--rm", "-c"},
+                        "", true, true, autoMode, true,
+                        0};
+    EXPECT_EQ(expected, options);
+  }
+}
+
+TEST(Options, NumThreads) {
+  {
+    Options options;
+    auto args = makeArray("x", "-dfo", "-");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "-p", "0", "-fo", "-");
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    auto args = makeArray("-f", "-p", "-o", "-");
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
+  }
+}
+
+TEST(Options, BadCompressionLevel) {
+  {
+    Options options;
+    auto args = makeArray("x", "-20");
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "--ultra", "-23");
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "--1"); // negative 1?
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
+  }
+}
+
+TEST(Options, InvalidOption) {
+  {
+    Options options;
+    auto args = makeArray("x", "-x");
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
+  }
+}
+
+TEST(Options, BadOutputFile) {
+  {
+    Options options;
+    auto args = makeArray("notzst", "-d", "-p", "1");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ("", options.getOutputFile(options.inputFiles.front()));
+  }
+}
+
+TEST(Options, BadOptionsWithArguments) {
+  {
+    Options options;
+    auto args = makeArray("x", "-pf");
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "-p", "10f");
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "-p");
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "-o");
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "-o");
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
+  }
+}
+
+TEST(Options, KeepSource) {
+  {
+    Options options;
+    auto args = makeArray("x", "--rm", "-k");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(true, options.keepSource);
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "--rm", "--keep");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(true, options.keepSource);
+  }
+  {
+    Options options;
+    auto args = makeArray("x");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(true, options.keepSource);
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "--rm");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(false, options.keepSource);
+  }
+}
+
+TEST(Options, Verbosity) {
+  {
+    Options options;
+    auto args = makeArray("x");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(2, options.verbosity);
+  }
+  {
+    Options options;
+    auto args = makeArray("--quiet", "-qq", "x");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(-1, options.verbosity);
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "y");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(1, options.verbosity);
+  }
+  {
+    Options options;
+    auto args = makeArray("--", "x", "y");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(1, options.verbosity);
+  }
+  {
+    Options options;
+    auto args = makeArray("-qv", "x", "y");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(1, options.verbosity);
+  }
+  {
+    Options options;
+    auto args = makeArray("-v", "x", "y");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(3, options.verbosity);
+  }
+  {
+    Options options;
+    auto args = makeArray("-v", "x");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(3, options.verbosity);
+  }
+}
+
+TEST(Options, TestMode) {
+  {
+    Options options;
+    auto args = makeArray("x", "-t");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(true, options.keepSource);
+    EXPECT_EQ(true, options.decompress);
+    EXPECT_EQ(nullOutput, options.outputFile);
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "--test", "--rm", "-ohello");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(true, options.keepSource);
+    EXPECT_EQ(true, options.decompress);
+    EXPECT_EQ(nullOutput, options.outputFile);
+  }
+}
+
+TEST(Options, Checksum) {
+  {
+    Options options;
+    auto args = makeArray("x.zst", "--no-check", "-Cd");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(true, options.checksum);
+  }
+  {
+    Options options;
+    auto args = makeArray("x");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(true, options.checksum);
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "--no-check", "--check");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(true, options.checksum);
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "--no-check");
+    EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+    EXPECT_EQ(false, options.checksum);
+  }
+}
+
+TEST(Options, InputFiles) {
+  {
+    Options options;
+    auto args = makeArray("-cd");
+    options.parse(args.size(), args.data());
+    EXPECT_EQ(1, options.inputFiles.size());
+    EXPECT_EQ("-", options.inputFiles[0]);
+    EXPECT_EQ("-", options.outputFile);
+  }
+  {
+    Options options;
+    auto args = makeArray();
+    options.parse(args.size(), args.data());
+    EXPECT_EQ(1, options.inputFiles.size());
+    EXPECT_EQ("-", options.inputFiles[0]);
+    EXPECT_EQ("-", options.outputFile);
+  }
+  {
+    Options options;
+    auto args = makeArray("-d");
+    options.parse(args.size(), args.data());
+    EXPECT_EQ(1, options.inputFiles.size());
+    EXPECT_EQ("-", options.inputFiles[0]);
+    EXPECT_EQ("-", options.outputFile);
+  }
+  {
+    Options options;
+    auto args = makeArray("x", "-");
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
+  }
+}
+
+TEST(Options, InvalidOptions) {
+  {
+    Options options;
+    auto args = makeArray("-ibasdf");
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    auto args = makeArray("- ");
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    auto args = makeArray("-n15");
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    auto args = makeArray("-0", "x");
+    EXPECT_FAILURE(options.parse(args.size(), args.data()));
+  }
+}
+
+TEST(Options, Extras) {
+  {
+    Options options;
+    auto args = makeArray("-h");
+    EXPECT_MESSAGE(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    auto args = makeArray("-H");
+    EXPECT_MESSAGE(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    auto args = makeArray("-V");
+    EXPECT_MESSAGE(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    auto args = makeArray("--help");
+    EXPECT_MESSAGE(options.parse(args.size(), args.data()));
+  }
+  {
+    Options options;
+    auto args = makeArray("--version");
+    EXPECT_MESSAGE(options.parse(args.size(), args.data()));
+  }
+}

dependencies/zstd-1.5.0/contrib/pzstd/test/PzstdTest.cpp

@@ -0,0 +1,149 @@
+/*
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+#include "Pzstd.h"
+extern "C" {
+#include "datagen.h"
+}
+#include "test/RoundTrip.h"
+#include "utils/ScopeGuard.h"
+
+#include <cstddef>
+#include <cstdio>
+#include <gtest/gtest.h>
+#include <memory>
+#include <random>
+
+using namespace std;
+using namespace pzstd;
+
+TEST(Pzstd, SmallSizes) {
+  unsigned seed = std::random_device{}();
+  std::fprintf(stderr, "Pzstd.SmallSizes seed: %u\n", seed);
+  std::mt19937 gen(seed);
+
+  for (unsigned len = 1; len < 256; ++len) {
+    if (len % 16 == 0) {
+      std::fprintf(stderr, "%u / 16\n", len / 16);
+    }
+    std::string inputFile = std::tmpnam(nullptr);
+    auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); });
+    {
+      static uint8_t buf[256];
+      RDG_genBuffer(buf, len, 0.5, 0.0, gen());
+      auto fd = std::fopen(inputFile.c_str(), "wb");
+      auto written = std::fwrite(buf, 1, len, fd);
+      std::fclose(fd);
+      ASSERT_EQ(written, len);
+    }
+    for (unsigned numThreads = 1; numThreads <= 2; ++numThreads) {
+      for (unsigned level = 1; level <= 4; level *= 4) {
+        auto errorGuard = makeScopeGuard([&] {
+          std::fprintf(stderr, "# threads: %u\n", numThreads);
+          std::fprintf(stderr, "compression level: %u\n", level);
+        });
+        Options options;
+        options.overwrite = true;
+        options.inputFiles = {inputFile};
+        options.numThreads = numThreads;
+        options.compressionLevel = level;
+        options.verbosity = 1;
+        ASSERT_TRUE(roundTrip(options));
+        errorGuard.dismiss();
+      }
+    }
+  }
+}
+
+TEST(Pzstd, LargeSizes) {
+  unsigned seed = std::random_device{}();
+  std::fprintf(stderr, "Pzstd.LargeSizes seed: %u\n", seed);
+  std::mt19937 gen(seed);
+
+  for (unsigned len = 1 << 20; len <= (1 << 24); len *= 2) {
+    std::string inputFile = std::tmpnam(nullptr);
+    auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); });
+    {
+      std::unique_ptr<uint8_t[]> buf(new uint8_t[len]);
+      RDG_genBuffer(buf.get(), len, 0.5, 0.0, gen());
+      auto fd = std::fopen(inputFile.c_str(), "wb");
+      auto written = std::fwrite(buf.get(), 1, len, fd);
+      std::fclose(fd);
+      ASSERT_EQ(written, len);
+    }
+    for (unsigned numThreads = 1; numThreads <= 16; numThreads *= 4) {
+      for (unsigned level = 1; level <= 4; level *= 4) {
+        auto errorGuard = makeScopeGuard([&] {
+          std::fprintf(stderr, "# threads: %u\n", numThreads);
+          std::fprintf(stderr, "compression level: %u\n", level);
+        });
+        Options options;
+        options.overwrite = true;
+        options.inputFiles = {inputFile};
+        options.numThreads = std::min(numThreads, options.numThreads);
+        options.compressionLevel = level;
+        options.verbosity = 1;
+        ASSERT_TRUE(roundTrip(options));
+        errorGuard.dismiss();
+      }
+    }
+  }
+}
+
+TEST(Pzstd, DISABLED_ExtremelyLargeSize) {
+  unsigned seed = std::random_device{}();
+  std::fprintf(stderr, "Pzstd.ExtremelyLargeSize seed: %u\n", seed);
+  std::mt19937 gen(seed);
+
+  std::string inputFile = std::tmpnam(nullptr);
+  auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); });
+
+  {
+    // Write 4GB + 64 MB
+    constexpr size_t kLength = 1 << 26;
+    std::unique_ptr<uint8_t[]> buf(new uint8_t[kLength]);
+    auto fd = std::fopen(inputFile.c_str(), "wb");
+    auto closeGuard = makeScopeGuard([&] { std::fclose(fd); });
+    for (size_t i = 0; i < (1 << 6) + 1; ++i) {
+      RDG_genBuffer(buf.get(), kLength, 0.5, 0.0, gen());
+      auto written = std::fwrite(buf.get(), 1, kLength, fd);
+      if (written != kLength) {
+        std::fprintf(stderr, "Failed to write file, skipping test\n");
+        return;
+      }
+    }
+  }
+
+  Options options;
+  options.overwrite = true;
+  options.inputFiles = {inputFile};
+  options.compressionLevel = 1;
+  if (options.numThreads == 0) {
+    options.numThreads = 1;
+  }
+  ASSERT_TRUE(roundTrip(options));
+}
+
+TEST(Pzstd, ExtremelyCompressible) {
+  std::string inputFile = std::tmpnam(nullptr);
+  auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); });
+  {
+    std::unique_ptr<uint8_t[]> buf(new uint8_t[10000]);
+    std::memset(buf.get(), 'a', 10000);
+    auto fd = std::fopen(inputFile.c_str(), "wb");
+    auto written = std::fwrite(buf.get(), 1, 10000, fd);
+    std::fclose(fd);
+    ASSERT_EQ(written, 10000);
+  }
+  Options options;
+  options.overwrite = true;
+  options.inputFiles = {inputFile};
+  options.numThreads = 1;
+  options.compressionLevel = 1;
+  ASSERT_TRUE(roundTrip(options));
+}

dependencies/zstd-1.5.0/contrib/pzstd/test/RoundTrip.h

@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+#pragma once
+
+#include "Options.h"
+#include "Pzstd.h"
+#include "utils/ScopeGuard.h"
+
+#include <cstdio>
+#include <string>
+#include <cstdint>
+#include <memory>
+
+namespace pzstd {
+
+inline bool check(std::string source, std::string decompressed) {
+  std::unique_ptr<std::uint8_t[]> sBuf(new std::uint8_t[1024]);
+  std::unique_ptr<std::uint8_t[]> dBuf(new std::uint8_t[1024]);
+
+  auto sFd = std::fopen(source.c_str(), "rb");
+  auto dFd = std::fopen(decompressed.c_str(), "rb");
+  auto guard = makeScopeGuard([&] {
+    std::fclose(sFd);
+    std::fclose(dFd);
+  });
+
+  size_t sRead, dRead;
+
+  do {
+    sRead = std::fread(sBuf.get(), 1, 1024, sFd);
+    dRead = std::fread(dBuf.get(), 1, 1024, dFd);
+    if (std::ferror(sFd) || std::ferror(dFd)) {
+      return false;
+    }
+    if (sRead != dRead) {
+      return false;
+    }
+
+    for (size_t i = 0; i < sRead; ++i) {
+      if (sBuf.get()[i] != dBuf.get()[i]) {
+        return false;
+      }
+    }
+  } while (sRead == 1024);
+  if (!std::feof(sFd) || !std::feof(dFd)) {
+    return false;
+  }
+  return true;
+}
+
+inline bool roundTrip(Options& options) {
+  if (options.inputFiles.size() != 1) {
+    return false;
+  }
+  std::string source = options.inputFiles.front();
+  std::string compressedFile = std::tmpnam(nullptr);
+  std::string decompressedFile = std::tmpnam(nullptr);
+  auto guard = makeScopeGuard([&] {
+    std::remove(compressedFile.c_str());
+    std::remove(decompressedFile.c_str());
+  });
+
+  {
+    options.outputFile = compressedFile;
+    options.decompress = false;
+    if (pzstdMain(options) != 0) {
+      return false;
+    }
+  }
+  {
+    options.decompress = true;
+    options.inputFiles.front() = compressedFile;
+    options.outputFile = decompressedFile;
+    if (pzstdMain(options) != 0) {
+      return false;
+    }
+  }
+  return check(source, decompressedFile);
+}
+}

dependencies/zstd-1.5.0/contrib/pzstd/test/RoundTripTest.cpp

@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+extern "C" {
+#include "datagen.h"
+}
+#include "Options.h"
+#include "test/RoundTrip.h"
+#include "utils/ScopeGuard.h"
+
+#include <cstddef>
+#include <cstdio>
+#include <cstdlib>
+#include <memory>
+#include <random>
+
+using namespace std;
+using namespace pzstd;
+
+namespace {
+string
+writeData(size_t size, double matchProba, double litProba, unsigned seed) {
+  std::unique_ptr<uint8_t[]> buf(new uint8_t[size]);
+  RDG_genBuffer(buf.get(), size, matchProba, litProba, seed);
+  string file = tmpnam(nullptr);
+  auto fd = std::fopen(file.c_str(), "wb");
+  auto guard = makeScopeGuard([&] { std::fclose(fd); });
+  auto bytesWritten = std::fwrite(buf.get(), 1, size, fd);
+  if (bytesWritten != size) {
+    std::abort();
+  }
+  return file;
+}
+
+template <typename Generator>
+string generateInputFile(Generator& gen) {
+  // Use inputs ranging from 1 Byte to 2^16 Bytes
+  std::uniform_int_distribution<size_t> size{1, 1 << 16};
+  std::uniform_real_distribution<> prob{0, 1};
+  return writeData(size(gen), prob(gen), prob(gen), gen());
+}
+
+template <typename Generator>
+Options generateOptions(Generator& gen, const string& inputFile) {
+  Options options;
+  options.inputFiles = {inputFile};
+  options.overwrite = true;
+
+  std::uniform_int_distribution<unsigned> numThreads{1, 32};
+  std::uniform_int_distribution<unsigned> compressionLevel{1, 10};
+
+  options.numThreads = numThreads(gen);
+  options.compressionLevel = compressionLevel(gen);
+
+  return options;
+}
+}
+
+int main() {
+  std::mt19937 gen(std::random_device{}());
+
+  auto newlineGuard = makeScopeGuard([] { std::fprintf(stderr, "\n"); });
+  for (unsigned i = 0; i < 10000; ++i) {
+    if (i % 100 == 0) {
+      std::fprintf(stderr, "Progress: %u%%\r", i / 100);
+    }
+    auto inputFile = generateInputFile(gen);
+    auto inputGuard = makeScopeGuard([&] { std::remove(inputFile.c_str()); });
+    for (unsigned i = 0; i < 10; ++i) {
+      auto options = generateOptions(gen, inputFile);
+      if (!roundTrip(options)) {
+        std::fprintf(stderr, "numThreads: %u\n", options.numThreads);
+        std::fprintf(stderr, "level: %u\n", options.compressionLevel);
+        std::fprintf(stderr, "decompress? %u\n", (unsigned)options.decompress);
+        std::fprintf(stderr, "file: %s\n", inputFile.c_str());
+        return 1;
+      }
+    }
+  }
+  return 0;
+}