Bump bundled libarchive version to 3.5.2

- Update bunlded libarchive version used on Windows/Mac
- Enable requested zstd support while we are at it. Closes #211

dependencies/zstd-1.5.0/examples/.gitignore

@@ -0,0 +1,15 @@
+#build
+simple_compression
+simple_decompression
+multiple_simple_compression
+dictionary_compression
+dictionary_decompression
+streaming_compression
+streaming_decompression
+multiple_streaming_compression
+streaming_memory_usage
+
+#test artefact
+tmp*
+test*
+*.zst

dependencies/zstd-1.5.0/examples/Makefile

@@ -0,0 +1,93 @@
+# ################################################################
+# Copyright (c) Yann Collet, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
+# ################################################################
+
+LIBDIR =../lib
+CPPFLAGS += -I$(LIBDIR)
+LIB = $(LIBDIR)/libzstd.a
+
+
+.PHONY: default
+default: all
+
+.PHONY: all
+all: simple_compression simple_decompression \
+	multiple_simple_compression\
+	dictionary_compression dictionary_decompression \
+	streaming_compression streaming_decompression \
+	multiple_streaming_compression streaming_memory_usage
+
+$(LIB) :
+	$(MAKE) -C $(LIBDIR) libzstd.a
+
+simple_compression.o: common.h
+simple_compression : $(LIB)
+
+simple_decompression.o: common.h
+simple_decompression : $(LIB)
+
+multiple_simple_compression.o: common.h
+multiple_simple_compression : $(LIB)
+
+dictionary_compression.o: common.h
+dictionary_compression : $(LIB)
+
+dictionary_decompression.o: common.h
+dictionary_decompression : $(LIB)
+
+streaming_compression.o: common.h
+streaming_compression : $(LIB)
+
+multiple_streaming_compression.o: common.h
+multiple_streaming_compression : $(LIB)
+
+streaming_decompression.o: common.h
+streaming_decompression : $(LIB)
+
+streaming_memory_usage.o: common.h
+streaming_memory_usage : $(LIB)
+
+
+.PHONY:clean
+clean:
+	@$(RM) core *.o tmp* result* *.zst \
+        simple_compression simple_decompression \
+        multiple_simple_compression \
+        dictionary_compression dictionary_decompression \
+        streaming_compression streaming_decompression \
+        multiple_streaming_compression streaming_memory_usage
+	@echo Cleaning completed
+
+.PHONY:test
+test: all
+	cp README.md tmp
+	cp Makefile tmp2
+	@echo -- Simple compression tests
+	./simple_compression tmp
+	./simple_decompression tmp.zst
+	./multiple_simple_compression *.c
+	./streaming_decompression tmp.zst > /dev/null
+	@echo -- Streaming memory usage
+	./streaming_memory_usage
+	@echo -- Streaming compression tests
+	./streaming_compression tmp
+	./streaming_decompression tmp.zst > /dev/null
+	@echo -- Edge cases detection
+	! ./streaming_decompression tmp    # invalid input, must fail
+	! ./simple_decompression tmp       # invalid input, must fail
+	touch tmpNull                      # create 0-size file
+	./simple_compression tmpNull
+	./simple_decompression tmpNull.zst # 0-size frame : must work
+	@echo -- Multiple streaming tests
+	./multiple_streaming_compression *.c
+	@echo -- Dictionary compression tests
+	./dictionary_compression tmp2 tmp README.md
+	./dictionary_decompression tmp2.zst tmp.zst README.md
+	$(RM) tmp* *.zst
+	@echo tests completed

dependencies/zstd-1.5.0/examples/README.md

@@ -0,0 +1,46 @@
+Zstandard library : usage examples
+==================================
+
+- [Simple compression](simple_compression.c) :
+  Compress a single file.
+  Introduces usage of : `ZSTD_compress()`
+
+- [Simple decompression](simple_decompression.c) :
+  Decompress a single file.
+  Only compatible with simple compression.
+  Result remains in memory.
+  Introduces usage of : `ZSTD_decompress()`
+
+- [Multiple simple compression](multiple_simple_compression.c) :
+  Compress multiple files (in simple mode) in a single command line.
+  Demonstrates memory preservation technique that
+  minimizes malloc()/free() calls by re-using existing resources.
+  Introduces usage of : `ZSTD_compressCCtx()`
+
+- [Streaming memory usage](streaming_memory_usage.c) :
+  Provides amount of memory used by streaming context.
+  Introduces usage of : `ZSTD_sizeof_CStream()`
+
+- [Streaming compression](streaming_compression.c) :
+  Compress a single file.
+  Introduces usage of : `ZSTD_compressStream()`
+
+- [Multiple Streaming compression](multiple_streaming_compression.c) :
+  Compress multiple files (in streaming mode) in a single command line.
+  Introduces memory usage preservation technique,
+  reducing impact of malloc()/free() and memset() by re-using existing resources.
+
+- [Streaming decompression](streaming_decompression.c) :
+  Decompress a single file compressed by zstd.
+  Compatible with both simple and streaming compression.
+  Result is sent to stdout.
+  Introduces usage of : `ZSTD_decompressStream()`
+
+- [Dictionary compression](dictionary_compression.c) :
+  Compress multiple files using the same dictionary.
+  Introduces usage of : `ZSTD_createCDict()` and `ZSTD_compress_usingCDict()`
+
+- [Dictionary decompression](dictionary_decompression.c) :
+  Decompress multiple files using the same dictionary.
+  Result remains in memory.
+  Introduces usage of : `ZSTD_createDDict()` and `ZSTD_decompress_usingDDict()`

dependencies/zstd-1.5.0/examples/common.h

@@ -0,0 +1,234 @@
+/*
+ * Copyright (c) Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/*
+ * This header file has common utility functions used in examples.
+ */
+#ifndef COMMON_H
+#define COMMON_H
+
+#include <stdlib.h>    // malloc, free, exit
+#include <stdio.h>     // fprintf, perror, fopen, etc.
+#include <string.h>    // strerror
+#include <errno.h>     // errno
+#include <sys/stat.h>  // stat
+#include <zstd.h>
+
+/*
+ * Define the returned error code from utility functions.
+ */
+typedef enum {
+    ERROR_fsize = 1,
+    ERROR_fopen = 2,
+    ERROR_fclose = 3,
+    ERROR_fread = 4,
+    ERROR_fwrite = 5,
+    ERROR_loadFile = 6,
+    ERROR_saveFile = 7,
+    ERROR_malloc = 8,
+    ERROR_largeFile = 9,
+} COMMON_ErrorCode;
+
+/*! CHECK
+ * Check that the condition holds. If it doesn't print a message and die.
+ */
+#define CHECK(cond, ...)                        \
+    do {                                        \
+        if (!(cond)) {                          \
+            fprintf(stderr,                     \
+                    "%s:%d CHECK(%s) failed: ", \
+                    __FILE__,                   \
+                    __LINE__,                   \
+                    #cond);                     \
+            fprintf(stderr, "" __VA_ARGS__);    \
+            fprintf(stderr, "\n");              \
+            exit(1);                            \
+        }                                       \
+    } while (0)
+
+/*! CHECK_ZSTD
+ * Check the zstd error code and die if an error occurred after printing a
+ * message.
+ */
+#define CHECK_ZSTD(fn, ...)                                      \
+    do {                                                         \
+        size_t const err = (fn);                                 \
+        CHECK(!ZSTD_isError(err), "%s", ZSTD_getErrorName(err)); \
+    } while (0)
+
+/*! fsize_orDie() :
+ * Get the size of a given file path.
+ *
+ * @return The size of a given file path.
+ */
+static size_t fsize_orDie(const char *filename)
+{
+    struct stat st;
+    if (stat(filename, &st) != 0) {
+        /* error */
+        perror(filename);
+        exit(ERROR_fsize);
+    }
+
+    off_t const fileSize = st.st_size;
+    size_t const size = (size_t)fileSize;
+    /* 1. fileSize should be non-negative,
+     * 2. if off_t -> size_t type conversion results in discrepancy,
+     *    the file size is too large for type size_t.
+     */
+    if ((fileSize < 0) || (fileSize != (off_t)size)) {
+        fprintf(stderr, "%s : filesize too large \n", filename);
+        exit(ERROR_largeFile);
+    }
+    return size;
+}
+
+/*! fopen_orDie() :
+ * Open a file using given file path and open option.
+ *
+ * @return If successful this function will return a FILE pointer to an
+ * opened file otherwise it sends an error to stderr and exits.
+ */
+static FILE* fopen_orDie(const char *filename, const char *instruction)
+{
+    FILE* const inFile = fopen(filename, instruction);
+    if (inFile) return inFile;
+    /* error */
+    perror(filename);
+    exit(ERROR_fopen);
+}
+
+/*! fclose_orDie() :
+ * Close an opened file using given FILE pointer.
+ */
+static void fclose_orDie(FILE* file)
+{
+    if (!fclose(file)) { return; };
+    /* error */
+    perror("fclose");
+    exit(ERROR_fclose);
+}
+
+/*! fread_orDie() :
+ *
+ * Read sizeToRead bytes from a given file, storing them at the
+ * location given by buffer.
+ *
+ * @return The number of bytes read.
+ */
+static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file)
+{
+    size_t const readSize = fread(buffer, 1, sizeToRead, file);
+    if (readSize == sizeToRead) return readSize;   /* good */
+    if (feof(file)) return readSize;   /* good, reached end of file */
+    /* error */
+    perror("fread");
+    exit(ERROR_fread);
+}
+
+/*! fwrite_orDie() :
+ *
+ * Write sizeToWrite bytes to a file pointed to by file, obtaining
+ * them from a location given by buffer.
+ *
+ * Note: This function will send an error to stderr and exit if it
+ * cannot write data to the given file pointer.
+ *
+ * @return The number of bytes written.
+ */
+static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file)
+{
+    size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file);
+    if (writtenSize == sizeToWrite) return sizeToWrite;   /* good */
+    /* error */
+    perror("fwrite");
+    exit(ERROR_fwrite);
+}
+
+/*! malloc_orDie() :
+ * Allocate memory.
+ *
+ * @return If successful this function returns a pointer to allo-
+ * cated memory.  If there is an error, this function will send that
+ * error to stderr and exit.
+ */
+static void* malloc_orDie(size_t size)
+{
+    void* const buff = malloc(size);
+    if (buff) return buff;
+    /* error */
+    perror("malloc");
+    exit(ERROR_malloc);
+}
+
+/*! loadFile_orDie() :
+ * load file into buffer (memory).
+ *
+ * Note: This function will send an error to stderr and exit if it
+ * cannot read data from the given file path.
+ *
+ * @return If successful this function will load file into buffer and
+ * return file size, otherwise it will printout an error to stderr and exit.
+ */
+static size_t loadFile_orDie(const char* fileName, void* buffer, size_t bufferSize)
+{
+    size_t const fileSize = fsize_orDie(fileName);
+    CHECK(fileSize <= bufferSize, "File too large!");
+
+    FILE* const inFile = fopen_orDie(fileName, "rb");
+    size_t const readSize = fread(buffer, 1, fileSize, inFile);
+    if (readSize != (size_t)fileSize) {
+        fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno));
+        exit(ERROR_fread);
+    }
+    fclose(inFile);  /* can't fail, read only */
+    return fileSize;
+}
+
+/*! mallocAndLoadFile_orDie() :
+ * allocate memory buffer and then load file into it.
+ *
+ * Note: This function will send an error to stderr and exit if memory allocation
+ * fails or it cannot read data from the given file path.
+ *
+ * @return If successful this function will return buffer and bufferSize(=fileSize),
+ * otherwise it will printout an error to stderr and exit.
+ */
+static void* mallocAndLoadFile_orDie(const char* fileName, size_t* bufferSize) {
+    size_t const fileSize = fsize_orDie(fileName);
+    *bufferSize = fileSize;
+    void* const buffer = malloc_orDie(*bufferSize);
+    loadFile_orDie(fileName, buffer, *bufferSize);
+    return buffer;
+}
+
+/*! saveFile_orDie() :
+ *
+ * Save buffSize bytes to a given file path, obtaining them from a location pointed
+ * to by buff.
+ *
+ * Note: This function will send an error to stderr and exit if it
+ * cannot write to a given file.
+ */
+static void saveFile_orDie(const char* fileName, const void* buff, size_t buffSize)
+{
+    FILE* const oFile = fopen_orDie(fileName, "wb");
+    size_t const wSize = fwrite(buff, 1, buffSize, oFile);
+    if (wSize != (size_t)buffSize) {
+        fprintf(stderr, "fwrite: %s : %s \n", fileName, strerror(errno));
+        exit(ERROR_fwrite);
+    }
+    if (fclose(oFile)) {
+        perror(fileName);
+        exit(ERROR_fclose);
+    }
+}
+
+#endif

dependencies/zstd-1.5.0/examples/dictionary_compression.c

@@ -0,0 +1,97 @@
+/*
+ * Copyright (c) Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+#include <stdio.h>     // printf
+#include <stdlib.h>    // free
+#include <string.h>    // memset, strcat
+#include <zstd.h>      // presumes zstd library is installed
+#include "common.h"    // Helper functions, CHECK(), and CHECK_ZSTD()
+
+/* createDict() :
+   `dictFileName` is supposed to have been created using `zstd --train` */
+static ZSTD_CDict* createCDict_orDie(const char* dictFileName, int cLevel)
+{
+    size_t dictSize;
+    printf("loading dictionary %s \n", dictFileName);
+    void* const dictBuffer = mallocAndLoadFile_orDie(dictFileName, &dictSize);
+    ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, cLevel);
+    CHECK(cdict != NULL, "ZSTD_createCDict() failed!");
+    free(dictBuffer);
+    return cdict;
+}
+
+
+static void compress(const char* fname, const char* oname, const ZSTD_CDict* cdict)
+{
+    size_t fSize;
+    void* const fBuff = mallocAndLoadFile_orDie(fname, &fSize);
+    size_t const cBuffSize = ZSTD_compressBound(fSize);
+    void* const cBuff = malloc_orDie(cBuffSize);
+
+    /* Compress using the dictionary.
+     * This function writes the dictionary id, and content size into the header.
+     * But, it doesn't use a checksum. You can control these options using the
+     * advanced API: ZSTD_CCtx_setParameter(), ZSTD_CCtx_refCDict(),
+     * and ZSTD_compress2().
+     */
+    ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+    CHECK(cctx != NULL, "ZSTD_createCCtx() failed!");
+    size_t const cSize = ZSTD_compress_usingCDict(cctx, cBuff, cBuffSize, fBuff, fSize, cdict);
+    CHECK_ZSTD(cSize);
+
+    saveFile_orDie(oname, cBuff, cSize);
+
+    /* success */
+    printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname);
+
+    ZSTD_freeCCtx(cctx);   /* never fails */
+    free(fBuff);
+    free(cBuff);
+}
+
+
+static char* createOutFilename_orDie(const char* filename)
+{
+    size_t const inL = strlen(filename);
+    size_t const outL = inL + 5;
+    void* outSpace = malloc_orDie(outL);
+    memset(outSpace, 0, outL);
+    strcat(outSpace, filename);
+    strcat(outSpace, ".zst");
+    return (char*)outSpace;
+}
+
+int main(int argc, const char** argv)
+{
+    const char* const exeName = argv[0];
+    int const cLevel = 3;
+
+    if (argc<3) {
+        fprintf(stderr, "wrong arguments\n");
+        fprintf(stderr, "usage:\n");
+        fprintf(stderr, "%s [FILES] dictionary\n", exeName);
+        return 1;
+    }
+
+    /* load dictionary only once */
+    const char* const dictName = argv[argc-1];
+    ZSTD_CDict* const dictPtr = createCDict_orDie(dictName, cLevel);
+
+    int u;
+    for (u=1; u<argc-1; u++) {
+        const char* inFilename = argv[u];
+        char* const outFilename = createOutFilename_orDie(inFilename);
+        compress(inFilename, outFilename, dictPtr);
+        free(outFilename);
+    }
+
+    ZSTD_freeCDict(dictPtr);
+    printf("All %u files compressed. \n", argc-2);
+    return 0;
+}

dependencies/zstd-1.5.0/examples/dictionary_decompression.c

@@ -0,0 +1,99 @@
+/*
+ * Copyright (c) Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+#include <stdio.h>     // printf
+#include <stdlib.h>    // free
+#include <zstd.h>      // presumes zstd library is installed
+#include "common.h"    // Helper functions, CHECK(), and CHECK_ZSTD()
+
+/* createDict() :
+   `dictFileName` is supposed to have been created using `zstd --train` */
+static ZSTD_DDict* createDict_orDie(const char* dictFileName)
+{
+    size_t dictSize;
+    printf("loading dictionary %s \n", dictFileName);
+    void* const dictBuffer = mallocAndLoadFile_orDie(dictFileName, &dictSize);
+    ZSTD_DDict* const ddict = ZSTD_createDDict(dictBuffer, dictSize);
+    CHECK(ddict != NULL, "ZSTD_createDDict() failed!");
+    free(dictBuffer);
+    return ddict;
+}
+
+static void decompress(const char* fname, const ZSTD_DDict* ddict)
+{
+    size_t cSize;
+    void* const cBuff = mallocAndLoadFile_orDie(fname, &cSize);
+    /* Read the content size from the frame header. For simplicity we require
+     * that it is always present. By default, zstd will write the content size
+     * in the header when it is known. If you can't guarantee that the frame
+     * content size is always written into the header, either use streaming
+     * decompression, or ZSTD_decompressBound().
+     */
+    unsigned long long const rSize = ZSTD_getFrameContentSize(cBuff, cSize);
+    CHECK(rSize != ZSTD_CONTENTSIZE_ERROR, "%s: not compressed by zstd!", fname);
+    CHECK(rSize != ZSTD_CONTENTSIZE_UNKNOWN, "%s: original size unknown!", fname);
+    void* const rBuff = malloc_orDie((size_t)rSize);
+
+    /* Check that the dictionary ID matches.
+     * If a non-zstd dictionary is used, then both will be zero.
+     * By default zstd always writes the dictionary ID into the frame.
+     * Zstd will check if there is a dictionary ID mismatch as well.
+     */
+    unsigned const expectedDictID = ZSTD_getDictID_fromDDict(ddict);
+    unsigned const actualDictID = ZSTD_getDictID_fromFrame(cBuff, cSize);
+    CHECK(actualDictID == expectedDictID,
+          "DictID mismatch: expected %u got %u",
+          expectedDictID,
+          actualDictID);
+
+    /* Decompress using the dictionary.
+     * If you need to control the decompression parameters, then use the
+     * advanced API: ZSTD_DCtx_setParameter(), ZSTD_DCtx_refDDict(), and
+     * ZSTD_decompressDCtx().
+     */
+    ZSTD_DCtx* const dctx = ZSTD_createDCtx();
+    CHECK(dctx != NULL, "ZSTD_createDCtx() failed!");
+    size_t const dSize = ZSTD_decompress_usingDDict(dctx, rBuff, rSize, cBuff, cSize, ddict);
+    CHECK_ZSTD(dSize);
+    /* When zstd knows the content size, it will error if it doesn't match. */
+    CHECK(dSize == rSize, "Impossible because zstd will check this condition!");
+
+    /* success */
+    printf("%25s : %6u -> %7u \n", fname, (unsigned)cSize, (unsigned)rSize);
+
+    ZSTD_freeDCtx(dctx);
+    free(rBuff);
+    free(cBuff);
+}
+
+
+int main(int argc, const char** argv)
+{
+    const char* const exeName = argv[0];
+
+    if (argc<3) {
+        printf("wrong arguments\n");
+        printf("usage:\n");
+        printf("%s [FILES] dictionary\n", exeName);
+        return 1;
+    }
+
+    /* load dictionary only once */
+    const char* const dictName = argv[argc-1];
+    ZSTD_DDict* const dictPtr = createDict_orDie(dictName);
+
+    int u;
+    for (u=1; u<argc-1; u++) decompress(argv[u], dictPtr);
+
+    ZSTD_freeDDict(dictPtr);
+    printf("All %u files correctly decoded (in memory) \n", argc-2);
+    return 0;
+}

dependencies/zstd-1.5.0/examples/multiple_simple_compression.c

@@ -0,0 +1,116 @@
+/*
+ * Copyright (c) Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include <stdio.h>     // printf
+#include <stdlib.h>    // free
+#include <string.h>    // memcpy, strlen
+#include <zstd.h>      // presumes zstd library is installed
+#include "common.h"    // Helper functions, CHECK(), and CHECK_ZSTD()
+
+typedef struct {
+    void* fBuffer;
+    void* cBuffer;
+    size_t fBufferSize;
+    size_t cBufferSize;
+    ZSTD_CCtx* cctx;
+} resources;
+
+/*
+ * allocate memory for buffers big enough to compress all files
+ * as well as memory for output file name (ofn)
+ */
+static resources createResources_orDie(int argc, const char** argv, char **ofn, size_t* ofnBufferLen)
+{
+    size_t maxFilenameLength=0;
+    size_t maxFileSize = 0;
+
+    int argNb;
+    for (argNb = 1; argNb < argc; argNb++) {
+      const char* const filename = argv[argNb];
+      size_t const filenameLength = strlen(filename);
+      size_t const fileSize = fsize_orDie(filename);
+
+      if (filenameLength > maxFilenameLength) maxFilenameLength = filenameLength;
+      if (fileSize > maxFileSize) maxFileSize = fileSize;
+    }
+
+    resources ress;
+    ress.fBufferSize = maxFileSize;
+    ress.cBufferSize = ZSTD_compressBound(maxFileSize);
+
+    *ofnBufferLen = maxFilenameLength + 5;
+    *ofn = (char*)malloc_orDie(*ofnBufferLen);
+    ress.fBuffer = malloc_orDie(ress.fBufferSize);
+    ress.cBuffer = malloc_orDie(ress.cBufferSize);
+    ress.cctx = ZSTD_createCCtx();
+    CHECK(ress.cctx != NULL, "ZSTD_createCCtx() failed!");
+    return ress;
+}
+
+static void freeResources(resources ress, char *outFilename)
+{
+    free(ress.fBuffer);
+    free(ress.cBuffer);
+    ZSTD_freeCCtx(ress.cctx);   /* never fails */
+    free(outFilename);
+}
+
+/* compress with pre-allocated context (ZSTD_CCtx) and input/output buffers*/
+static void compressFile_orDie(resources ress, const char* fname, const char* oname)
+{
+    size_t fSize = loadFile_orDie(fname, ress.fBuffer, ress.fBufferSize);
+
+    /* Compress using the context.
+     * If you need more control over parameters, use the advanced API:
+     * ZSTD_CCtx_setParameter(), and ZSTD_compress2().
+     */
+    size_t const cSize = ZSTD_compressCCtx(ress.cctx, ress.cBuffer, ress.cBufferSize, ress.fBuffer, fSize, 1);
+    CHECK_ZSTD(cSize);
+
+    saveFile_orDie(oname, ress.cBuffer, cSize);
+
+    /* success */
+    printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname);
+}
+
+int main(int argc, const char** argv)
+{
+    const char* const exeName = argv[0];
+
+    if (argc<2) {
+        printf("wrong arguments\n");
+        printf("usage:\n");
+        printf("%s FILE(s)\n", exeName);
+        return 1;
+    }
+
+    /* memory allocation for outFilename and resources */
+    char* outFilename;
+    size_t outFilenameBufferLen;
+    resources const ress = createResources_orDie(argc, argv, &outFilename, &outFilenameBufferLen);
+
+    /* compress files with shared context, input and output buffers */
+    int argNb;
+    for (argNb = 1; argNb < argc; argNb++) {
+        const char* const inFilename = argv[argNb];
+        size_t const inFilenameLen = strlen(inFilename);
+        CHECK(inFilenameLen + 5 <= outFilenameBufferLen, "File name too long!");
+        memcpy(outFilename, inFilename, inFilenameLen);
+        memcpy(outFilename+inFilenameLen, ".zst", 5);
+        compressFile_orDie(ress, inFilename, outFilename);
+    }
+
+    /* free memory */
+    freeResources(ress,outFilename);
+
+    printf("compressed %i files \n", argc-1);
+
+    return 0;
+}

dependencies/zstd-1.5.0/examples/multiple_streaming_compression.c

@@ -0,0 +1,133 @@
+/*
+ * Copyright (c) Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+/* The objective of this example is to show of to compress multiple successive files
+*  while preserving memory management.
+*  All structures and buffers will be created only once,
+*  and shared across all compression operations */
+
+#include <stdio.h>     // printf
+#include <stdlib.h>    // free
+#include <string.h>    // memset, strcat
+#include <zstd.h>      // presumes zstd library is installed
+#include "common.h"    // Helper functions, CHECK(), and CHECK_ZSTD()
+
+typedef struct {
+    void* buffIn;
+    void* buffOut;
+    size_t buffInSize;
+    size_t buffOutSize;
+    ZSTD_CCtx* cctx;
+} resources;
+
+static resources createResources_orDie(int cLevel)
+{
+    resources ress;
+    ress.buffInSize = ZSTD_CStreamInSize();   /* can always read one full block */
+    ress.buffOutSize= ZSTD_CStreamOutSize();  /* can always flush a full block */
+    ress.buffIn = malloc_orDie(ress.buffInSize);
+    ress.buffOut= malloc_orDie(ress.buffOutSize);
+    ress.cctx = ZSTD_createCCtx();
+    CHECK(ress.cctx != NULL, "ZSTD_createCCtx() failed!");
+
+    /* Set any compression parameters you want here.
+     * They will persist for every compression operation.
+     * Here we set the compression level, and enable the checksum.
+     */
+    CHECK_ZSTD( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, cLevel) );
+    CHECK_ZSTD( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_checksumFlag, 1) );
+    return ress;
+}
+
+static void freeResources(resources ress)
+{
+    ZSTD_freeCCtx(ress.cctx);
+    free(ress.buffIn);
+    free(ress.buffOut);
+}
+
+static void compressFile_orDie(resources ress, const char* fname, const char* outName)
+{
+    // Open the input and output files.
+    FILE* const fin  = fopen_orDie(fname, "rb");
+    FILE* const fout = fopen_orDie(outName, "wb");
+
+    /* Reset the context to a clean state to start a new compression operation.
+     * The parameters are sticky, so we keep the compression level and extra
+     * parameters that we set in createResources_orDie().
+     */
+    CHECK_ZSTD( ZSTD_CCtx_reset(ress.cctx, ZSTD_reset_session_only) );
+
+    size_t const toRead = ress.buffInSize;
+    size_t read;
+    while ( (read = fread_orDie(ress.buffIn, toRead, fin)) ) {
+        /* This loop is the same as streaming_compression.c.
+         * See that file for detailed comments.
+         */
+        int const lastChunk = (read < toRead);
+        ZSTD_EndDirective const mode = lastChunk ? ZSTD_e_end : ZSTD_e_continue;
+
+        ZSTD_inBuffer input = { ress.buffIn, read, 0 };
+        int finished;
+        do {
+            ZSTD_outBuffer output = { ress.buffOut, ress.buffOutSize, 0 };
+            size_t const remaining = ZSTD_compressStream2(ress.cctx, &output, &input, mode);
+            CHECK_ZSTD(remaining);
+            fwrite_orDie(ress.buffOut, output.pos, fout);
+            finished = lastChunk ? (remaining == 0) : (input.pos == input.size);
+        } while (!finished);
+        CHECK(input.pos == input.size,
+              "Impossible: zstd only returns 0 when the input is completely consumed!");
+    }
+
+    fclose_orDie(fout);
+    fclose_orDie(fin);
+}
+
+int main(int argc, const char** argv)
+{
+    const char* const exeName = argv[0];
+
+    if (argc<2) {
+        printf("wrong arguments\n");
+        printf("usage:\n");
+        printf("%s FILE(s)\n", exeName);
+        return 1;
+    }
+
+    int const cLevel = 7;
+    resources const ress = createResources_orDie(cLevel);
+    void* ofnBuffer = NULL;
+    size_t ofnbSize = 0;
+
+    int argNb;
+    for (argNb = 1; argNb < argc; argNb++) {
+        const char* const ifn = argv[argNb];
+        size_t const ifnSize = strlen(ifn);
+        size_t const ofnSize = ifnSize + 5;
+        if (ofnbSize <= ofnSize) {
+            ofnbSize = ofnSize + 16;
+            free(ofnBuffer);
+            ofnBuffer = malloc_orDie(ofnbSize);
+        }
+        memset(ofnBuffer, 0, ofnSize);
+        strcat(ofnBuffer, ifn);
+        strcat(ofnBuffer, ".zst");
+        compressFile_orDie(ress, ifn, ofnBuffer);
+    }
+
+    freeResources(ress);
+    free(ofnBuffer);
+
+    printf("compressed %i files \n", argc-1);
+
+    return 0;
+}

dependencies/zstd-1.5.0/examples/simple_compression.c

@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include <stdio.h>     // printf
+#include <stdlib.h>    // free
+#include <string.h>    // strlen, strcat, memset
+#include <zstd.h>      // presumes zstd library is installed
+#include "common.h"    // Helper functions, CHECK(), and CHECK_ZSTD()
+
+static void compress_orDie(const char* fname, const char* oname)
+{
+    size_t fSize;
+    void* const fBuff = mallocAndLoadFile_orDie(fname, &fSize);
+    size_t const cBuffSize = ZSTD_compressBound(fSize);
+    void* const cBuff = malloc_orDie(cBuffSize);
+
+    /* Compress.
+     * If you are doing many compressions, you may want to reuse the context.
+     * See the multiple_simple_compression.c example.
+     */
+    size_t const cSize = ZSTD_compress(cBuff, cBuffSize, fBuff, fSize, 1);
+    CHECK_ZSTD(cSize);
+
+    saveFile_orDie(oname, cBuff, cSize);
+
+    /* success */
+    printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname);
+
+    free(fBuff);
+    free(cBuff);
+}
+
+static char* createOutFilename_orDie(const char* filename)
+{
+    size_t const inL = strlen(filename);
+    size_t const outL = inL + 5;
+    void* const outSpace = malloc_orDie(outL);
+    memset(outSpace, 0, outL);
+    strcat(outSpace, filename);
+    strcat(outSpace, ".zst");
+    return (char*)outSpace;
+}
+
+int main(int argc, const char** argv)
+{
+    const char* const exeName = argv[0];
+
+    if (argc!=2) {
+        printf("wrong arguments\n");
+        printf("usage:\n");
+        printf("%s FILE\n", exeName);
+        return 1;
+    }
+
+    const char* const inFilename = argv[1];
+
+    char* const outFilename = createOutFilename_orDie(inFilename);
+    compress_orDie(inFilename, outFilename);
+    free(outFilename);
+    return 0;
+}

dependencies/zstd-1.5.0/examples/simple_decompression.c

@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include <stdio.h>     // printf
+#include <stdlib.h>    // free
+#include <zstd.h>      // presumes zstd library is installed
+#include "common.h"    // Helper functions, CHECK(), and CHECK_ZSTD()
+
+static void decompress(const char* fname)
+{
+    size_t cSize;
+    void* const cBuff = mallocAndLoadFile_orDie(fname, &cSize);
+    /* Read the content size from the frame header. For simplicity we require
+     * that it is always present. By default, zstd will write the content size
+     * in the header when it is known. If you can't guarantee that the frame
+     * content size is always written into the header, either use streaming
+     * decompression, or ZSTD_decompressBound().
+     */
+    unsigned long long const rSize = ZSTD_getFrameContentSize(cBuff, cSize);
+    CHECK(rSize != ZSTD_CONTENTSIZE_ERROR, "%s: not compressed by zstd!", fname);
+    CHECK(rSize != ZSTD_CONTENTSIZE_UNKNOWN, "%s: original size unknown!", fname);
+
+    void* const rBuff = malloc_orDie((size_t)rSize);
+
+    /* Decompress.
+     * If you are doing many decompressions, you may want to reuse the context
+     * and use ZSTD_decompressDCtx(). If you want to set advanced parameters,
+     * use ZSTD_DCtx_setParameter().
+     */
+    size_t const dSize = ZSTD_decompress(rBuff, rSize, cBuff, cSize);
+    CHECK_ZSTD(dSize);
+    /* When zstd knows the content size, it will error if it doesn't match. */
+    CHECK(dSize == rSize, "Impossible because zstd will check this condition!");
+
+    /* success */
+    printf("%25s : %6u -> %7u \n", fname, (unsigned)cSize, (unsigned)rSize);
+
+    free(rBuff);
+    free(cBuff);
+}
+
+int main(int argc, const char** argv)
+{
+    const char* const exeName = argv[0];
+
+    if (argc!=2) {
+        printf("wrong arguments\n");
+        printf("usage:\n");
+        printf("%s FILE\n", exeName);
+        return 1;
+    }
+
+    decompress(argv[1]);
+
+    printf("%s correctly decoded (in memory). \n", argv[1]);
+
+    return 0;
+}

dependencies/zstd-1.5.0/examples/streaming_compression.c

@@ -0,0 +1,124 @@
+/*
+ * Copyright (c) Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+#include <stdio.h>     // printf
+#include <stdlib.h>    // free
+#include <string.h>    // memset, strcat, strlen
+#include <zstd.h>      // presumes zstd library is installed
+#include "common.h"    // Helper functions, CHECK(), and CHECK_ZSTD()
+
+
+static void compressFile_orDie(const char* fname, const char* outName, int cLevel)
+{
+    /* Open the input and output files. */
+    FILE* const fin  = fopen_orDie(fname, "rb");
+    FILE* const fout = fopen_orDie(outName, "wb");
+    /* Create the input and output buffers.
+     * They may be any size, but we recommend using these functions to size them.
+     * Performance will only suffer significantly for very tiny buffers.
+     */
+    size_t const buffInSize = ZSTD_CStreamInSize();
+    void*  const buffIn  = malloc_orDie(buffInSize);
+    size_t const buffOutSize = ZSTD_CStreamOutSize();
+    void*  const buffOut = malloc_orDie(buffOutSize);
+
+    /* Create the context. */
+    ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+    CHECK(cctx != NULL, "ZSTD_createCCtx() failed!");
+
+    /* Set any parameters you want.
+     * Here we set the compression level, and enable the checksum.
+     */
+    CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel) );
+    CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) );
+    ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 4);
+
+    /* This loop read from the input file, compresses that entire chunk,
+     * and writes all output produced to the output file.
+     */
+    size_t const toRead = buffInSize;
+    for (;;) {
+        size_t read = fread_orDie(buffIn, toRead, fin);
+        /* Select the flush mode.
+         * If the read may not be finished (read == toRead) we use
+         * ZSTD_e_continue. If this is the last chunk, we use ZSTD_e_end.
+         * Zstd optimizes the case where the first flush mode is ZSTD_e_end,
+         * since it knows it is compressing the entire source in one pass.
+         */
+        int const lastChunk = (read < toRead);
+        ZSTD_EndDirective const mode = lastChunk ? ZSTD_e_end : ZSTD_e_continue;
+        /* Set the input buffer to what we just read.
+         * We compress until the input buffer is empty, each time flushing the
+         * output.
+         */
+        ZSTD_inBuffer input = { buffIn, read, 0 };
+        int finished;
+        do {
+            /* Compress into the output buffer and write all of the output to
+             * the file so we can reuse the buffer next iteration.
+             */
+            ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
+            size_t const remaining = ZSTD_compressStream2(cctx, &output , &input, mode);
+            CHECK_ZSTD(remaining);
+            fwrite_orDie(buffOut, output.pos, fout);
+            /* If we're on the last chunk we're finished when zstd returns 0,
+             * which means its consumed all the input AND finished the frame.
+             * Otherwise, we're finished when we've consumed all the input.
+             */
+            finished = lastChunk ? (remaining == 0) : (input.pos == input.size);
+        } while (!finished);
+        CHECK(input.pos == input.size,
+              "Impossible: zstd only returns 0 when the input is completely consumed!");
+
+        if (lastChunk) {
+            break;
+        }
+    }
+
+    ZSTD_freeCCtx(cctx);
+    fclose_orDie(fout);
+    fclose_orDie(fin);
+    free(buffIn);
+    free(buffOut);
+}
+
+
+static char* createOutFilename_orDie(const char* filename)
+{
+    size_t const inL = strlen(filename);
+    size_t const outL = inL + 5;
+    void* const outSpace = malloc_orDie(outL);
+    memset(outSpace, 0, outL);
+    strcat(outSpace, filename);
+    strcat(outSpace, ".zst");
+    return (char*)outSpace;
+}
+
+int main(int argc, const char** argv)
+{
+    const char* const exeName = argv[0];
+
+    if (argc!=2) {
+        printf("wrong arguments\n");
+        printf("usage:\n");
+        printf("%s FILE\n", exeName);
+        return 1;
+    }
+
+    const char* const inFilename = argv[1];
+
+    char* const outFilename = createOutFilename_orDie(inFilename);
+    compressFile_orDie(inFilename, outFilename, 1);
+
+    free(outFilename);   /* not strictly required, since program execution stops there,
+                          * but some static analyzer main complain otherwise */
+    return 0;
+}

dependencies/zstd-1.5.0/examples/streaming_compression_thread_pool.c

@@ -0,0 +1,178 @@
+/*
+ * Copyright (c) Martin Liska, SUSE, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+#include <stdio.h>     // printf
+#include <stdlib.h>    // free
+#include <string.h>    // memset, strcat, strlen
+#include <zstd.h>      // presumes zstd library is installed
+#include "common.h"    // Helper functions, CHECK(), and CHECK_ZSTD()
+#include <pthread.h>
+
+typedef struct compress_args
+{
+  const char *fname;
+  char *outName;
+  int cLevel;
+#if defined(ZSTD_STATIC_LINKING_ONLY)
+  ZSTD_threadPool *pool;
+#endif
+} compress_args_t;
+
+static void *compressFile_orDie(void *data)
+{
+    compress_args_t *args = (compress_args_t *)data;
+    fprintf (stderr, "Starting compression of %s with level %d\n", args->fname, args->cLevel);
+    /* Open the input and output files. */
+    FILE* const fin  = fopen_orDie(args->fname, "rb");
+    FILE* const fout = fopen_orDie(args->outName, "wb");
+    /* Create the input and output buffers.
+     * They may be any size, but we recommend using these functions to size them.
+     * Performance will only suffer significantly for very tiny buffers.
+     */
+    size_t const buffInSize = ZSTD_CStreamInSize();
+    void*  const buffIn  = malloc_orDie(buffInSize);
+    size_t const buffOutSize = ZSTD_CStreamOutSize();
+    void*  const buffOut = malloc_orDie(buffOutSize);
+
+    /* Create the context. */
+    ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+    CHECK(cctx != NULL, "ZSTD_createCCtx() failed!");
+
+#if defined(ZSTD_STATIC_LINKING_ONLY)
+    size_t r = ZSTD_CCtx_refThreadPool(cctx, args->pool);
+    CHECK(r == 0, "ZSTD_CCtx_refThreadPool failed!");
+#endif
+
+    /* Set any parameters you want.
+     * Here we set the compression level, and enable the checksum.
+     */
+    CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, args->cLevel) );
+    CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) );
+    ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 16);
+
+    /* This loop read from the input file, compresses that entire chunk,
+     * and writes all output produced to the output file.
+     */
+    size_t const toRead = buffInSize;
+    for (;;) {
+        size_t read = fread_orDie(buffIn, toRead, fin);
+        /* Select the flush mode.
+         * If the read may not be finished (read == toRead) we use
+         * ZSTD_e_continue. If this is the last chunk, we use ZSTD_e_end.
+         * Zstd optimizes the case where the first flush mode is ZSTD_e_end,
+         * since it knows it is compressing the entire source in one pass.
+         */
+        int const lastChunk = (read < toRead);
+        ZSTD_EndDirective const mode = lastChunk ? ZSTD_e_end : ZSTD_e_continue;
+        /* Set the input buffer to what we just read.
+         * We compress until the input buffer is empty, each time flushing the
+         * output.
+         */
+        ZSTD_inBuffer input = { buffIn, read, 0 };
+        int finished;
+        do {
+            /* Compress into the output buffer and write all of the output to
+             * the file so we can reuse the buffer next iteration.
+             */
+            ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
+            size_t const remaining = ZSTD_compressStream2(cctx, &output , &input, mode);
+            CHECK_ZSTD(remaining);
+            fwrite_orDie(buffOut, output.pos, fout);
+            /* If we're on the last chunk we're finished when zstd returns 0,
+             * which means its consumed all the input AND finished the frame.
+             * Otherwise, we're finished when we've consumed all the input.
+             */
+            finished = lastChunk ? (remaining == 0) : (input.pos == input.size);
+        } while (!finished);
+        CHECK(input.pos == input.size,
+              "Impossible: zstd only returns 0 when the input is completely consumed!");
+
+        if (lastChunk) {
+            break;
+        }
+    }
+
+    fprintf (stderr, "Finishing compression of %s\n", args->outName);
+
+    ZSTD_freeCCtx(cctx);
+    fclose_orDie(fout);
+    fclose_orDie(fin);
+    free(buffIn);
+    free(buffOut);
+    free(args->outName);
+
+    return NULL;
+}
+
+
+static char* createOutFilename_orDie(const char* filename)
+{
+    size_t const inL = strlen(filename);
+    size_t const outL = inL + 5;
+    void* const outSpace = malloc_orDie(outL);
+    memset(outSpace, 0, outL);
+    strcat(outSpace, filename);
+    strcat(outSpace, ".zst");
+    return (char*)outSpace;
+}
+
+int main(int argc, const char** argv)
+{
+    const char* const exeName = argv[0];
+
+    if (argc<=3) {
+        printf("wrong arguments\n");
+        printf("usage:\n");
+        printf("%s POOL_SIZE LEVEL FILES\n", exeName);
+        return 1;
+    }
+
+    int pool_size = atoi (argv[1]);
+    CHECK(pool_size != 0, "can't parse POOL_SIZE!");
+
+    int level = atoi (argv[2]);
+    CHECK(level != 0, "can't parse LEVEL!");
+
+    argc -= 3;
+    argv += 3;
+
+#if defined(ZSTD_STATIC_LINKING_ONLY)
+    ZSTD_threadPool *pool = ZSTD_createThreadPool (pool_size);
+    CHECK(pool != NULL, "ZSTD_createThreadPool() failed!");
+    fprintf (stderr, "Using shared thread pool of size %d\n", pool_size);
+#else
+    fprintf (stderr, "All threads use its own thread pool\n");
+#endif
+
+    pthread_t *threads = malloc_orDie(argc * sizeof(pthread_t));
+    compress_args_t *args = malloc_orDie(argc * sizeof(compress_args_t));
+
+    for (unsigned i = 0; i < argc; i++)
+    {
+      args[i].fname = argv[i];
+      args[i].outName = createOutFilename_orDie(args[i].fname);
+      args[i].cLevel = level;
+#if defined(ZSTD_STATIC_LINKING_ONLY)
+      args[i].pool = pool;
+#endif
+
+      pthread_create (&threads[i], NULL, compressFile_orDie, &args[i]);
+    }
+
+    for (unsigned i = 0; i < argc; i++)
+      pthread_join (threads[i], NULL);
+
+#if defined(ZSTD_STATIC_LINKING_ONLY)
+    ZSTD_freeThreadPool (pool);
+#endif
+
+    return 0;
+}

dependencies/zstd-1.5.0/examples/streaming_decompression.c

@@ -0,0 +1,100 @@
+/*
+ * Copyright (c) Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+#include <stdio.h>     // fprintf
+#include <stdlib.h>    // free
+#include <zstd.h>      // presumes zstd library is installed
+#include "common.h"    // Helper functions, CHECK(), and CHECK_ZSTD()
+
+static void decompressFile_orDie(const char* fname)
+{
+    FILE* const fin  = fopen_orDie(fname, "rb");
+    size_t const buffInSize = ZSTD_DStreamInSize();
+    void*  const buffIn  = malloc_orDie(buffInSize);
+    FILE* const fout = stdout;
+    size_t const buffOutSize = ZSTD_DStreamOutSize();  /* Guarantee to successfully flush at least one complete compressed block in all circumstances. */
+    void*  const buffOut = malloc_orDie(buffOutSize);
+
+    ZSTD_DCtx* const dctx = ZSTD_createDCtx();
+    CHECK(dctx != NULL, "ZSTD_createDCtx() failed!");
+
+    /* This loop assumes that the input file is one or more concatenated zstd
+     * streams. This example won't work if there is trailing non-zstd data at
+     * the end, but streaming decompression in general handles this case.
+     * ZSTD_decompressStream() returns 0 exactly when the frame is completed,
+     * and doesn't consume input after the frame.
+     */
+    size_t const toRead = buffInSize;
+    size_t read;
+    size_t lastRet = 0;
+    int isEmpty = 1;
+    while ( (read = fread_orDie(buffIn, toRead, fin)) ) {
+        isEmpty = 0;
+        ZSTD_inBuffer input = { buffIn, read, 0 };
+        /* Given a valid frame, zstd won't consume the last byte of the frame
+         * until it has flushed all of the decompressed data of the frame.
+         * Therefore, instead of checking if the return code is 0, we can
+         * decompress just check if input.pos < input.size.
+         */
+        while (input.pos < input.size) {
+            ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
+            /* The return code is zero if the frame is complete, but there may
+             * be multiple frames concatenated together. Zstd will automatically
+             * reset the context when a frame is complete. Still, calling
+             * ZSTD_DCtx_reset() can be useful to reset the context to a clean
+             * state, for instance if the last decompression call returned an
+             * error.
+             */
+            size_t const ret = ZSTD_decompressStream(dctx, &output , &input);
+            CHECK_ZSTD(ret);
+            fwrite_orDie(buffOut, output.pos, fout);
+            lastRet = ret;
+        }
+    }
+
+    if (isEmpty) {
+        fprintf(stderr, "input is empty\n");
+        exit(1);
+    }
+
+    if (lastRet != 0) {
+        /* The last return value from ZSTD_decompressStream did not end on a
+         * frame, but we reached the end of the file! We assume this is an
+         * error, and the input was truncated.
+         */
+        fprintf(stderr, "EOF before end of stream: %zu\n", lastRet);
+        exit(1);
+    }
+
+    ZSTD_freeDCtx(dctx);
+    fclose_orDie(fin);
+    fclose_orDie(fout);
+    free(buffIn);
+    free(buffOut);
+}
+
+
+int main(int argc, const char** argv)
+{
+    const char* const exeName = argv[0];
+
+    if (argc!=2) {
+        fprintf(stderr, "wrong arguments\n");
+        fprintf(stderr, "usage:\n");
+        fprintf(stderr, "%s FILE\n", exeName);
+        return 1;
+    }
+
+    const char* const inFilename = argv[1];
+
+    decompressFile_orDie(inFilename);
+    return 0;
+}

dependencies/zstd-1.5.0/examples/streaming_memory_usage.c

@@ -0,0 +1,137 @@
+/*
+ * Copyright (c) Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+/*===   Tuning parameter   ===*/
+#ifndef MAX_TESTED_LEVEL
+#define MAX_TESTED_LEVEL 12
+#endif
+
+
+/*===   Dependencies   ===*/
+#include <stdio.h>     // printf
+#define ZSTD_STATIC_LINKING_ONLY
+#include <zstd.h>      // presumes zstd library is installed
+#include "common.h"    // Helper functions, CHECK(), and CHECK_ZSTD()
+
+
+/*===   functions   ===*/
+
+/*! readU32FromChar() :
+    @return : unsigned integer value read from input in `char` format
+    allows and interprets K, KB, KiB, M, MB and MiB suffix.
+    Will also modify `*stringPtr`, advancing it to position where it stopped reading.
+    Note : function result can overflow if digit string > MAX_UINT */
+static unsigned readU32FromChar(const char** stringPtr)
+{
+    unsigned result = 0;
+    while ((**stringPtr >='0') && (**stringPtr <='9'))
+        result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
+    if ((**stringPtr=='K') || (**stringPtr=='M')) {
+        result <<= 10;
+        if (**stringPtr=='M') result <<= 10;
+        (*stringPtr)++ ;
+        if (**stringPtr=='i') (*stringPtr)++;
+        if (**stringPtr=='B') (*stringPtr)++;
+    }
+    return result;
+}
+
+
+int main(int argc, char const *argv[]) {
+
+    printf("\n Zstandard (v%s) memory usage for streaming : \n\n", ZSTD_versionString());
+
+    unsigned wLog = 0;
+    if (argc > 1) {
+        const char* valStr = argv[1];
+        wLog = readU32FromChar(&valStr);
+    }
+
+    int compressionLevel;
+    for (compressionLevel = 1; compressionLevel <= MAX_TESTED_LEVEL; compressionLevel++) {
+#define INPUT_SIZE 5
+#define COMPRESSED_SIZE 128
+        char const dataToCompress[INPUT_SIZE] = "abcde";
+        char compressedData[COMPRESSED_SIZE];
+        char decompressedData[INPUT_SIZE];
+        /* the ZSTD_CCtx_params structure is a way to save parameters and use
+         * them across multiple contexts. We use them here so we can call the
+         * function ZSTD_estimateCStreamSize_usingCCtxParams().
+         */
+        ZSTD_CCtx_params* const cctxParams = ZSTD_createCCtxParams();
+        CHECK(cctxParams != NULL, "ZSTD_createCCtxParams() failed!");
+
+        /* Set the compression level. */
+        CHECK_ZSTD( ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_compressionLevel, compressionLevel) );
+        /* Set the window log.
+         * The value 0 means use the default window log, which is equivalent to
+         * not setting it.
+         */
+        CHECK_ZSTD( ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, wLog) );
+
+        /* Force the compressor to allocate the maximum memory size for a given
+         * level by not providing the pledged source size, or calling
+         * ZSTD_compressStream2() with ZSTD_e_end.
+         */
+        ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+        CHECK(cctx != NULL, "ZSTD_createCCtx() failed!");
+        CHECK_ZSTD( ZSTD_CCtx_setParametersUsingCCtxParams(cctx, cctxParams) );
+        size_t compressedSize;
+        {
+            ZSTD_inBuffer inBuff = { dataToCompress, sizeof(dataToCompress), 0 };
+            ZSTD_outBuffer outBuff = { compressedData, sizeof(compressedData), 0 };
+            CHECK_ZSTD( ZSTD_compressStream(cctx, &outBuff, &inBuff) );
+            size_t const remaining = ZSTD_endStream(cctx, &outBuff);
+            CHECK_ZSTD(remaining);
+            CHECK(remaining == 0, "Frame not flushed!");
+            compressedSize = outBuff.pos;
+        }
+
+        ZSTD_DCtx* const dctx = ZSTD_createDCtx();
+        CHECK(dctx != NULL, "ZSTD_createDCtx() failed!");
+        /* Set the maximum allowed window log.
+         * The value 0 means use the default window log, which is equivalent to
+         * not setting it.
+         */
+        CHECK_ZSTD( ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, wLog) );
+        /* forces decompressor to use maximum memory size, since the
+         * decompressed size is not stored in the frame header.
+         */
+        {   ZSTD_inBuffer inBuff = { compressedData, compressedSize, 0 };
+            ZSTD_outBuffer outBuff = { decompressedData, sizeof(decompressedData), 0 };
+            size_t const remaining = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
+            CHECK_ZSTD(remaining);
+            CHECK(remaining == 0, "Frame not complete!");
+            CHECK(outBuff.pos == sizeof(dataToCompress), "Bad decompression!");
+        }
+
+        size_t const cstreamSize = ZSTD_sizeof_CStream(cctx);
+        size_t const cstreamEstimatedSize = ZSTD_estimateCStreamSize_usingCCtxParams(cctxParams);
+        size_t const dstreamSize = ZSTD_sizeof_DStream(dctx);
+        size_t const dstreamEstimatedSize = ZSTD_estimateDStreamSize_fromFrame(compressedData, compressedSize);
+
+        CHECK(cstreamSize <= cstreamEstimatedSize, "Compression mem (%u) > estimated (%u)",
+                (unsigned)cstreamSize, (unsigned)cstreamEstimatedSize);
+        CHECK(dstreamSize <= dstreamEstimatedSize, "Decompression mem (%u) > estimated (%u)",
+                (unsigned)dstreamSize, (unsigned)dstreamEstimatedSize);
+
+        printf("Level %2i : Compression Mem = %5u KB (estimated : %5u KB) ; Decompression Mem = %4u KB (estimated : %5u KB)\n",
+                compressionLevel,
+                (unsigned)(cstreamSize>>10), (unsigned)(cstreamEstimatedSize>>10),
+                (unsigned)(dstreamSize>>10), (unsigned)(dstreamEstimatedSize>>10));
+
+        ZSTD_freeDCtx(dctx);
+        ZSTD_freeCCtx(cctx);
+        ZSTD_freeCCtxParams(cctxParams);
+        if (wLog) break;  /* single test */
+    }
+    return 0;
+}