src/std/zlib.d

// Written in the D programming language.

/**
 * Compress/decompress data using the $(HTTP www._zlib.net, _zlib library).
 *
 * Examples:
 *
 * If you have a small buffer you can use $(LREF compress) and
 * $(LREF uncompress) directly.
 *
 * -------
 * import std.zlib;
 *
 * auto src =
 * "the quick brown fox jumps over the lazy dog\r
 *  the quick brown fox jumps over the lazy dog\r";
 *
 * ubyte[] dst;
 * ubyte[] result;
 *
 * dst = compress(src);
 * result = cast(ubyte[]) uncompress(dst);
 * assert(result == src);
 * -------
 *
 * When the data to be compressed doesn't fit in one buffer, use
 * $(LREF Compress) and $(LREF UnCompress).
 *
 * -------
 * import std.zlib;
 * import std.stdio;
 * import std.conv : to;
 * import std.algorithm.iteration : map;
 *
 * UnCompress decmp = new UnCompress;
 * foreach (chunk; stdin.byChunk(4096).map!(x => decmp.uncompress(x)))
 * {
 *     chunk.to!string.write;
 * }

 * -------
 *
 * References:
 *  $(HTTP en.wikipedia.org/wiki/Zlib, Wikipedia)
 *
 * Copyright: Copyright Digital Mars 2000 - 2011.
 * License:   $(HTTP www.boost.org/LICENSE_1_0.txt, Boost License 1.0).
 * Authors:   $(HTTP digitalmars.com, Walter Bright)
 * Source:    $(PHOBOSSRC std/_zlib.d)
 */
/*          Copyright Digital Mars 2000 - 2011.
 * Distributed under the Boost Software License, Version 1.0.
 *    (See accompanying file LICENSE_1_0.txt or copy at
 *          http://www.boost.org/LICENSE_1_0.txt)
 */
module std.zlib;

//debug=zlib;       // uncomment to turn on debugging printf's

import etc.c.zlib;

// Values for 'mode'

enum
{
    Z_NO_FLUSH      = 0,
    Z_SYNC_FLUSH    = 2,
    Z_FULL_FLUSH    = 3,
    Z_FINISH        = 4,
}

/*************************************
 * Errors throw a ZlibException.
 */

class ZlibException : Exception
{
    this(int errnum)
    {   string msg;

        switch (errnum)
        {
            case Z_STREAM_END:      msg = "stream end"; break;
            case Z_NEED_DICT:       msg = "need dict"; break;
            case Z_ERRNO:           msg = "errno"; break;
            case Z_STREAM_ERROR:    msg = "stream error"; break;
            case Z_DATA_ERROR:      msg = "data error"; break;
            case Z_MEM_ERROR:       msg = "mem error"; break;
            case Z_BUF_ERROR:       msg = "buf error"; break;
            case Z_VERSION_ERROR:   msg = "version error"; break;
            default:                msg = "unknown error";  break;
        }
        super(msg);
    }
}

/**
 * $(P Compute the Adler-32 checksum of a buffer's worth of data.)
 *
 * Params:
 *     adler = the starting checksum for the computation. Use 1
 *             for a new checksum. Use the output of this function
 *             for a cumulative checksum.
 *     buf = buffer containing input data
 *
 * Returns:
 *     A $(D uint) checksum for the provided input data and starting checksum
 *
 * See_Also:
 *     $(LINK http://en.wikipedia.org/wiki/Adler-32)
 */

uint adler32(uint adler, const(void)[] buf)
{
    import std.range : chunks;
    foreach (chunk; (cast(ubyte[]) buf).chunks(0xFFFF0000))
    {
        adler = etc.c.zlib.adler32(adler, chunk.ptr, cast(uint) chunk.length);
    }
    return adler;
}

///
@system unittest
{
    static ubyte[] data = [1,2,3,4,5,6,7,8,9,10];

    uint adler = adler32(0u, data);
    assert(adler == 0xdc0037);
}

@system unittest
{
    static string data = "test";

    uint adler = adler32(1, data);
    assert(adler == 0x045d01c1);
}

/**
 * $(P Compute the CRC32 checksum of a buffer's worth of data.)
 *
 * Params:
 *     crc = the starting checksum for the computation. Use 0
 *             for a new checksum. Use the output of this function
 *             for a cumulative checksum.
 *     buf = buffer containing input data
 *
 * Returns:
 *     A $(D uint) checksum for the provided input data and starting checksum
 *
 * See_Also:
 *     $(LINK http://en.wikipedia.org/wiki/Cyclic_redundancy_check)
 */

uint crc32(uint crc, const(void)[] buf)
{
    import std.range : chunks;
    foreach (chunk; (cast(ubyte[]) buf).chunks(0xFFFF0000))
    {
        crc = etc.c.zlib.crc32(crc, chunk.ptr, cast(uint) chunk.length);
    }
    return crc;
}

@system unittest
{
    static ubyte[] data = [1,2,3,4,5,6,7,8,9,10];

    uint crc;

    debug(zlib) printf("D.zlib.crc32.unittest\n");
    crc = crc32(0u, cast(void[]) data);
    debug(zlib) printf("crc = %x\n", crc);
    assert(crc == 0x2520577b);
}

/**
 * $(P Compress data)
 *
 * Params:
 *     srcbuf = buffer containing the data to compress
 *     level = compression level. Legal values are -1 .. 9, with -1 indicating
 *             the default level (6), 0 indicating no compression, 1 being the
 *             least compression and 9 being the most.
 *
 * Returns:
 *     the compressed data
 */

ubyte[] compress(const(void)[] srcbuf, int level)
in
{
    assert(-1 <= level && level <= 9);
}
body
{
    import core.memory : GC;
    auto destlen = srcbuf.length + ((srcbuf.length + 1023) / 1024) + 12;
    auto destbuf = new ubyte[destlen];
    auto err = etc.c.zlib.compress2(destbuf.ptr, &destlen, cast(ubyte *) srcbuf.ptr, srcbuf.length, level);
    if (err)
    {
        GC.free(destbuf.ptr);
        throw new ZlibException(err);
    }

    destbuf.length = destlen;
    return destbuf;
}

/*********************************************
 * ditto
 */

ubyte[] compress(const(void)[] srcbuf)
{
    return compress(srcbuf, Z_DEFAULT_COMPRESSION);
}

/*********************************************
 * Decompresses the data in srcbuf[].
 * Params:
 *  srcbuf  = buffer containing the compressed data.
 *  destlen = size of the uncompressed data.
 *            It need not be accurate, but the decompression will be faster
 *            if the exact size is supplied.
 *  winbits = the base two logarithm of the maximum window size.
 * Returns: the decompressed data.
 */

void[] uncompress(const(void)[] srcbuf, size_t destlen = 0u, int winbits = 15)
{
    import std.conv : to;
    int err;
    ubyte[] destbuf;

    if (!destlen)
        destlen = srcbuf.length * 2 + 1;

    etc.c.zlib.z_stream zs;
    zs.next_in = cast(typeof(zs.next_in)) srcbuf.ptr;
    zs.avail_in = to!uint(srcbuf.length);
    err = etc.c.zlib.inflateInit2(&zs, winbits);
    if (err)
    {
        throw new ZlibException(err);
    }

    size_t olddestlen = 0u;

    loop:
    while (true)
    {
        destbuf.length = destlen;
        zs.next_out = cast(typeof(zs.next_out)) &destbuf[olddestlen];
        zs.avail_out = to!uint(destlen - olddestlen);
        olddestlen = destlen;

        err = etc.c.zlib.inflate(&zs, Z_NO_FLUSH);
        switch (err)
        {
            case Z_OK:
                destlen = destbuf.length * 2;
                continue loop;

            case Z_STREAM_END:
                destbuf.length = zs.total_out;
                err = etc.c.zlib.inflateEnd(&zs);
                if (err != Z_OK)
                    throw new ZlibException(err);
                return destbuf;

            default:
                etc.c.zlib.inflateEnd(&zs);
                throw new ZlibException(err);
        }
    }
    assert(0);
}

@system unittest
{
    auto src =
"the quick brown fox jumps over the lazy dog\r
the quick brown fox jumps over the lazy dog\r
";
    ubyte[] dst;
    ubyte[] result;

    //arrayPrint(src);
    dst = compress(src);
    //arrayPrint(dst);
    result = cast(ubyte[]) uncompress(dst);
    //arrayPrint(result);
    assert(result == src);
}

@system unittest
{
    ubyte[] src = new ubyte[1000000];
    ubyte[] dst;
    ubyte[] result;

    src[] = 0x80;
    dst = compress(src);
    assert(dst.length*2 + 1 < src.length);
    result = cast(ubyte[]) uncompress(dst);
    assert(result == src);
}

/+
void arrayPrint(ubyte[] array)
{
    //printf("array %p,%d\n", cast(void*) array, array.length);
    for (size_t i = 0; i < array.length; i++)
    {
        printf("%02x ", array[i]);
        if (((i + 1) & 15) == 0)
            printf("\n");
    }
    printf("\n\n");
}
+/

/// the header format the compressed stream is wrapped in
enum HeaderFormat {
    deflate, /// a standard zlib header
    gzip, /// a gzip file format header
    determineFromData /// used when decompressing. Try to automatically detect the stream format by looking at the data
}

/*********************************************
 * Used when the data to be compressed is not all in one buffer.
 */

class Compress
{
    import std.conv : to;

  private:
    z_stream zs;
    int level = Z_DEFAULT_COMPRESSION;
    int inited;
    immutable bool gzip;

    void error(int err)
    {
        if (inited)
        {   deflateEnd(&zs);
            inited = 0;
        }
        throw new ZlibException(err);
    }

  public:

    /**
     * Constructor.
     *
     * Params:
     *    level = compression level. Legal values are 1 .. 9, with 1 being the least
     *            compression and 9 being the most. The default value is 6.
     *    header = sets the compression type to one of the options available
     *             in $(LREF HeaderFormat). Defaults to HeaderFormat.deflate.
     *
     * See_Also:
     *    $(LREF compress), $(LREF HeaderFormat)
     */
    this(int level, HeaderFormat header = HeaderFormat.deflate)
    in
    {
        assert(1 <= level && level <= 9);
    }
    body
    {
        this.level = level;
        this.gzip = header == HeaderFormat.gzip;
    }

    /// ditto
    this(HeaderFormat header = HeaderFormat.deflate)
    {
        this.gzip = header == HeaderFormat.gzip;
    }

    ~this()
    {   int err;

        if (inited)
        {
            inited = 0;
            deflateEnd(&zs);
        }
    }

    /**
     * Compress the data in buf and return the compressed data.
     * Params:
     *    buf = data to compress
     *
     * Returns:
     *    the compressed data. The buffers returned from successive calls to this should be concatenated together.
     *
     */
    const(void)[] compress(const(void)[] buf)
    {
        import core.memory : GC;
        int err;
        ubyte[] destbuf;

        if (buf.length == 0)
            return null;

        if (!inited)
        {
            err = deflateInit2(&zs, level, Z_DEFLATED, 15 + (gzip ? 16 : 0), 8, Z_DEFAULT_STRATEGY);
            if (err)
                error(err);
            inited = 1;
        }

        destbuf = new ubyte[zs.avail_in + buf.length];
        zs.next_out = destbuf.ptr;
        zs.avail_out = to!uint(destbuf.length);

        if (zs.avail_in)
            buf = zs.next_in[0 .. zs.avail_in] ~ cast(ubyte[]) buf;

        zs.next_in = cast(typeof(zs.next_in)) buf.ptr;
        zs.avail_in = to!uint(buf.length);

        err = deflate(&zs, Z_NO_FLUSH);
        if (err != Z_STREAM_END && err != Z_OK)
        {
            GC.free(destbuf.ptr);
            error(err);
        }
        destbuf.length = destbuf.length - zs.avail_out;
        return destbuf;
    }

    /***
     * Compress and return any remaining data.
     * The returned data should be appended to that returned by compress().
     * Params:
     *  mode = one of the following:
     *          $(DL
                    $(DT Z_SYNC_FLUSH )
                    $(DD Syncs up flushing to the next byte boundary.
                        Used when more data is to be compressed later on.)
                    $(DT Z_FULL_FLUSH )
                    $(DD Syncs up flushing to the next byte boundary.
                        Used when more data is to be compressed later on,
                        and the decompressor needs to be restartable at this
                        point.)
                    $(DT Z_FINISH)
                    $(DD (default) Used when finished compressing the data. )
                )
     */
    void[] flush(int mode = Z_FINISH)
    in
    {
        assert(mode == Z_FINISH || mode == Z_SYNC_FLUSH || mode == Z_FULL_FLUSH);
    }
    body
    {
        import core.memory : GC;
        ubyte[] destbuf;
        ubyte[512] tmpbuf = void;
        int err;

        if (!inited)
            return null;

        /* may be  zs.avail_out+<some constant>
         * zs.avail_out is set nonzero by deflate in previous compress()
         */
        //tmpbuf = new void[zs.avail_out];
        zs.next_out = tmpbuf.ptr;
        zs.avail_out = tmpbuf.length;

        while ( (err = deflate(&zs, mode)) != Z_STREAM_END)
        {
            if (err == Z_OK)
            {
                if (zs.avail_out != 0 && mode != Z_FINISH)
                    break;
                else if (zs.avail_out == 0)
                {
                    destbuf ~= tmpbuf;
                    zs.next_out = tmpbuf.ptr;
                    zs.avail_out = tmpbuf.length;
                    continue;
                }
                err = Z_BUF_ERROR;
            }
            GC.free(destbuf.ptr);
            error(err);
        }
        destbuf ~= tmpbuf[0 .. (tmpbuf.length - zs.avail_out)];

        if (mode == Z_FINISH)
        {
            err = deflateEnd(&zs);
            inited = 0;
            if (err)
                error(err);
        }
        return destbuf;
    }
}

/******
 * Used when the data to be decompressed is not all in one buffer.
 */

class UnCompress
{
    import std.conv : to;

  private:
    z_stream zs;
    int inited;
    int done;
    size_t destbufsize;

    HeaderFormat format;

    void error(int err)
    {
        if (inited)
        {   inflateEnd(&zs);
            inited = 0;
        }
        throw new ZlibException(err);
    }

  public:

    /**
     * Construct. destbufsize is the same as for D.zlib.uncompress().
     */
    this(uint destbufsize)
    {
        this.destbufsize = destbufsize;
    }

    /** ditto */
    this(HeaderFormat format = HeaderFormat.determineFromData)
    {
        this.format = format;
    }

    ~this()
    {   int err;

        if (inited)
        {
            inited = 0;
            inflateEnd(&zs);
        }
        done = 1;
    }

    /**
     * Decompress the data in buf and return the decompressed data.
     * The buffers returned from successive calls to this should be concatenated
     * together.
     */
    const(void)[] uncompress(const(void)[] buf)
    in
    {
        assert(!done);
    }
    body
    {
        import core.memory : GC;
        int err;
        ubyte[] destbuf;

        if (buf.length == 0)
            return null;

        if (!inited)
        {
        int windowBits = 15;
        if (format == HeaderFormat.gzip)
            windowBits += 16;
            else if (format == HeaderFormat.determineFromData)
            windowBits += 32;

            err = inflateInit2(&zs, windowBits);
            if (err)
                error(err);
            inited = 1;
        }

        if (!destbufsize)
            destbufsize = to!uint(buf.length) * 2;
        destbuf = new ubyte[zs.avail_in * 2 + destbufsize];
        zs.next_out = destbuf.ptr;
        zs.avail_out = to!uint(destbuf.length);

        if (zs.avail_in)
            buf = zs.next_in[0 .. zs.avail_in] ~ cast(ubyte[]) buf;

        zs.next_in = cast(ubyte*) buf.ptr;
        zs.avail_in = to!uint(buf.length);

        err = inflate(&zs, Z_NO_FLUSH);
        if (err != Z_STREAM_END && err != Z_OK)
        {
            GC.free(destbuf.ptr);
            error(err);
        }
        destbuf.length = destbuf.length - zs.avail_out;
        return destbuf;
    }

    /**
     * Decompress and return any remaining data.
     * The returned data should be appended to that returned by uncompress().
     * The UnCompress object cannot be used further.
     */
    void[] flush()
    in
    {
        assert(!done);
    }
    out
    {
        assert(done);
    }
    body
    {
        import core.memory : GC;
        ubyte[] extra;
        ubyte[] destbuf;
        int err;

        done = 1;
        if (!inited)
            return null;

      L1:
        destbuf = new ubyte[zs.avail_in * 2 + 100];
        zs.next_out = destbuf.ptr;
        zs.avail_out = to!uint(destbuf.length);

        err = etc.c.zlib.inflate(&zs, Z_NO_FLUSH);
        if (err == Z_OK && zs.avail_out == 0)
        {
            extra ~= destbuf;
            goto L1;
        }
        if (err != Z_STREAM_END)
        {
            GC.free(destbuf.ptr);
            if (err == Z_OK)
                err = Z_BUF_ERROR;
            error(err);
        }
        destbuf = destbuf.ptr[0 .. zs.next_out - destbuf.ptr];
        err = etc.c.zlib.inflateEnd(&zs);
        inited = 0;
        if (err)
            error(err);
        if (extra.length)
            destbuf = extra ~ destbuf;
        return destbuf;
    }
}

/* ========================== unittest ========================= */

import std.random;
import std.stdio;

@system unittest // by Dave
{
    debug(zlib) writeln("std.zlib.unittest");

    bool CompressThenUncompress (void[] src)
    {
        ubyte[] dst = std.zlib.compress(src);
        double ratio = (dst.length / cast(double) src.length);
        debug(zlib) writef("src.length: %1$d, dst: %2$d, Ratio = %3$f", src.length, dst.length, ratio);
        ubyte[] uncompressedBuf;
        uncompressedBuf = cast(ubyte[]) std.zlib.uncompress(dst);
        assert(src.length == uncompressedBuf.length);
        assert(src == uncompressedBuf);

        return true;
    }


    // smallish buffers
    for (int idx = 0; idx < 25; idx++)
    {
        char[] buf = new char[uniform(0, 100)];

        // Alternate between more & less compressible
        foreach (ref char c; buf)
            c = cast(char) (' ' + (uniform(0, idx % 2 ? 91 : 2)));

        if (CompressThenUncompress(buf))
        {
            debug(zlib) writeln("; Success.");
        }
        else
        {
            return;
        }
    }

    // larger buffers
    for (int idx = 0; idx < 25; idx++)
    {
        char[] buf = new char[uniform(0, 1000/*0000*/)];

        // Alternate between more & less compressible
        foreach (ref char c; buf)
            c = cast(char) (' ' + (uniform(0, idx % 2 ? 91 : 10)));

        if (CompressThenUncompress(buf))
        {
            debug(zlib) writefln("; Success.");
        }
        else
        {
            return;
        }
    }

    debug(zlib) writefln("PASSED std.zlib.unittest");
}


@system unittest // by Artem Rebrov
{
    Compress cmp = new Compress;
    UnCompress decmp = new UnCompress;

    const(void)[] input;
    input = "tesatdffadf";

    const(void)[] buf = cmp.compress(input);
    buf ~= cmp.flush();
    const(void)[] output = decmp.uncompress(buf);

    //writefln("input = '%s'", cast(char[]) input);
    //writefln("output = '%s'", cast(char[]) output);
    assert( output[] == input[] );
}

@system unittest
{
    static assert(__traits(compiles, etc.c.zlib.gzclose(null)));        // bugzilla 15457
}