[Zlib-devel] Request for feature in zlib: Start and stop markers (patch attached)

[Ingvar Hagelund]

Hello, zlib developers!

My name is Ingvar Hagelund. I maintain the Fedora package of Varnish.
Varnish is a very fast web cache. Fedora is, (as you of course know,) a
community driven operating system, sponsored and used as a test bed by
Red Hat. In my day job, I work at Redpill Linpro as a system
administrator.

Varnish uses libvgz, which is zlib with a small patch. This patch gives
access to start and stop markers in a stream of compressed files.
Varnish uses this to sort out edge side includes when putting together
http pages from backend servers for http clients.

The original author of the changes is Paul-Henning Kamp*. He has
written some more high level info about the usage here:
https://www.varnish-cache.org/docs/4.0/phk/gzip.html

For security reasons, standardization, avoidance of forks, (and
of course, elegance,) Fedora has a policy to avoid bundled libraries at
all cost. All applications should use the system libraries if they
exist**.

This means that the varnish package maintainer (me), must bully the zlib
package maintainer to add an extra patch to the Fedora package, and
change the Varnish build system for every release. Or I could parley for
an exception from the Fedora packaging policy.

As I am, as most system administrators, a bit lazy, I would instead
propose that the patch is included upstream, thus giving Fedora's zlib
maintainer and myself less work, and giving the opportunity for
other zlib users to take delight in using these start and stop markers.

I stripped down the changes from zlib-1.2.8 to a minimal patch attached
below. It applies cleanly, and builds fine on my Fedora boxes. It's also
available at http://users.linpro.no/ingvar/varnish/varnish_zlib_patch

Please consider including it in your next release of zlib.

Best regards,
Ingvar Hagelund


*) This guy:
http://en.wikipedia.org/wiki/Poul-Henning_Kamp

**) Fedora packaging policy details on bundled libraries:
https://fedoraproject.org/wiki/Packaging:No_Bundled_Libraries

-------------- next part --------------
--- deflate.c	2013-04-29 00:57:10.000000000 +0200
+++ ../libvgz/deflate.c	2014-07-31 12:04:44.802342047 +0200
@@ -400,6 +400,7 @@
     }
 
     strm->total_in = strm->total_out = 0;
+    strm->start_bit = strm->stop_bit = strm->last_bit = 0;
     strm->msg = Z_NULL; /* use zfree if we ever allocate msg dynamically */
     strm->data_type = Z_UNKNOWN;
 
@@ -894,6 +895,9 @@
         ERR_RETURN(strm, Z_BUF_ERROR);
     }
 
+    if (strm->start_bit == 0)
+        strm->start_bit = (strm->total_out + s->pending) * 8 + s->bi_valid;
+
     /* Start a new block or continue the current one.
      */
     if (strm->avail_in != 0 || s->lookahead != 0 ||
--- inflate.c	2012-08-13 09:02:40.000000000 +0200
+++ ../libvgz/inflate.c	2014-07-31 12:12:38.984687363 +0200
@@ -109,6 +109,7 @@
     if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR;
     state = (struct inflate_state FAR *)strm->state;
     strm->total_in = strm->total_out = state->total = 0;
+    strm->start_bit = strm->stop_bit = strm->last_bit = 0;
     strm->msg = Z_NULL;
     if (state->wrap)        /* to support ill-conceived Java test suite */
         strm->adler = state->wrap & 1;
@@ -826,13 +827,18 @@
         case TYPE:
             if (flush == Z_BLOCK || flush == Z_TREES) goto inf_leave;
         case TYPEDO:
+            if (strm->start_bit == 0)
+                strm->start_bit = 8 * (strm->total_in + in - have) - bits;
             if (state->last) {
+                strm->stop_bit = 8 * (strm->total_in + in - have) - bits;
                 BYTEBITS();
                 state->mode = CHECK;
                 break;
             }
             NEEDBITS(3);
             state->last = BITS(1);
+	    if (state->last)
+                strm->last_bit = 8 * (strm->total_in + in - have) - bits;
             DROPBITS(1);
             switch (BITS(2)) {
             case 0:                             /* stored block */
--- trees.c	2012-08-13 09:02:40.000000000 +0200
+++ ../libvgz/trees.c	2014-07-31 11:18:19.451292381 +0200
@@ -913,6 +913,10 @@
     ulg opt_lenb, static_lenb; /* opt_len and static_len in bytes */
     int max_blindex = 0;  /* index of last bit length code of non zero freq */
 
+    if (last)
+        s->strm->last_bit =
+           (s->strm->total_out + s->pending) * 8 + s->bi_valid;
+
     /* Build the Huffman trees unless a stored block is forced */
     if (s->level > 0) {
 
@@ -994,6 +998,8 @@
     init_block(s);
 
     if (last) {
+        s->strm->stop_bit =
+           (s->strm->total_out + s->pending) * 8 + s->bi_valid;
         bi_windup(s);
 #ifdef DEBUG
         s->compressed_len += 7;  /* align on byte boundary */
--- zlib.h	2013-04-29 02:23:49.000000000 +0200
+++ ../libvgz/zlib.h	2014-07-31 11:18:19.318619433 +0200
@@ -101,6 +101,10 @@
     int     data_type;  /* best guess about the data type: binary or text */
     uLong   adler;      /* adler32 value of the uncompressed data */
     uLong   reserved;   /* reserved for future use */
+
+    uLong   start_bit;	/* Bit pos of first deflate block */
+    uLong   stop_bit;	/* Bit pos after last deflate block */
+    uLong   last_bit;	/* Bit pos of 'last' bit */
 } z_stream;
 
 typedef z_stream FAR *z_streamp;