Add option -d (decompress) to fpga_compress. Allows testing.

Improve zlib deflate (note: no change required to inflate).

client/Makefile

@@ -103,8 +103,8 @@ CMDSRCS = 	nonce2key/crapto1.c\
 			aes.c\
 			protocols.c\
 
-ZLIBSRCS = deflate.c adler32.c trees.c zutil.c
+ZLIBSRCS = deflate.c adler32.c trees.c zutil.c inflate.c inffast.c inftrees.c
-ZLIB_FLAGS = -DZ_SOLO -DZ_PREFIX -DNO_GZIP -DZLIB_PM3_TUNED
+ZLIB_FLAGS = -DZ_SOLO -DZ_PREFIX -DNO_GZIP -DZLIB_PM3_TUNED -DDEBUG -Dverbose=1
 
 
 COREOBJS = $(CORESRCS:%.c=$(OBJDIR)/%.o)

client/fpga_compress.c

@@ -45,14 +45,12 @@ static void usage(char *argv0)
 
 static voidpf fpga_deflate_malloc(voidpf opaque, uInt items, uInt size)
 {
-	fprintf(stderr, "zlib requested %d bytes\n", items*size);
 	return malloc(items*size);
 }
 
 
 static void fpga_deflate_free(voidpf opaque, voidpf address)
 {
-	fprintf(stderr, "zlib frees memory\n");
 	return free(address);
 }
 
@@ -119,7 +117,6 @@ int zlib_compress(FILE *infile[], uint8_t num_infiles, FILE *outfile)
 
 	// estimate the size of the compressed output
 	unsigned int outsize_max = deflateBound(&compressed_fpga_stream, compressed_fpga_stream.avail_in);
-	fprintf(stderr, "Allocating %ld bytes for output file (estimated upper bound)\n", outsize_max);
 	uint8_t *outbuf = malloc(outsize_max);
 	compressed_fpga_stream.next_out = outbuf;
 	compressed_fpga_stream.avail_out = outsize_max;
@@ -137,7 +134,7 @@ int zlib_compress(FILE *infile[], uint8_t num_infiles, FILE *outfile)
 		ret = deflate(&compressed_fpga_stream, Z_FINISH);
 	}
 	
-	fprintf(stderr, "produced %d bytes of output\n", compressed_fpga_stream.total_out);
+	fprintf(stderr, "\ncompressed %d input bytes to %d output bytes\n", i, compressed_fpga_stream.total_out);
 
 	if (ret != Z_STREAM_END) {
 		fprintf(stderr, "Error in deflate(): %d %s\n", ret, compressed_fpga_stream.msg);
@@ -170,6 +167,72 @@ int zlib_compress(FILE *infile[], uint8_t num_infiles, FILE *outfile)
 }
 
 
+int zlib_decompress(FILE *infile, FILE *outfile)
+{
+	#define DECOMPRESS_BUF_SIZE 1024
+	uint8_t outbuf[DECOMPRESS_BUF_SIZE];
+	uint8_t inbuf[DECOMPRESS_BUF_SIZE];
+	int ret;
+	
+	z_stream compressed_fpga_stream;
+	// initialize zlib structures
+	compressed_fpga_stream.next_in = inbuf;
+	compressed_fpga_stream.avail_in = 0;
+	compressed_fpga_stream.next_out = outbuf;
+	compressed_fpga_stream.avail_out = DECOMPRESS_BUF_SIZE;
+	compressed_fpga_stream.zalloc = fpga_deflate_malloc;
+	compressed_fpga_stream.zfree = fpga_deflate_free;
+	
+	ret = inflateInit2(&compressed_fpga_stream, 0);
+	
+	do {
+		if (compressed_fpga_stream.avail_in == 0) {
+			compressed_fpga_stream.next_in = inbuf;
+			uint16_t i = 0;
+			do {
+				uint8_t c = fgetc(infile);
+				if (!feof(infile)) {
+					inbuf[i++] = c;
+					compressed_fpga_stream.avail_in++;
+				} else {
+					break;
+				}
+			} while (i < DECOMPRESS_BUF_SIZE);
+		}
+
+		ret = inflate(&compressed_fpga_stream, Z_SYNC_FLUSH);
+
+		if (ret != Z_OK && ret != Z_STREAM_END) {
+			break;
+		}
+
+		if (compressed_fpga_stream.avail_out == 0) {
+			for (uint16_t i = 0; i < DECOMPRESS_BUF_SIZE; i++) {
+				fputc(outbuf[i], outfile);
+			}
+			compressed_fpga_stream.avail_out = DECOMPRESS_BUF_SIZE;
+			compressed_fpga_stream.next_out = outbuf;
+		}
+	} while (ret == Z_OK);
+
+	if (ret == Z_STREAM_END) {  // reached end of input
+		uint16_t i = 0;
+		while (compressed_fpga_stream.avail_out < DECOMPRESS_BUF_SIZE) {
+			fputc(outbuf[i++], outfile);
+			compressed_fpga_stream.avail_out++;
+		}
+		fclose(outfile);
+		fclose(infile);
+		return 0;
+	} else {
+		fprintf(stderr, "Error. Inflate() returned error %d, %s", ret, compressed_fpga_stream.msg);
+		fclose(outfile);
+		fclose(infile);
+		return -1;
+	}
+	
+}
+
 
 int main(int argc, char **argv)
 {
@@ -181,6 +244,26 @@ int main(int argc, char **argv)
 		return -1;
 	}
 	
+	if (!strcmp(argv[1], "-d")) {
+		infiles = calloc(1, sizeof(FILE*));
+		if (argc != 4) {
+			usage(argv[0]);
+			return -1;
+		} 
+		infiles[0] = fopen(argv[2], "rb");
+		if (infiles[0] == NULL) {
+			fprintf(stderr, "Error. Cannot open input file %s", argv[2]);
+			return -1;
+		}
+		outfile = fopen(argv[3], "wb");
+		if (outfile == NULL) {
+			fprintf(stderr, "Error. Cannot open output file %s", argv[3]);
+			return -1;
+		}
+		return zlib_decompress(infiles[0], outfile);
+	}
+	
+
 	infiles = calloc(argc-2, sizeof(FILE*));
 	
 	for (uint16_t i = 0; i < argc-2; i++) { 
@@ -195,7 +278,7 @@ int main(int argc, char **argv)
 	if (outfile == NULL) {
 		fprintf(stderr, "Error. Cannot open output file %s", argv[argc-1]);
 		return -1;
-		}
+	}
 		
 	return zlib_compress(infiles, argc-2, outfile);
 }

zlib/deflate.c

@@ -1153,7 +1153,11 @@ local uInt longest_match(s, cur_match)
     register Bytef *scan = s->window + s->strstart; /* current string */
     register Bytef *match;                       /* matched string */
     register int len;                           /* length of current match */
+#ifdef ZLIB_PM3_TUNED
+	int best_len = 	MIN_MATCH-1;				// lift the restriction on prev-length
+#else
     int best_len = s->prev_length;              /* best match length so far */
+#endif
     int nice_match = s->nice_match;             /* stop if match long enough */
     IPos limit = s->strstart > (IPos)MAX_DIST(s) ?
         s->strstart - (IPos)MAX_DIST(s) : NIL;
@@ -1721,6 +1725,104 @@ local block_state deflate_fast(s, flush)
     return block_done;
 }
 
+
+#ifdef ZLIB_PM3_TUNED
+local uInt try_harder(s, strstart, lookahead, hash_head, level)
+	deflate_state *s;
+	uInt strstart;
+	uInt lookahead;
+	IPos hash_head;
+	uInt level;
+{
+	uInt strstart_save = s->strstart;
+	s->strstart = strstart;
+	uInt lookahead_save = s->lookahead;
+	s->lookahead = lookahead;
+	uInt ins_h_save = s->ins_h;
+	uInt combined_gain;
+	uInt best_combined_gain = 0;
+	uInt match_length;
+	uInt prev_length = s->prev_length < MIN_MATCH ? 1 : s->prev_length;
+	uInt best_prev_length = prev_length;
+	uInt current_match_start = s->match_start;
+	uInt current_match_length = s->match_length;
+
+	do {
+        if (hash_head != NIL && s->strstart - hash_head <= MAX_DIST(s)) {
+			match_length = longest_match (s, hash_head);
+			/* longest_match() sets match_start */
+		} else {
+			match_length = MIN_MATCH - 1;
+		}
+#if TOO_FAR <= 32767
+		if (match_length == MIN_MATCH && s->strstart - s->match_start > TOO_FAR) {
+			match_length = MIN_MATCH-1;
+		}
+#endif
+		if (s->strstart == strstart) {						// store match at current position
+			current_match_length = match_length;
+			current_match_start = s->match_start;
+		}
+		if (s->strstart - strstart + 1 < MIN_MATCH) { 		// previous match reduced to one or two literals
+			combined_gain = 0;								// need one literal per byte: no gain (assuming 8 bits per literal)
+		} else {
+			combined_gain = s->strstart - strstart + 1 - MIN_MATCH;  // (possibly truncated) previous_length - 3 literals
+		}
+		if (level > 1 && s->strstart+1 <= s->window_size - MIN_LOOKAHEAD) {	// test one level more
+			s->prev_length = match_length;
+			uInt save_ins_h = s->ins_h;
+			UPDATE_HASH(s, s->ins_h, s->window[(s->strstart+1) + (MIN_MATCH-1)]);
+			combined_gain += try_harder(s, s->strstart+1, s->lookahead-1, s->head[s->ins_h], level-1);
+			s->ins_h = save_ins_h;
+		} else {
+			if (match_length < MIN_MATCH) { 
+				combined_gain += 0;							// no gain
+			} else {
+				combined_gain += match_length - MIN_MATCH; 	// match_length bytes coded as approx three literals
+			}
+		}
+		// if (combined_length > s->lookahead - 1) {
+			// combined_length = s->lookahead;
+		// }
+		if (combined_gain >= best_combined_gain) {			// in case of a tie we prefer the longer prev_length
+			best_combined_gain = combined_gain;
+			best_prev_length = s->strstart - strstart + 1;
+		}
+		s->strstart++;
+		s->lookahead--;
+		UPDATE_HASH(s, s->ins_h, s->window[(s->strstart) + (MIN_MATCH-1)]);
+		hash_head = s->head[s->ins_h];
+		// if (s->strstart - strstart + 1 == MIN_MATCH-1) {		// a match with length == 2 is not possible
+			// s->strstart++;
+			// s->lookahead--;
+			// UPDATE_HASH(s, s->ins_h, s->window[(s->strstart) + (MIN_MATCH-1)]);
+			// hash_head = s->head[s->ins_h];
+		// }
+	} while (s->strstart <= strstart-1 + prev_length 			// try to truncate the previous match to 1, 3, ... prev_length
+			&& s->strstart <= s->window_size - MIN_LOOKAHEAD);	// watch out for the end of the input
+	
+	s->strstart = strstart_save;
+	s->lookahead = lookahead_save;
+	s->ins_h = ins_h_save;
+	s->match_length = current_match_length;
+	s->match_start = current_match_start;
+	if (prev_length >= MIN_MATCH) {
+		if (best_prev_length != prev_length && best_prev_length >= MIN_MATCH) {
+			printf("at %d, level %d: Reducing prev_length from %d to %d\n", s->strstart, level, prev_length, best_prev_length);
+		}
+	}
+	if (best_prev_length >= MIN_MATCH) {
+		s->prev_length = best_prev_length;
+		s->match_length = MIN_MATCH - 1;
+	} else {
+		s->prev_length = MIN_MATCH - 1;
+	}
+	return best_combined_gain;
+}
+#endif
+
+
+
 #ifndef FASTEST
 /* ===========================================================================
  * Same as above, but achieves better compression. We use a lazy
@@ -1757,11 +1859,16 @@ local block_state deflate_slow(s, flush)
             INSERT_STRING(s, s->strstart, hash_head);
         }
 
-        /* Find the longest match, discarding those <= prev_length.
+        /* Find the longest match, discarding those <= prev_length. */
-         */
         s->prev_length = s->match_length, s->prev_match = s->match_start;
         s->match_length = MIN_MATCH-1;
 
+#ifdef ZLIB_PM3_TUNED
+		if (s->prev_length < s->max_lazy_match) {
+			try_harder(s, s->strstart, s->lookahead, hash_head, 1);
+		}
+
+#else
         if (hash_head != NIL && s->prev_length < s->max_lazy_match &&
             s->strstart - hash_head <= MAX_DIST(s)) {
             /* To simplify the code, we prevent matches with the string
@@ -1784,6 +1891,7 @@ local block_state deflate_slow(s, flush)
                 s->match_length = MIN_MATCH-1;
             }
         }
+#endif /* ZLIB_PM3_TUNED */
         /* If there was a match at the previous step and the current
          * match is not better, output the previous match:
          */

zlib/inflate.c

@@ -847,8 +847,8 @@ int flush;
                 break;
             case 1:                             /* fixed block */
 #ifdef ZLIB_PM3_TUNED
-				Dbprintf("FATAL error. Compressed FPGA files with fixed code blocks are not supported!");
+                strm->msg = (char *)"fixed block coding not supported";
-				for(;;);
+				state->mode = BAD;
 #else				
                 fixedtables(state);
                 Tracev((stderr, "inflate:     fixed codes block%s\n",

zlib/trees.c

@@ -989,7 +989,9 @@ void ZLIB_INTERNAL _tr_flush_block(s, buf, stored_len, last)
 #ifdef DEBUG
         s->compressed_len += 3 + s->opt_len;
 #endif
+#ifndef ZLIB_PM3_TUNED	
     }
+#endif	
     Assert (s->compressed_len == s->bits_sent, "bad compressed size");
     /* The above check is made mod 2^32, for files larger than 512 MB
      * and uLong implemented on 32 bits.