implement dictionary support when packing

Cargo.lock

@@ -308,7 +308,7 @@ checksum = "dcc811dc4066ac62f84f11307873c4850cb653bfa9b1719cee2bd2204a4bc5dd"
 
 [[package]]
 name = "upkr"
-version = "0.2.0"
+version = "0.2.2"
 dependencies = [
  "anyhow",
  "cdivsufsort",

Cargo.toml

@@ -1,15 +1,18 @@
 [package]
 name = "upkr"
-version = "0.2.0"
+version = "0.2.2"
 edition = "2021"
 
 [profile.release]
 strip = "debuginfo"
 
+[features]
+terminal = ["crossterm", "pbr"]
+
 [dependencies]
 cdivsufsort = "2"
 lexopt = "0.2.1"
 anyhow = "1"
 thiserror = "1.0.36"
-pbr = "1"
+pbr = { version = "1", optional = true }
 crossterm = { version = "0.25.0", default-features = false, optional = true }

README.md

@@ -22,6 +22,12 @@ c_unpacker and asm_unpackers unpack the default upkr compressed format. The z80_
 is based on some variations to the compressed format. (Use `upkr --z80` to select those variations.)
 The 16 bit dos unpacker also uses some variations. (`upkr --x86`)
 
+### More unpackers outside this repository
+
+* [Atari Lynx](https://github.com/42Bastian/new_bll/blob/master/demos/depacker/unupkr.asm)
+* [Atari Jaguar](https://github.com/42Bastian/new_bjl/blob/main/exp/depacker/unupkr.js)
+* [8080, R800](https://github.com/ivagorRetrocomp/DeUpkr)
+
 ## Usage
 
 ```
@@ -32,10 +38,17 @@ The 16 bit dos unpacker also uses some variations. (`upkr --x86`)
 
  -l, --level N       compression level 0-9
  -0, ..., -9         short form for setting compression level
- -u, --unpack        unpack infile
+ -d, --decompress    decompress infile
  --heatmap           calculate heatmap from compressed file
+   --raw-cost        report raw cost of literals in heatmap
+                     (the cost of literals is spread across all matches
+                     that reference the literal by default.)
+   --hexdump         print heatmap as colored hexdump
  --margin            calculate margin for overlapped unpacking of a packed file
 
+When no infile is given, or the infile is '-', read from stdin.
+When no outfile is given and reading from stdin, or when outfile is '-', write to stdout.
+
 Config presets for specific unpackers:
  --z80               --big-endian-bitstream --invert-bit-encoding --simplified-prob-update -9
  --x86               --bitstream --invert-is-match-bit --invert-continue-value-bit --invert-new-offset-bit

c_library/.gitignore

@@ -0,0 +1,2 @@
+/target/
+/upkr

c_library/Cargo.lock

@@ -0,0 +1,127 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "anyhow"
+version = "1.0.69"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800"
+
+[[package]]
+name = "autocfg"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
+
+[[package]]
+name = "cc"
+version = "1.0.79"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f"
+
+[[package]]
+name = "cdivsufsort"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "edefce019197609da416762da75bb000bbd2224b2d89a7e722c2296cbff79b8c"
+dependencies = [
+ "cc",
+ "sacabase",
+]
+
+[[package]]
+name = "lexopt"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "478ee9e62aaeaf5b140bd4138753d1f109765488581444218d3ddda43234f3e8"
+
+[[package]]
+name = "num-traits"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.51"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "sacabase"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9883fc3d6ce3d78bb54d908602f8bc1f7b5f983afe601dabe083009d86267a84"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "syn"
+version = "1.0.109"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "thiserror"
+version = "1.0.39"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a5ab016db510546d856297882807df8da66a16fb8c4101cb8b30054b0d5b2d9c"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.39"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5420d42e90af0c38c3290abcca25b9b3bdf379fc9f55c528f53a269d9c9a267e"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4"
+
+[[package]]
+name = "upkr"
+version = "0.2.1"
+dependencies = [
+ "anyhow",
+ "cdivsufsort",
+ "lexopt",
+ "thiserror",
+]
+
+[[package]]
+name = "upkr_c"
+version = "0.0.1"
+dependencies = [
+ "upkr",
+]

c_library/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "upkr_c"
+version = "0.0.1"
+edition = "2021"
+
+[lib]
+name = "upkr"
+crate-type = ["staticlib"]
+
+[profile.release]
+opt-level = "s"
+strip = "debuginfo"
+lto = true
+panic = "abort"
+
+[dependencies]
+upkr = { path="..", default-features=false }

c_library/Makefile

@@ -0,0 +1,8 @@
+upkr: upkr.c upkr.h target/release/libupkr.a
+	gcc -O2 -Ltarget/release -o upkr upkr.c -lupkr -lm
+	strip upkr
+
+target/release/libupkr.a: cargo
+	cargo build --release
+
+.PHONY: cargo

c_library/Readme.md

@@ -0,0 +1,11 @@
+This is a simple example of compiling upkr to a library that can be linked in a
+c program. It consists of a small rust crate which implements the c api and
+compiles to a static library and a matching c header file. As is, the rust
+crate offers two simple functions to compress/uncompress data with the default
+upkr config.
+
+The provided makefile will only work on linux. Building the example upkr.c on
+other platforms is left as an exercise for the reader ;)
+
+On Windows you might have to make sure to install and use the correct rust
+toolchain version (mingw vs. msvc) to match your c compiler.

c_library/src/lib.rs

@@ -0,0 +1,42 @@
+use std::ffi::c_int;
+
+// the upkr config to use, this can be modified to use other configs
+fn config() -> upkr::Config {
+    upkr::Config::default()
+}
+
+#[no_mangle]
+pub extern "C" fn upkr_compress(
+    output_buffer: *mut u8,
+    output_buffer_size: usize,
+    input_buffer: *const u8,
+    input_size: usize,
+    compression_level: c_int,
+) -> usize {
+    let output_buffer = unsafe { std::slice::from_raw_parts_mut(output_buffer, output_buffer_size) };
+    let input_buffer = unsafe { std::slice::from_raw_parts(input_buffer, input_size) };
+
+    let packed_data = upkr::pack(input_buffer, compression_level.max(0).min(9) as u8, &config(), None);
+    let copy_size = packed_data.len().min(output_buffer.len());
+    output_buffer[..copy_size].copy_from_slice(&packed_data[..copy_size]);
+
+    packed_data.len()
+}
+
+#[no_mangle]
+pub extern "C" fn upkr_uncompress(output_buffer: *mut u8, output_buffer_size: usize, input_buffer: *const u8, input_size: usize) -> isize {
+    let output_buffer = unsafe { std::slice::from_raw_parts_mut(output_buffer, output_buffer_size)};
+    let input_buffer = unsafe { std::slice::from_raw_parts(input_buffer, input_size)};
+
+    match upkr::unpack(input_buffer, &config(), output_buffer.len()) {
+        Ok(unpacked_data) => {
+            output_buffer[..unpacked_data.len()].copy_from_slice(&unpacked_data);
+            unpacked_data.len() as isize
+        }
+        Err(upkr::UnpackError::OverSize { size, .. }) => size as isize,
+        Err(other) => {
+            eprintln!("[upkr] compressed data corrupt: {}", other);
+            -1
+        }
+    }
+}

c_library/upkr.c

@@ -0,0 +1,99 @@
+#include "upkr.h"
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+
+int main(int argc, char** argv) {
+    if(argc < 2) {
+        fprintf(stdout, "Usage:\n  upkr [compress] [-0 .. -9] <file> [<out-file>]\n  upkr [uncompress] <file> [<out-file>]\n");
+        return 1;
+    }
+
+    int argi = 1;
+    int uncompress = 0;
+    int compression_level = 4;
+    if(strcmp(argv[argi], "compress") == 0) {
+        ++argi;
+    } else if(strcmp(argv[argi], "uncompress") == 0) {
+        uncompress = 1;
+        ++argi;
+    }
+
+    if(argi < argc && argv[argi][0] == '-') {
+        compression_level = atoi(argv[argi] + 1);
+        ++argi;
+    }
+
+    if(argi == argc) {
+        fprintf(stdout, "intput filename missing\n");
+        return 1;
+    }
+
+    const char* input_name = argv[argi++];
+    char* output_name;
+    if(argi < argc) {
+        output_name = argv[argi];
+    } else {
+        output_name = malloc(strlen(input_name) + 5);
+        strcpy(output_name, input_name);
+        strcat(output_name, uncompress ? ".unp" : ".upk");
+    }
+
+    FILE* file = fopen(input_name, "rb");
+    if(file == 0) {
+        fprintf(stdout, "failed to open input file '%s'\n", file);
+        return 1;
+    }
+    fseek(file, 0, SEEK_END);
+    long input_size = ftell(file);
+    rewind(file);
+
+    char* input_buffer = (char*)malloc(input_size);
+    long offset = 0;
+    while(offset < input_size) {
+        long read_size = fread(input_buffer + offset, 1, input_size - offset, file);
+        if(read_size <= 0) {
+            fprintf(stdout, "error reading input file\n");
+            return 1;
+        }
+        offset += read_size;
+    }
+    fclose(file);
+
+    long output_buffer_size = input_size * 8;
+    long output_size;
+    char* output_buffer = (char*)malloc(output_buffer_size);
+    for(;;) {
+        if(uncompress) {
+            output_size = upkr_uncompress(output_buffer, output_buffer_size, input_buffer, input_size);
+        } else {
+            output_size = upkr_compress(output_buffer, output_buffer_size, input_buffer, input_size, compression_level);
+        }
+        if(output_size < 0) {
+            return 1;
+        }
+        if(output_size <= output_buffer_size) {
+            break;
+        }
+        output_buffer = (char*)realloc(output_buffer, output_size);
+        output_buffer_size = output_size;
+    }
+
+    file = fopen(output_name, "wb");
+    if(file == 0) {
+        fprintf(stdout, "failed to open output file '%s'\n", output_name);
+        return 1;
+    }
+    offset = 0;
+    while(offset < output_size) {
+        long written_size = fwrite(output_buffer + offset, 1, output_size - offset, file);
+        if(written_size <= 0) {
+            fprintf(stdout, "error writing output file\n");
+            return 1;
+        }
+        offset += written_size;
+    }
+    fclose(file);
+
+    return 0;
+}

c_library/upkr.h

@@ -0,0 +1,25 @@
+#ifndef UPKR_H_INCLUDED
+
+#include <stddef.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// input_buffer/input_size: input data to compress
+// output_buffer/output_buffer_size: buffer to compress into
+// compression_level: 0-9
+// returns the size of the compressed data, even if it didn't fit into the output buffer
+size_t upkr_compress(void* output_buffer, size_t output_buffer_size, void* input_buffer, size_t input_size, int compression_level);
+
+// input_buffer/input_size: compressed data
+// output_buffer/output_buffer_size: buffer to uncompress into
+// return value:
+//  >= 0 : size of uncompressed data, even if it didn't fit into the output buffer
+//  < 0  : input data corrupt, unable to decompress
+ptrdiff_t upkr_uncompress(void* output_buffer, size_t output_buffer_size, void* input_buffer, size_t input_size);
+
+#ifdef __cplusplus
+}
+#endif
+#endif

dos_unpacker/unpack_x86_16_DOS.asm

@@ -105,15 +105,14 @@ upkr_decode_bit:
     dec     dx
     jns     upkr_load_bit
     movzx   ax, byte [bx]   ; u16 prob = upkr_probs[context_index]
+    neg     byte [bx]
     push    ax              ; save prob, tmp = prob
     cmp     dl, al          ; int bit = (upkr_state & 255) < prob ? 1 : 0; (carry = bit)
     pushf                   ; save bit flags
     jc      .bit            ; (skip if bit)
-    neg     al              ;   tmp = 256 - tmp;
+    xchg    [bx], al        ;   tmp = 256 - tmp;
 .bit:
-    mov     [bx], al        ; upkr_probs[context_index] = tmp + (256 - tmp + 8) >> 4;
-    neg     byte [bx]
-    shr     byte [bx], 4
+    shr     byte [bx], 4    ; upkr_probs[context_index] = tmp + (256 - tmp + 8) >> 4;
     adc     [bx], al
     mul     dh              ; upkr_state = tmp * (upkr_state >> 8) + (upkr_state & 255);
     mov     dh, 0

dos_unpacker/unpack_x86_16_DOS_no_relocation.asm

@@ -95,16 +95,15 @@ upkr_decode_bit:
     inc     dx
     dec     dx              ; inc dx, dec dx is used to test the top (sign) bit of dx
     jns     upkr_load_bit
-    movzx   ax, byte [bx]   ; int prob = upkr_probs[context_index]
-    push    ax              ; save prob
+    movzx   ax, byte [bx]   ; u16 prob = upkr_probs[context_index]
+    neg     byte [bx]
+    push    ax              ; save prob, tmp = prob
     cmp     dl, al          ; int bit = (upkr_state & 255) < prob ? 1 : 0; (carry = bit)
     pushf                   ; save bit flags
     jc      .bit            ; (skip if bit)
-    neg     al              ;   tmp = 256 - tmp;
+    xchg    [bx], al        ;   tmp = 256 - tmp;
 .bit:
-    mov     [bx], al        ; tmp_new = tmp + (256 - tmp + 8) >> 4;
-    neg     byte [bx]
-    shr     byte [bx], 4
+    shr     byte [bx], 4    ; upkr_probs[context_index] = tmp + (256 - tmp + 8) >> 4;
     adc     [bx], al
     mul     dh              ; upkr_state = tmp * (upkr_state >> 8) + (upkr_state & 255);
     mov     dh, 0

dos_unpacker/unpack_x86_16_DOS_no_repeated_offset.asm

@@ -98,16 +98,15 @@ upkr_decode_bit:
     inc     dx
     dec     dx              ; or whatever other test for the top bit there is
     jns     upkr_load_bit
-    movzx   ax, byte [bx]   ; int prob = upkr_probs[context_index]
-    push    ax              ; save prob
+    movzx   ax, byte [bx]   ; u16 prob = upkr_probs[context_index]
+    neg     byte [bx]
+    push    ax              ; save prob, tmp = prob
     cmp     dl, al          ; int bit = (upkr_state & 255) < prob ? 1 : 0; (carry = bit)
     pushf                   ; save bit flags
     jc      .bit            ; (skip if bit)
-    neg     al              ;   tmp = 256 - tmp;
+    xchg    [bx], al        ;   tmp = 256 - tmp;
 .bit:
-    mov     [bx], al        ; tmp += (256 - tmp + 8) >> 4;
-    neg     byte [bx]
-    shr     byte [bx],4
+    shr     byte [bx], 4    ; upkr_probs[context_index] = tmp + (256 - tmp + 8) >> 4;
     adc     [bx], al        ; upkr_probs[context_index] = tmp;
     mul     dh              ; upkr_state = tmp * (upkr_state >> 8) + (upkr_state & 255);
     mov     dh, 0

release/.gitignore

@@ -2,3 +2,4 @@
 *.tgz
 upkr-linux/
 upkr-windows/
+upkr-windows-32/

release/Makefile

@@ -1,11 +1,12 @@
 VERSION := $(shell cargo run --release -- --version)
 
-all: clean upkr-linux-$(VERSION).tgz upkr-windows-$(VERSION).zip
+all: clean upkr-linux-$(VERSION).tgz upkr-windows-$(VERSION).zip upkr-windows-32-$(VERSION).zip
 
 clean:
 	rm -rf upkr-linux
 	rm -f upkr-linux*.tgz
 	rm -rf upkr-windows
+	rm -rf upkr-windows-32
 	rm -f upkr-windows*.zip
 
 upkr-linux-$(VERSION).tgz: upkr-linux/upkr PHONY
@@ -22,14 +23,29 @@ upkr-windows-$(VERSION).zip: upkr-windows/upkr.exe PHONY
 	cd .. && git archive HEAD asm_unpackers | tar -xC release/upkr-windows
 	zip -r -9 $@ upkr-windows
 
+upkr-windows-32-$(VERSION).zip: upkr-windows-32/upkr.exe PHONY
+	cp ../README.md upkr-windows-32/
+	cd .. && git archive HEAD c_unpacker | tar -xC release/upkr-windows-32
+	cd .. && git archive HEAD z80_unpacker | tar -xC release/upkr-windows-32
+	cd .. && git archive HEAD asm_unpackers | tar -xC release/upkr-windows-32
+	zip -r -9 $@ upkr-windows-32
+
 upkr-linux/upkr:
-	cargo build --target x86_64-unknown-linux-musl --release -F crossterm
+	cargo build --target x86_64-unknown-linux-musl --release -F terminal
 	mkdir -p upkr-linux
 	cp ../target/x86_64-unknown-linux-musl/release/upkr upkr-linux/
+	strip upkr-linux/upkr
 
 upkr-windows/upkr.exe:
-	cargo build --target x86_64-pc-windows-gnu --release -F crossterm
+	cargo build --target x86_64-pc-windows-gnu --release -F terminal
 	mkdir -p upkr-windows
 	cp ../target/x86_64-pc-windows-gnu/release/upkr.exe upkr-windows/
+	x86_64-w64-mingw32-strip upkr-windows/upkr.exe
+
+upkr-windows-32/upkr.exe:
+	cargo build --target i686-pc-windows-gnu --release -F terminal
+	mkdir -p upkr-windows-32
+	cp ../target/i686-pc-windows-gnu/release/upkr.exe upkr-windows-32/
+	i686-w64-mingw32-strip upkr-windows-32/upkr.exe
 	
-PHONY:
+PHONY:

src/greedy_packer.rs

@@ -12,7 +12,7 @@ pub fn pack(
     let mut rans_coder = RansCoder::new(config);
     let mut state = lz::CoderState::new(config);
 
-    let mut pos = 0;
+    let mut pos = config.dictionary_size;
     while pos < data.len() {
         if let Some(ref mut cb) = progress_callback {
             cb(pos);

src/heatmap.rs

@@ -1,25 +1,38 @@
+/// Heatmap information about a compressed block of data.
+///
+/// For each byte in the uncompressed data, the heatmap provides two pieces of intormation:
+/// 1. whether this byte was encoded as a literal or as part of a match
+/// 2. how many (fractional) bits where spend on encoding this byte
+///
+/// For the sake of the heatmap, the cost of literals are spread out across all matches
+/// that reference the literal.
+///
+/// If the `terminal` feature is enabled, there is a function to write out the
+/// heatmap as a colored hexdump.
 pub struct Heatmap {
     data: Vec<u8>,
     cost: Vec<f32>,
+    raw_cost: Vec<f32>,
     literal_index: Vec<usize>,
 }
 
 impl Heatmap {
-    pub fn new() -> Heatmap {
+    pub(crate) fn new() -> Heatmap {
         Heatmap {
             data: Vec::new(),
             cost: Vec::new(),
+            raw_cost: Vec::new(),
             literal_index: Vec::new(),
         }
     }
 
-    pub fn add_literal(&mut self, byte: u8, cost: f32) {
+    pub(crate) fn add_literal(&mut self, byte: u8, cost: f32) {
         self.data.push(byte);
         self.cost.push(cost);
         self.literal_index.push(self.literal_index.len());
     }
 
-    pub fn add_match(&mut self, offset: usize, length: usize, mut cost: f32) {
+    pub(crate) fn add_match(&mut self, offset: usize, length: usize, mut cost: f32) {
         cost /= length as f32;
         for _ in 0..length {
             self.data.push(self.data[self.data.len() - offset]);
@@ -29,7 +42,9 @@ impl Heatmap {
         }
     }
 
-    pub fn finish(&mut self) {
+    pub(crate) fn finish(&mut self) {
+        self.raw_cost = self.cost.clone();
+
         let mut ref_count = vec![0usize; self.literal_index.len()];
         for &index in &self.literal_index {
             ref_count[index] += 1;
@@ -47,6 +62,7 @@ impl Heatmap {
         }
     }
 
+    /// Reverses the heatmap
     pub fn reverse(&mut self) {
         self.data.reverse();
         self.cost.reverse();
@@ -56,24 +72,47 @@ impl Heatmap {
         }
     }
 
+    /// The number of (uncompressed) bytes of data in this heatmap
     pub fn len(&self) -> usize {
         self.cost.len()
     }
 
+    /// Returns whether the byte at `index` was encoded as a literal
     pub fn is_literal(&self, index: usize) -> bool {
         self.literal_index[index] == index
     }
 
+    /// Returns the cost of encoding the byte at `index` in (fractional) bits.
+    /// The cost of literal bytes is spread across the matches that reference it.
+    /// See `raw_cost` for the raw encoding cost of each byte.
     pub fn cost(&self, index: usize) -> f32 {
         self.cost[index]
     }
 
+    /// Returns the raw cost of encoding the byte at `index` in (fractional) bits
+    pub fn raw_cost(&self, index: usize) -> f32 {
+        self.raw_cost[index]
+    }
+
+    /// Returns the uncompressed data byte at `index`
     pub fn byte(&self, index: usize) -> u8 {
         self.data[index]
     }
 
     #[cfg(feature = "crossterm")]
+    /// Print the heatmap as a colored hexdump
     pub fn print_as_hex(&self) -> std::io::Result<()> {
+        self.print_as_hex_internal(false)
+    }
+
+    #[cfg(feature = "crossterm")]
+    /// Print the heatmap as a colored hexdump, based on `raw_cost`.
+    pub fn print_as_hex_raw_cost(&self) -> std::io::Result<()> {
+        self.print_as_hex_internal(true)
+    }
+
+    #[cfg(feature = "crossterm")]
+    fn print_as_hex_internal(&self, report_raw_cost: bool) -> std::io::Result<()> {
         use crossterm::{
             style::{Attribute, Color, Print, SetAttribute, SetBackgroundColor},
             QueueableCommand,
@@ -85,8 +124,13 @@ impl Heatmap {
             heatmap: &Heatmap,
             index: usize,
             num_colors: u16,
+            report_raw_cost: bool,
         ) -> std::io::Result<()> {
-            let cost = heatmap.cost(index);
+            let cost = if report_raw_cost {
+                heatmap.raw_cost(index)
+            } else {
+                heatmap.cost(index)
+            };
             if num_colors < 256 {
                 let colors = [
                     Color::Red,
@@ -132,7 +176,7 @@ impl Heatmap {
             stdout.queue(Print(&format!("{:04x}  ", row_start)))?;
 
             for i in row_range.clone() {
-                set_color(&mut stdout, self, i, num_colors)?;
+                set_color(&mut stdout, self, i, num_colors, report_raw_cost)?;
                 stdout.queue(Print(&format!("{:02x} ", self.data[i])))?;
             }
 
@@ -143,7 +187,7 @@ impl Heatmap {
                 .queue(Print(&gap))?;
 
             for i in row_range.clone() {
-                set_color(&mut stdout, self, i, num_colors)?;
+                set_color(&mut stdout, self, i, num_colors, report_raw_cost)?;
                 let byte = self.data[i];
                 if byte >= 32 && byte < 127 {
                     stdout.queue(Print(format!("{}", byte as char)))?;

src/lib.rs

@@ -1,4 +1,4 @@
-#![warn(missing_docs)]
+#![deny(missing_docs)]
 
 //! Compression and decompression of the upkr format and variants.
 //!
@@ -71,6 +71,9 @@ pub struct Config {
     pub max_offset: usize,
     /// The maximum match length value to encode when compressing.
     pub max_length: usize,
+
+    /// Size of dictionary at the beginning of data (how many bytes to skip when compressing.)
+    pub dictionary_size: usize,
 }
 
 impl Default for Config {
@@ -92,6 +95,8 @@ impl Default for Config {
 
             max_offset: usize::MAX,
             max_length: usize::MAX,
+
+            dictionary_size: 0,
         }
     }
 }

src/lz.rs

@@ -133,21 +133,47 @@ impl CoderState {
     }
 }
 
+/// The error type for the uncompressing related functions
 #[derive(Error, Debug)]
 pub enum UnpackError {
+    /// a match offset pointing beyond the start of the unpacked data was encountered
     #[error("match offset out of range: {offset} > {position}")]
-    OffsetOutOfRange { offset: usize, position: usize },
+    OffsetOutOfRange {
+        /// the match offset
+        offset: usize,
+        /// the current position in the uncompressed stream
+        position: usize,
+    },
+    /// The passed size limit was exceeded
     #[error("Unpacked data over size limit: {size} > {limit}")]
-    OverSize { size: usize, limit: usize },
+    OverSize {
+        /// the size of the uncompressed data
+        size: usize,
+        /// the size limit passed into the function
+        limit: usize,
+    },
+    /// The end of the packed data was reached without an encoded EOF marker
     #[error("Unexpected end of input data")]
     UnexpectedEOF {
         #[from]
+        /// the underlying EOF error in the rANS decoder
         source: crate::rans::UnexpectedEOF,
     },
+    /// An offset or length value was found that exceeded 32bit
     #[error("Overflow while reading value")]
     ValueOverflow,
 }
 
+/// Uncompress a piece of compressed data
+///
+/// Returns either the uncompressed data, or an `UnpackError`
+///
+/// # Parameters
+///
+/// - `packed_data`: the compressed data
+/// - `config`: the exact compression format config used to compress the data
+/// - `max_size`: the maximum size of uncompressed data to return. When this is exceeded,
+///   `UnpackError::OverSize` is returned
 pub fn unpack(
     packed_data: &[u8],
     config: &Config,
@@ -158,10 +184,22 @@ pub fn unpack(
     Ok(result)
 }
 
+/// Calculates the minimum margin when overlapping buffers.
+///
+/// Returns the minimum margin needed between the end of the compressed data and the
+/// end of the uncompressed data when overlapping the two buffers to save on RAM.
 pub fn calculate_margin(packed_data: &[u8], config: &Config) -> Result<isize, UnpackError> {
     unpack_internal(None, None, packed_data, config, usize::MAX)
 }
 
+/// Calculates a `Heatmap` from compressed data.
+///
+/// # Parameters
+///
+/// - `packed_data`: the compressed data
+/// - `config`: the exact compression format config used to compress the data
+/// - `max_size`: the maximum size of the heatmap to return. When this is exceeded,
+///   `UnpackError::OverSize` is returned
 pub fn create_heatmap(
     packed_data: &[u8],
     config: &Config,

src/main.rs

@@ -10,11 +10,13 @@ fn main() -> Result<()> {
     let mut unpack = false;
     let mut calculate_margin = false;
     let mut create_heatmap = false;
+    let mut report_raw_cost = false;
     #[allow(unused_mut)]
     let mut do_hexdump = false;
     let mut level = 2;
     let mut infile: Option<PathBuf> = None;
     let mut outfile: Option<PathBuf> = None;
+    let mut dictionary: Option<PathBuf> = None;
     let mut max_unpacked_size = 512 * 1024 * 1024;
 
     let mut parser = lexopt::Parser::from_env();
@@ -59,9 +61,10 @@ fn main() -> Result<()> {
                 level = 9;
             }
 
-            Short('u') | Long("unpack") => unpack = true,
+            Short('u') | Long("unpack") | Short('d') | Long("decompress") => unpack = true,
             Long("margin") => calculate_margin = true,
             Long("heatmap") => create_heatmap = true,
+            Long("raw-cost") => report_raw_cost = true,
             #[cfg(feature = "crossterm")]
             Long("hexdump") => do_hexdump = true,
             Short('l') | Long("level") => level = parser.value()?.parse()?,
@@ -72,44 +75,15 @@ fn main() -> Result<()> {
                 process::exit(0);
             }
             Long("max-unpacked-size") => max_unpacked_size = parser.value()?.parse()?,
+            Long("dictionary") => dictionary = Some(parser.value()?.try_into()?),
             Value(val) if infile.is_none() => infile = Some(val.try_into()?),
             Value(val) if outfile.is_none() => outfile = Some(val.try_into()?),
             _ => return Err(arg.unexpected().into()),
         }
     }
 
-    let infile = infile.unwrap_or_else(|| print_help(1));
-    enum OutFileType {
-        Packed,
-        Unpacked,
-        Heatmap,
-    }
-    let outfile = |tpe: OutFileType| {
-        outfile.clone().unwrap_or_else(|| {
-            let mut name = infile.clone();
-            match tpe {
-                OutFileType::Packed => {
-                    let mut filename = name
-                        .file_name()
-                        .unwrap_or_else(|| OsStr::new(""))
-                        .to_os_string();
-                    filename.push(".upk");
-                    name.set_file_name(filename);
-                }
-                OutFileType::Unpacked => {
-                    if name.extension().filter(|&e| e == "upk").is_some() {
-                        name.set_extension("");
-                    } else {
-                        name.set_extension("bin");
-                    }
-                }
-                OutFileType::Heatmap => {
-                    name.set_extension("heatmap");
-                }
-            }
-            name
-        })
-    };
+    let infile = IoTarget::from_filename(infile);
+    let outfile = |tpe: OutFileType| infile.output(tpe, &outfile);
 
     if config.parity_contexts != 1 && config.parity_contexts != 2 && config.parity_contexts != 4 {
         eprintln!("--parity has to be 1, 2, or 4");
@@ -117,38 +91,52 @@ fn main() -> Result<()> {
     }
 
     if !unpack && !calculate_margin && !create_heatmap {
-        let mut data = vec![];
-        File::open(&infile)?.read_to_end(&mut data)?;
+        let mut data = infile.read()?;
         if reverse {
             data.reverse();
         }
 
-        let mut pb = pbr::ProgressBar::new(data.len() as u64);
-        pb.set_units(pbr::Units::Bytes);
-        let mut packed_data = upkr::pack(
-            &data,
-            level,
-            &config,
-            Some(&mut |pos| {
-                pb.set(pos as u64);
-            }),
-        );
-        pb.finish();
+        if let Some(dictionary) = dictionary {
+            let mut dict = vec![];
+            File::open(dictionary)?.read_to_end(&mut dict)?;
+            config.dictionary_size = dict.len();
+            // prepend dict
+            dict.append(&mut data);
+            data = dict;
+        }
+
+        #[cfg(feature = "terminal")]
+        let mut packed_data = {
+            let mut pb = pbr::ProgressBar::on(std::io::stderr(), data.len() as u64);
+            pb.set_units(pbr::Units::Bytes);
+            let packed_data = upkr::pack(
+                &data,
+                level,
+                &config,
+                Some(&mut |pos| {
+                    pb.set(pos as u64);
+                }),
+            );
+            pb.finish();
+            eprintln!();
+            packed_data
+        };
+        #[cfg(not(feature = "terminal"))]
+        let mut packed_data = upkr::pack(&data, level, &config, None);
 
         if reverse {
             packed_data.reverse();
         }
 
-        println!(
+        eprintln!(
             "Compressed {} bytes to {} bytes ({}%)",
             data.len(),
             packed_data.len(),
             packed_data.len() as f32 * 100. / data.len() as f32
         );
-        File::create(outfile(OutFileType::Packed))?.write_all(&packed_data)?;
+        outfile(OutFileType::Packed).write(&packed_data)?;
     } else {
-        let mut data = vec![];
-        File::open(&infile)?.read_to_end(&mut data)?;
+        let mut data = infile.read()?;
         if reverse {
             data.reverse();
         }
@@ -157,7 +145,7 @@ fn main() -> Result<()> {
             if reverse {
                 unpacked_data.reverse();
             }
-            File::create(outfile(OutFileType::Unpacked))?.write_all(&unpacked_data)?;
+            outfile(OutFileType::Unpacked).write(&unpacked_data)?;
         }
         if create_heatmap {
             let mut heatmap = upkr::create_heatmap(&data, &config, max_unpacked_size)?;
@@ -166,17 +154,25 @@ fn main() -> Result<()> {
             }
             match do_hexdump {
                 #[cfg(feature = "crossterm")]
-                true => heatmap.print_as_hex()?,
+                true => {
+                    if report_raw_cost {
+                        heatmap.print_as_hex_raw_cost()?
+                    } else {
+                        heatmap.print_as_hex()?
+                    }
+                }
                 _ => {
                     let mut heatmap_bin = Vec::with_capacity(heatmap.len());
                     for i in 0..heatmap.len() {
-                        let cost = (heatmap.cost(i).log2() * 8. + 64.)
-                            .round()
-                            .max(0.)
-                            .min(127.) as u8;
+                        let cost = if report_raw_cost {
+                            heatmap.raw_cost(i)
+                        } else {
+                            heatmap.cost(i)
+                        };
+                        let cost = (cost.log2() * 8. + 64.).round().max(0.).min(127.) as u8;
                         heatmap_bin.push((cost << 1) | heatmap.is_literal(i) as u8);
                     }
-                    File::create(outfile(OutFileType::Heatmap))?.write_all(&heatmap_bin)?;
+                    outfile(OutFileType::Heatmap).write(&heatmap_bin)?;
                 }
             }
         }
@@ -188,6 +184,81 @@ fn main() -> Result<()> {
     Ok(())
 }
 
+enum OutFileType {
+    Packed,
+    Unpacked,
+    Heatmap,
+}
+
+enum IoTarget {
+    StdInOut,
+    File(PathBuf),
+}
+
+impl IoTarget {
+    fn from_filename(filename: Option<PathBuf>) -> IoTarget {
+        if let Some(path) = filename {
+            if path.as_os_str() == "-" {
+                IoTarget::StdInOut
+            } else {
+                IoTarget::File(path)
+            }
+        } else {
+            IoTarget::StdInOut
+        }
+    }
+
+    fn read(&self) -> Result<Vec<u8>> {
+        let mut buffer = vec![];
+        match *self {
+            IoTarget::StdInOut => std::io::stdin().read_to_end(&mut buffer)?,
+            IoTarget::File(ref path) => File::open(path)?.read_to_end(&mut buffer)?,
+        };
+        Ok(buffer)
+    }
+
+    fn write(&self, data: &[u8]) -> Result<()> {
+        match *self {
+            IoTarget::StdInOut => std::io::stdout().write_all(data)?,
+            IoTarget::File(ref path) => File::create(path)?.write_all(data)?,
+        };
+        Ok(())
+    }
+
+    fn output(&self, tpe: OutFileType, outname: &Option<PathBuf>) -> IoTarget {
+        if outname.is_some() {
+            return IoTarget::from_filename(outname.clone());
+        }
+        match *self {
+            IoTarget::StdInOut => IoTarget::StdInOut,
+            IoTarget::File(ref path) => {
+                let mut name = path.clone();
+                match tpe {
+                    OutFileType::Packed => {
+                        let mut filename = name
+                            .file_name()
+                            .unwrap_or_else(|| OsStr::new(""))
+                            .to_os_string();
+                        filename.push(".upk");
+                        name.set_file_name(filename);
+                    }
+                    OutFileType::Unpacked => {
+                        if name.extension().filter(|&e| e == "upk").is_some() {
+                            name.set_extension("");
+                        } else {
+                            name.set_extension("bin");
+                        }
+                    }
+                    OutFileType::Heatmap => {
+                        name.set_extension("heatmap");
+                    }
+                }
+                IoTarget::File(name)
+            }
+        }
+    }
+}
+
 fn print_help(exit_code: i32) -> ! {
     eprintln!("Usage:");
     eprintln!("  upkr [-l level(0-9)] [config options] <infile> [<outfile>]");
@@ -197,10 +268,18 @@ fn print_help(exit_code: i32) -> ! {
     eprintln!();
     eprintln!(" -l, --level N       compression level 0-9");
     eprintln!(" -0, ..., -9         short form for setting compression level");
-    eprintln!(" -u, --unpack        unpack infile");
+    eprintln!(" -d, --decompress    decompress infile");
     eprintln!(" --heatmap           calculate heatmap from compressed file");
+    eprintln!("   --raw-cost        report raw cost of literals in heatmap");
+    #[cfg(feature = "crossterm")]
+    eprintln!("   --hexdump         print heatmap as colored hexdump");
     eprintln!(" --margin            calculate margin for overlapped unpacking of a packed file");
     eprintln!();
+    eprintln!("When no infile is given, or the infile is '-', read from stdin.");
+    eprintln!(
+        "When no outfile is given and reading from stdin, or when outfile is '-', write to stdout."
+    );
+    eprintln!();
     eprintln!("Version: {}", env!("CARGO_PKG_VERSION"));
     eprintln!();
     eprintln!("Config presets for specific unpackers:");

src/parsing_packer.rs

@@ -137,7 +137,7 @@ fn parse(
     }
     add_arrival(
         &mut arrivals,
-        0,
+        encoding_config.dictionary_size,
         Arrival {
             parse: None,
             state: lz::CoderState::new(encoding_config),
@@ -148,7 +148,7 @@ fn parse(
 
     let cost_counter = &mut CostCounter::new(encoding_config);
     let mut best_per_offset = HashMap::new();
-    for pos in 0..data.len() {
+    for pos in encoding_config.dictionary_size..data.len() {
         let match_length = |offset: usize| {
             data[pos..]
                 .iter()