implement dictionary support when packing

Cargo.lock

@@ -308,7 +308,7 @@ checksum = "dcc811dc4066ac62f84f11307873c4850cb653bfa9b1719cee2bd2204a4bc5dd"
 
 [[package]]
 name = "upkr"
-version = "0.2.1"
+version = "0.2.2"
 dependencies = [
  "anyhow",
  "cdivsufsort",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "upkr"
-version = "0.2.1"
+version = "0.2.2"
 edition = "2021"
 
 [profile.release]

README.md

@@ -22,6 +22,12 @@ c_unpacker and asm_unpackers unpack the default upkr compressed format. The z80_
 is based on some variations to the compressed format. (Use `upkr --z80` to select those variations.)
 The 16 bit dos unpacker also uses some variations. (`upkr --x86`)
 
+### More unpackers outside this repository
+
+* [Atari Lynx](https://github.com/42Bastian/new_bll/blob/master/demos/depacker/unupkr.asm)
+* [Atari Jaguar](https://github.com/42Bastian/new_bjl/blob/main/exp/depacker/unupkr.js)
+* [8080, R800](https://github.com/ivagorRetrocomp/DeUpkr)
+
 ## Usage
 
 ```
@@ -32,10 +38,17 @@ The 16 bit dos unpacker also uses some variations. (`upkr --x86`)
 
  -l, --level N       compression level 0-9
  -0, ..., -9         short form for setting compression level
- -u, --unpack        unpack infile
+ -d, --decompress    decompress infile
  --heatmap           calculate heatmap from compressed file
+   --raw-cost        report raw cost of literals in heatmap
+                     (the cost of literals is spread across all matches
+                     that reference the literal by default.)
+   --hexdump         print heatmap as colored hexdump
  --margin            calculate margin for overlapped unpacking of a packed file
 
+When no infile is given, or the infile is '-', read from stdin.
+When no outfile is given and reading from stdin, or when outfile is '-', write to stdout.
+
 Config presets for specific unpackers:
  --z80               --big-endian-bitstream --invert-bit-encoding --simplified-prob-update -9
  --x86               --bitstream --invert-is-match-bit --invert-continue-value-bit --invert-new-offset-bit

release/.gitignore

@@ -2,3 +2,4 @@
 *.tgz
 upkr-linux/
 upkr-windows/
+upkr-windows-32/

release/Makefile

@@ -1,11 +1,12 @@
 VERSION := $(shell cargo run --release -- --version)
 
-all: clean upkr-linux-$(VERSION).tgz upkr-windows-$(VERSION).zip
+all: clean upkr-linux-$(VERSION).tgz upkr-windows-$(VERSION).zip upkr-windows-32-$(VERSION).zip
 
 clean:
 	rm -rf upkr-linux
 	rm -f upkr-linux*.tgz
 	rm -rf upkr-windows
+	rm -rf upkr-windows-32
 	rm -f upkr-windows*.zip
 
 upkr-linux-$(VERSION).tgz: upkr-linux/upkr PHONY
@@ -22,14 +23,29 @@ upkr-windows-$(VERSION).zip: upkr-windows/upkr.exe PHONY
 	cd .. && git archive HEAD asm_unpackers | tar -xC release/upkr-windows
 	zip -r -9 $@ upkr-windows
 
+upkr-windows-32-$(VERSION).zip: upkr-windows-32/upkr.exe PHONY
+	cp ../README.md upkr-windows-32/
+	cd .. && git archive HEAD c_unpacker | tar -xC release/upkr-windows-32
+	cd .. && git archive HEAD z80_unpacker | tar -xC release/upkr-windows-32
+	cd .. && git archive HEAD asm_unpackers | tar -xC release/upkr-windows-32
+	zip -r -9 $@ upkr-windows-32
+
 upkr-linux/upkr:
 	cargo build --target x86_64-unknown-linux-musl --release -F terminal
 	mkdir -p upkr-linux
 	cp ../target/x86_64-unknown-linux-musl/release/upkr upkr-linux/
+	strip upkr-linux/upkr
 
 upkr-windows/upkr.exe:
 	cargo build --target x86_64-pc-windows-gnu --release -F terminal
 	mkdir -p upkr-windows
 	cp ../target/x86_64-pc-windows-gnu/release/upkr.exe upkr-windows/
+	x86_64-w64-mingw32-strip upkr-windows/upkr.exe
+
+upkr-windows-32/upkr.exe:
+	cargo build --target i686-pc-windows-gnu --release -F terminal
+	mkdir -p upkr-windows-32
+	cp ../target/i686-pc-windows-gnu/release/upkr.exe upkr-windows-32/
+	i686-w64-mingw32-strip upkr-windows-32/upkr.exe
 	
-PHONY:
+PHONY:

src/greedy_packer.rs

@@ -12,7 +12,7 @@ pub fn pack(
     let mut rans_coder = RansCoder::new(config);
     let mut state = lz::CoderState::new(config);
 
-    let mut pos = 0;
+    let mut pos = config.dictionary_size;
     while pos < data.len() {
         if let Some(ref mut cb) = progress_callback {
             cb(pos);

src/heatmap.rs

@@ -12,6 +12,7 @@
 pub struct Heatmap {
     data: Vec<u8>,
     cost: Vec<f32>,
+    raw_cost: Vec<f32>,
     literal_index: Vec<usize>,
 }
 
@@ -20,6 +21,7 @@ impl Heatmap {
         Heatmap {
             data: Vec::new(),
             cost: Vec::new(),
+            raw_cost: Vec::new(),
             literal_index: Vec::new(),
         }
     }
@@ -41,6 +43,8 @@ impl Heatmap {
     }
 
     pub(crate) fn finish(&mut self) {
+        self.raw_cost = self.cost.clone();
+
         let mut ref_count = vec![0usize; self.literal_index.len()];
         for &index in &self.literal_index {
             ref_count[index] += 1;
@@ -78,11 +82,18 @@ impl Heatmap {
         self.literal_index[index] == index
     }
 
-    /// Returns the cost of encoding the byte at `index` in (fractional) bits
+    /// Returns the cost of encoding the byte at `index` in (fractional) bits.
+    /// The cost of literal bytes is spread across the matches that reference it.
+    /// See `raw_cost` for the raw encoding cost of each byte.
     pub fn cost(&self, index: usize) -> f32 {
         self.cost[index]
     }
 
+    /// Returns the raw cost of encoding the byte at `index` in (fractional) bits
+    pub fn raw_cost(&self, index: usize) -> f32 {
+        self.raw_cost[index]
+    }
+
     /// Returns the uncompressed data byte at `index`
     pub fn byte(&self, index: usize) -> u8 {
         self.data[index]
@@ -91,6 +102,17 @@ impl Heatmap {
     #[cfg(feature = "crossterm")]
     /// Print the heatmap as a colored hexdump
     pub fn print_as_hex(&self) -> std::io::Result<()> {
+        self.print_as_hex_internal(false)
+    }
+
+    #[cfg(feature = "crossterm")]
+    /// Print the heatmap as a colored hexdump, based on `raw_cost`.
+    pub fn print_as_hex_raw_cost(&self) -> std::io::Result<()> {
+        self.print_as_hex_internal(true)
+    }
+
+    #[cfg(feature = "crossterm")]
+    fn print_as_hex_internal(&self, report_raw_cost: bool) -> std::io::Result<()> {
         use crossterm::{
             style::{Attribute, Color, Print, SetAttribute, SetBackgroundColor},
             QueueableCommand,
@@ -102,8 +124,13 @@ impl Heatmap {
             heatmap: &Heatmap,
             index: usize,
             num_colors: u16,
+            report_raw_cost: bool,
         ) -> std::io::Result<()> {
-            let cost = heatmap.cost(index);
+            let cost = if report_raw_cost {
+                heatmap.raw_cost(index)
+            } else {
+                heatmap.cost(index)
+            };
             if num_colors < 256 {
                 let colors = [
                     Color::Red,
@@ -149,7 +176,7 @@ impl Heatmap {
             stdout.queue(Print(&format!("{:04x}  ", row_start)))?;
 
             for i in row_range.clone() {
-                set_color(&mut stdout, self, i, num_colors)?;
+                set_color(&mut stdout, self, i, num_colors, report_raw_cost)?;
                 stdout.queue(Print(&format!("{:02x} ", self.data[i])))?;
             }
 
@@ -160,7 +187,7 @@ impl Heatmap {
                 .queue(Print(&gap))?;
 
             for i in row_range.clone() {
-                set_color(&mut stdout, self, i, num_colors)?;
+                set_color(&mut stdout, self, i, num_colors, report_raw_cost)?;
                 let byte = self.data[i];
                 if byte >= 32 && byte < 127 {
                     stdout.queue(Print(format!("{}", byte as char)))?;

src/lib.rs

@@ -71,6 +71,9 @@ pub struct Config {
     pub max_offset: usize,
     /// The maximum match length value to encode when compressing.
     pub max_length: usize,
+
+    /// Size of dictionary at the beginning of data (how many bytes to skip when compressing.)
+    pub dictionary_size: usize,
 }
 
 impl Default for Config {
@@ -92,6 +95,8 @@ impl Default for Config {
 
             max_offset: usize::MAX,
             max_length: usize::MAX,
+
+            dictionary_size: 0,
         }
     }
 }

src/main.rs

@@ -10,11 +10,13 @@ fn main() -> Result<()> {
     let mut unpack = false;
     let mut calculate_margin = false;
     let mut create_heatmap = false;
+    let mut report_raw_cost = false;
     #[allow(unused_mut)]
     let mut do_hexdump = false;
     let mut level = 2;
     let mut infile: Option<PathBuf> = None;
     let mut outfile: Option<PathBuf> = None;
+    let mut dictionary: Option<PathBuf> = None;
     let mut max_unpacked_size = 512 * 1024 * 1024;
 
     let mut parser = lexopt::Parser::from_env();
@@ -59,19 +61,21 @@ fn main() -> Result<()> {
                 level = 9;
             }
 
-            Short('u') | Long("unpack") => unpack = true,
+            Short('u') | Long("unpack") | Short('d') | Long("decompress") => unpack = true,
             Long("margin") => calculate_margin = true,
             Long("heatmap") => create_heatmap = true,
+            Long("raw-cost") => report_raw_cost = true,
             #[cfg(feature = "crossterm")]
             Long("hexdump") => do_hexdump = true,
             Short('l') | Long("level") => level = parser.value()?.parse()?,
             Short(n) if n.is_ascii_digit() => level = n as u8 - b'0',
             Short('h') | Long("help") => print_help(0),
             Long("version") => {
-                eprintln!("{}", env!("CARGO_PKG_VERSION"));
+                println!("{}", env!("CARGO_PKG_VERSION"));
                 process::exit(0);
             }
             Long("max-unpacked-size") => max_unpacked_size = parser.value()?.parse()?,
+            Long("dictionary") => dictionary = Some(parser.value()?.try_into()?),
             Value(val) if infile.is_none() => infile = Some(val.try_into()?),
             Value(val) if outfile.is_none() => outfile = Some(val.try_into()?),
             _ => return Err(arg.unexpected().into()),
@@ -92,9 +96,18 @@ fn main() -> Result<()> {
             data.reverse();
         }
 
+        if let Some(dictionary) = dictionary {
+            let mut dict = vec![];
+            File::open(dictionary)?.read_to_end(&mut dict)?;
+            config.dictionary_size = dict.len();
+            // prepend dict
+            dict.append(&mut data);
+            data = dict;
+        }
+
         #[cfg(feature = "terminal")]
         let mut packed_data = {
-            let mut pb = pbr::ProgressBar::new(data.len() as u64);
+            let mut pb = pbr::ProgressBar::on(std::io::stderr(), data.len() as u64);
             pb.set_units(pbr::Units::Bytes);
             let packed_data = upkr::pack(
                 &data,
@@ -105,6 +118,7 @@ fn main() -> Result<()> {
                 }),
             );
             pb.finish();
+            eprintln!();
             packed_data
         };
         #[cfg(not(feature = "terminal"))]
@@ -140,14 +154,22 @@ fn main() -> Result<()> {
             }
             match do_hexdump {
                 #[cfg(feature = "crossterm")]
-                true => heatmap.print_as_hex()?,
+                true => {
+                    if report_raw_cost {
+                        heatmap.print_as_hex_raw_cost()?
+                    } else {
+                        heatmap.print_as_hex()?
+                    }
+                }
                 _ => {
                     let mut heatmap_bin = Vec::with_capacity(heatmap.len());
                     for i in 0..heatmap.len() {
-                        let cost = (heatmap.cost(i).log2() * 8. + 64.)
-                            .round()
-                            .max(0.)
-                            .min(127.) as u8;
+                        let cost = if report_raw_cost {
+                            heatmap.raw_cost(i)
+                        } else {
+                            heatmap.cost(i)
+                        };
+                        let cost = (cost.log2() * 8. + 64.).round().max(0.).min(127.) as u8;
                         heatmap_bin.push((cost << 1) | heatmap.is_literal(i) as u8);
                     }
                     outfile(OutFileType::Heatmap).write(&heatmap_bin)?;
@@ -246,8 +268,11 @@ fn print_help(exit_code: i32) -> ! {
     eprintln!();
     eprintln!(" -l, --level N       compression level 0-9");
     eprintln!(" -0, ..., -9         short form for setting compression level");
-    eprintln!(" -u, --unpack        unpack infile");
+    eprintln!(" -d, --decompress    decompress infile");
     eprintln!(" --heatmap           calculate heatmap from compressed file");
+    eprintln!("   --raw-cost        report raw cost of literals in heatmap");
+    #[cfg(feature = "crossterm")]
+    eprintln!("   --hexdump         print heatmap as colored hexdump");
     eprintln!(" --margin            calculate margin for overlapped unpacking of a packed file");
     eprintln!();
     eprintln!("When no infile is given, or the infile is '-', read from stdin.");

src/parsing_packer.rs

@@ -137,7 +137,7 @@ fn parse(
     }
     add_arrival(
         &mut arrivals,
-        0,
+        encoding_config.dictionary_size,
         Arrival {
             parse: None,
             state: lz::CoderState::new(encoding_config),
@@ -148,7 +148,7 @@ fn parse(
 
     let cost_counter = &mut CostCounter::new(encoding_config);
     let mut best_per_offset = HashMap::new();
-    for pos in 0..data.len() {
+    for pos in encoding_config.dictionary_size..data.len() {
         let match_length = |offset: usize| {
             data[pos..]
                 .iter()