Merge branch 'ped7g-z80_ped7g'

missing 512 byte table generator, which doesn't look trivial to do
(especially in terms of code size).
Not tested, but looks as decent speed up.

Cargo.lock

@@ -95,6 +95,24 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "proc-macro2"
+version = "1.0.44"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7bd7356a8122b6c4a24a82b278680c73357984ca2fc79a0f9fa6dea7dced7c58"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179"
+dependencies = [
+ "proc-macro2",
+]
+
 [[package]]
 name = "sacabase"
 version = "2.0.0"
@@ -104,6 +122,37 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "syn"
+version = "1.0.101"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e90cde112c4b9690b8cbe810cba9ddd8bc1d7472e2cae317b69e9438c1cba7d2"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "thiserror"
+version = "1.0.36"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0a99cb8c4b9a8ef0e7907cd3b617cc8dc04d571c4e73c8ae403d80ac160bb122"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.36"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a891860d3c8d66fec8e73ddb3765f90082374dbaaa833407b904a94f1a7eb43"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "time"
 version = "0.1.44"
@@ -115,14 +164,21 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "unicode-ident"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dcc811dc4066ac62f84f11307873c4850cb653bfa9b1719cee2bd2204a4bc5dd"
+
 [[package]]
 name = "upkr"
-version = "0.2.0-pre2"
+version = "0.2.0-pre3"
 dependencies = [
  "anyhow",
  "cdivsufsort",
  "lexopt",
  "pbr",
+ "thiserror",
 ]
 
 [[package]]

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "upkr"
-version = "0.2.0-pre2"
+version = "0.2.0-pre3"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -9,4 +9,5 @@ edition = "2021"
 cdivsufsort = "2"
 lexopt = "0.2.1"
 anyhow = "1"
+thiserror = "1.0.36"
 pbr = "1"

fuzz/.gitignore

@@ -0,0 +1,3 @@
+target
+corpus
+artifacts

fuzz/Cargo.lock

@@ -0,0 +1,247 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "anyhow"
+version = "1.0.65"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "98161a4e3e2184da77bb14f02184cdd111e83bbbcc9979dfee3c44b9a85f5602"
+
+[[package]]
+name = "arbitrary"
+version = "1.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f44124848854b941eafdb34f05b3bcf59472f643c7e151eba7c2b69daa469ed5"
+
+[[package]]
+name = "autocfg"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
+
+[[package]]
+name = "cc"
+version = "1.0.73"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
+dependencies = [
+ "jobserver",
+]
+
+[[package]]
+name = "cdivsufsort"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "edefce019197609da416762da75bb000bbd2224b2d89a7e722c2296cbff79b8c"
+dependencies = [
+ "cc",
+ "sacabase",
+]
+
+[[package]]
+name = "cfg-if"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+
+[[package]]
+name = "crossbeam-channel"
+version = "0.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521"
+dependencies = [
+ "cfg-if",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51887d4adc7b564537b15adcfb307936f8075dfcd5f00dde9a9f1d29383682bc"
+dependencies = [
+ "cfg-if",
+ "once_cell",
+]
+
+[[package]]
+name = "jobserver"
+version = "0.1.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "068b1ee6743e4d11fb9c6a1e6064b3693a1b600e7f5f5988047d98b3dc9fb90b"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "lexopt"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "478ee9e62aaeaf5b140bd4138753d1f109765488581444218d3ddda43234f3e8"
+
+[[package]]
+name = "libc"
+version = "0.2.133"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0f80d65747a3e43d1596c7c5492d95d5edddaabd45a7fcdb02b95f644164966"
+
+[[package]]
+name = "libfuzzer-sys"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae185684fe19814afd066da15a7cc41e126886c21282934225d9fc847582da58"
+dependencies = [
+ "arbitrary",
+ "cc",
+ "once_cell",
+]
+
+[[package]]
+name = "num-traits"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1"
+
+[[package]]
+name = "pbr"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff5751d87f7c00ae6403eb1fcbba229b9c76c9a30de8c1cf87182177b168cea2"
+dependencies = [
+ "crossbeam-channel",
+ "libc",
+ "time",
+ "winapi",
+]
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.44"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7bd7356a8122b6c4a24a82b278680c73357984ca2fc79a0f9fa6dea7dced7c58"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "sacabase"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9883fc3d6ce3d78bb54d908602f8bc1f7b5f983afe601dabe083009d86267a84"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "syn"
+version = "1.0.101"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e90cde112c4b9690b8cbe810cba9ddd8bc1d7472e2cae317b69e9438c1cba7d2"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "thiserror"
+version = "1.0.36"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0a99cb8c4b9a8ef0e7907cd3b617cc8dc04d571c4e73c8ae403d80ac160bb122"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.36"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a891860d3c8d66fec8e73ddb3765f90082374dbaaa833407b904a94f1a7eb43"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "time"
+version = "0.1.44"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255"
+dependencies = [
+ "libc",
+ "wasi",
+ "winapi",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dcc811dc4066ac62f84f11307873c4850cb653bfa9b1719cee2bd2204a4bc5dd"
+
+[[package]]
+name = "upkr"
+version = "0.2.0-pre3"
+dependencies = [
+ "anyhow",
+ "cdivsufsort",
+ "lexopt",
+ "pbr",
+ "thiserror",
+]
+
+[[package]]
+name = "upkr-fuzz"
+version = "0.0.0"
+dependencies = [
+ "libfuzzer-sys",
+ "upkr",
+]
+
+[[package]]
+name = "wasi"
+version = "0.10.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"
+
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"

fuzz/Cargo.toml

@@ -0,0 +1,31 @@
+[package]
+name = "upkr-fuzz"
+version = "0.0.0"
+authors = ["Automatically generated"]
+publish = false
+edition = "2018"
+
+[package.metadata]
+cargo-fuzz = true
+
+[dependencies]
+libfuzzer-sys = "0.4"
+
+[dependencies.upkr]
+path = ".."
+
+# Prevent this from interfering with workspaces
+[workspace]
+members = ["."]
+
+[[bin]]
+name = "all_configs"
+path = "fuzz_targets/all_configs.rs"
+test = false
+doc = false
+
+[[bin]]
+name = "unpack"
+path = "fuzz_targets/unpack.rs"
+test = false
+doc = false

fuzz/fuzz_targets/all_configs.rs

@@ -0,0 +1,29 @@
+#![no_main]
+use libfuzzer_sys::fuzz_target;
+
+fuzz_target!(|data: &[u8]| {
+    let mut config = upkr::Config::default();
+    let mut level = 1;
+    let mut data = data;
+    if data.len() > 2 {
+        let flags1 = data[0];
+        let flags2 = data[1];
+        data = &data[2..];
+        config.use_bitstream = (flags1 & 1) != 0;
+        config.parity_contexts = if (flags1 & 2) == 0 { 1 } else { 2 };
+        config.invert_bit_encoding = (flags1 & 4) != 0;
+        config.is_match_bit = (flags1 & 8) != 0;
+        config.new_offset_bit = (flags1 & 16) != 0;
+        config.continue_value_bit = (flags1 & 32) != 0;
+        config.bitstream_is_big_endian = (flags1 & 64) != 0;
+        config.simplified_prob_update = (flags1 & 128) != 0;
+        config.no_repeated_offsets = (flags2 & 32) != 0;
+        config.eof_in_length = (flags2 & 1) != 0;
+        config.max_offset = if (flags2 & 2) == 0 { usize::MAX } else { 32 };
+        config.max_length = if (flags2 & 4) == 0 { usize::MAX } else { 5 };
+        level = (flags2 >> 3) & 3;
+    }
+    let packed = upkr::pack(data, level, &config, None);
+    let unpacked = upkr::unpack(&packed, &config, 1024 * 1024).unwrap();
+    assert!(unpacked == data);
+});

fuzz/fuzz_targets/unpack.rs

@@ -0,0 +1,6 @@
+#![no_main]
+use libfuzzer_sys::fuzz_target;
+
+fuzz_target!(|data: &[u8]| {
+    let _ = upkr::unpack(data, &upkr::Config::default(), 64 * 1024);
+});

src/greedy_packer.rs

@@ -19,15 +19,16 @@ pub fn pack(
         }
         let mut encoded_match = false;
         if let Some(m) = match_finder.matches(pos).next() {
-            let max_offset = 1 << (m.length * 3 - 1).min(31);
+            let max_offset = config.max_offset.min(1 << (m.length * 3 - 1).min(31));
             let offset = pos - m.pos;
             if offset < max_offset && m.length >= config.min_length() {
+                let length = m.length.min(config.max_length);
                 lz::Op::Match {
                     offset: offset as u32,
-                    len: m.length as u32,
+                    len: length as u32,
                 }
                 .encode(&mut rans_coder, &mut state, config);
-                pos += m.length;
+                pos += length;
                 encoded_match = true;
             }
         }
@@ -39,7 +40,8 @@ pub fn pack(
                     .iter()
                     .zip(data[(pos - offset)..].iter())
                     .take_while(|(a, b)| a == b)
-                    .count();
+                    .count()
+                    .min(config.max_length);
                 if length >= config.min_length() {
                     lz::Op::Match {
                         offset: offset as u32,

src/lib.rs

@@ -5,10 +5,11 @@ mod match_finder;
 mod parsing_packer;
 mod rans;
 
-pub use lz::{calculate_margin, unpack};
+pub use lz::{calculate_margin, unpack, UnpackError};
 
 pub type ProgressCallback<'a> = &'a mut dyn FnMut(usize);
 
+#[derive(Debug)]
 pub struct Config {
     pub use_bitstream: bool,
     pub parity_contexts: usize,
@@ -23,6 +24,9 @@ pub struct Config {
 
     pub no_repeated_offsets: bool,
     pub eof_in_length: bool,
+
+    pub max_offset: usize,
+    pub max_length: usize,
 }
 
 impl Default for Config {
@@ -41,6 +45,9 @@ impl Default for Config {
 
             no_repeated_offsets: false,
             eof_in_length: false,
+
+            max_offset: usize::MAX,
+            max_length: usize::MAX,
         }
     }
 }
@@ -58,13 +65,13 @@ impl Config {
 pub fn pack(
     data: &[u8],
     level: u8,
-    config: Config,
+    config: &Config,
     progress_callback: Option<ProgressCallback>,
 ) -> Vec<u8> {
     if level == 0 {
-        greedy_packer::pack(data, &config, progress_callback)
+        greedy_packer::pack(data, config, progress_callback)
     } else {
-        parsing_packer::pack(data, level, &config, progress_callback)
+        parsing_packer::pack(data, level, config, progress_callback)
     }
 }
 

src/lz.rs

@@ -1,6 +1,7 @@
 use crate::context_state::ContextState;
 use crate::rans::{EntropyCoder, RansDecoder};
 use crate::Config;
+use thiserror::Error;
 
 #[derive(Copy, Clone, Debug)]
 pub enum Op {
@@ -25,17 +26,18 @@ impl Op {
             }
             &Op::Match { offset, len } => {
                 encode_bit(coder, state, literal_base, config.is_match_bit);
+                let mut new_offset = true;
                 if !state.prev_was_match && !config.no_repeated_offsets {
+                    new_offset = offset != state.last_offset;
                     encode_bit(
                         coder,
                         state,
                         256 * state.parity_contexts,
-                        (offset != state.last_offset) == config.new_offset_bit,
+                        new_offset == config.new_offset_bit,
                     );
-                } else {
-                    assert!(offset != state.last_offset || config.no_repeated_offsets);
                 }
-                if offset != state.last_offset || config.no_repeated_offsets {
+                assert!(offset as usize <= config.max_offset);
+                if new_offset {
                     encode_length(
                         coder,
                         state,
@@ -45,7 +47,7 @@ impl Op {
                     );
                     state.last_offset = offset;
                 }
-                assert!(!config.eof_in_length || len > 1);
+                assert!(len as usize >= config.min_length() && len as usize <= config.max_length);
                 encode_length(coder, state, 256 * state.parity_contexts + 65, len, config);
                 state.prev_was_match = true;
                 state.pos += len as usize;
@@ -69,7 +71,7 @@ pub fn encode_eof(coder: &mut dyn EntropyCoder, state: &mut CoderState, config:
             config.new_offset_bit ^ config.eof_in_length,
         );
     }
-    if !config.eof_in_length || config.no_repeated_offsets {
+    if !config.eof_in_length || state.prev_was_match || config.no_repeated_offsets {
         encode_length(coder, state, 256 * state.parity_contexts + 1, 1, config);
     }
     if config.eof_in_length {
@@ -130,24 +132,44 @@ impl CoderState {
     }
 }
 
-pub fn unpack(packed_data: &[u8], config: &Config) -> Vec<u8> {
+#[derive(Error, Debug)]
-    let mut result = vec![];
+pub enum UnpackError {
-    let _ = unpack_internal(Some(&mut result), packed_data, config);
+    #[error("match offset out of range: {offset} > {position}")]
-    result
+    OffsetOutOfRange { offset: usize, position: usize },
+    #[error("Unpacked data over size limit: {size} > {limit}")]
+    OverSize { size: usize, limit: usize },
+    #[error("Unexpected end of input data")]
+    UnexpectedEOF {
+        #[from]
+        source: crate::rans::UnexpectedEOF,
+    },
+    #[error("Overflow while reading value")]
+    ValueOverflow,
 }
 
-pub fn calculate_margin(packed_data: &[u8], config: &Config) -> isize {
+pub fn unpack(
-    unpack_internal(None, packed_data, config)
+    packed_data: &[u8],
+    config: &Config,
+    max_size: usize,
+) -> Result<Vec<u8>, UnpackError> {
+    let mut result = vec![];
+    let _ = unpack_internal(Some(&mut result), packed_data, config, max_size)?;
+    Ok(result)
+}
+
+pub fn calculate_margin(packed_data: &[u8], config: &Config) -> Result<isize, UnpackError> {
+    unpack_internal(None, packed_data, config, usize::MAX)
 }
 
 pub fn unpack_internal(
     mut result: Option<&mut Vec<u8>>,
     packed_data: &[u8],
     config: &Config,
-) -> isize {
+    max_size: usize,
+) -> Result<isize, UnpackError> {
     let mut decoder = RansDecoder::new(packed_data, &config);
     let mut contexts = ContextState::new((1 + 255) * config.parity_contexts + 1 + 64 + 64, &config);
-    let mut offset = 0;
+    let mut offset = usize::MAX;
     let mut position = 0usize;
     let mut prev_was_match = false;
     let mut margin = 0isize;
@@ -157,31 +179,34 @@ pub fn unpack_internal(
         contexts: &mut ContextState,
         mut context_index: usize,
         config: &Config,
-    ) -> usize {
+    ) -> Result<usize, UnpackError> {
         let mut length = 0;
         let mut bit_pos = 0;
-        while decoder.decode_with_context(&mut contexts.context_mut(context_index))
+        while decoder.decode_with_context(&mut contexts.context_mut(context_index))?
             == config.continue_value_bit
         {
-            length |= (decoder.decode_with_context(&mut contexts.context_mut(context_index + 1))
+            length |= (decoder.decode_with_context(&mut contexts.context_mut(context_index + 1))?
                 as usize)
                 << bit_pos;
             bit_pos += 1;
+            if bit_pos >= 32 {
+                return Err(UnpackError::ValueOverflow);
+            }
             context_index += 2;
         }
-        length | (1 << bit_pos)
+        Ok(length | (1 << bit_pos))
     }
 
     loop {
         margin = margin.max(position as isize - decoder.pos() as isize);
         let literal_base = position % config.parity_contexts * 256;
-        if decoder.decode_with_context(&mut contexts.context_mut(literal_base))
+        if decoder.decode_with_context(&mut contexts.context_mut(literal_base))?
             == config.is_match_bit
         {
             if config.no_repeated_offsets
                 || prev_was_match
                 || decoder
-                    .decode_with_context(&mut contexts.context_mut(256 * config.parity_contexts))
+                    .decode_with_context(&mut contexts.context_mut(256 * config.parity_contexts))?
                     == config.new_offset_bit
             {
                 offset = decode_length(
@@ -189,7 +214,7 @@ pub fn unpack_internal(
                     &mut contexts,
                     256 * config.parity_contexts + 1,
                     &config,
-                ) - if config.eof_in_length { 0 } else { 1 };
+                )? - if config.eof_in_length { 0 } else { 1 };
                 if offset == 0 {
                     break;
                 }
@@ -199,13 +224,20 @@ pub fn unpack_internal(
                 &mut contexts,
                 256 * config.parity_contexts + 65,
                 &config,
-            );
+            )?;
             if config.eof_in_length && length == 1 {
                 break;
             }
+            if offset > position {
+                return Err(UnpackError::OffsetOutOfRange { offset, position });
+            }
             if let Some(ref mut result) = result {
                 for _ in 0..length {
+                    if result.len() < max_size {
                         result.push(result[result.len() - offset]);
+                    } else {
+                        break;
+                    }
                 }
             }
             position += length;
@@ -215,17 +247,26 @@ pub fn unpack_internal(
             let mut byte = 0;
             for i in (0..8).rev() {
                 let bit = decoder
-                    .decode_with_context(&mut contexts.context_mut(literal_base + context_index));
+                    .decode_with_context(&mut contexts.context_mut(literal_base + context_index))?;
                 context_index = (context_index << 1) | bit as usize;
                 byte |= (bit as u8) << i;
             }
             if let Some(ref mut result) = result {
+                if result.len() < max_size {
                     result.push(byte);
                 }
+            }
             position += 1;
             prev_was_match = false;
         }
     }
 
-    margin + decoder.pos() as isize - position as isize
+    if position > max_size {
+        return Err(UnpackError::OverSize {
+            size: position,
+            limit: max_size,
+        });
+    }
+
+    Ok(margin + decoder.pos() as isize - position as isize)
 }

src/main.rs

@@ -12,6 +12,7 @@ fn main() -> Result<()> {
     let mut level = 2;
     let mut infile: Option<PathBuf> = None;
     let mut outfile: Option<PathBuf> = None;
+    let mut max_unpacked_size = 512 * 1024 * 1024;
 
     let mut parser = lexopt::Parser::from_env();
     while let Some(arg) = parser.next()? {
@@ -32,6 +33,9 @@ fn main() -> Result<()> {
             Long("no-repeated-offsets") => config.no_repeated_offsets = true,
             Long("eof-in-length") => config.eof_in_length = true,
 
+            Long("max-offset") => config.max_offset = parser.value()?.parse()?,
+            Long("max-length") => config.max_length = parser.value()?.parse()?,
+
             Long("z80") => {
                 config.use_bitstream = true;
                 config.bitstream_is_big_endian = true;
@@ -43,12 +47,15 @@ fn main() -> Result<()> {
                 config.use_bitstream = true;
                 config.continue_value_bit = false;
                 config.is_match_bit = false;
+                config.new_offset_bit = false;
             }
 
             Short('u') | Long("unpack") => unpack = true,
             Long("margin") => calculate_margin = true,
             Short('l') | Long("level") => level = parser.value()?.parse()?,
+            Short(n) if n.is_ascii_digit() => level = n as u8 - b'0',
             Short('h') | Long("help") => print_help(0),
+            Long("max-unpacked-size") => max_unpacked_size = parser.value()?.parse()?,
             Value(val) if infile.is_none() => infile = Some(val.try_into()?),
             Value(val) if outfile.is_none() => outfile = Some(val.try_into()?),
             _ => return Err(arg.unexpected().into()),
@@ -92,7 +99,7 @@ fn main() -> Result<()> {
         let mut packed_data = upkr::pack(
             &data,
             level,
-            config,
+            &config,
             Some(&mut |pos| {
                 pb.set(pos as u64);
             }),
@@ -117,14 +124,14 @@ fn main() -> Result<()> {
             data.reverse();
         }
         if unpack {
-            let mut unpacked_data = upkr::unpack(&data, &config);
+            let mut unpacked_data = upkr::unpack(&data, &config, max_unpacked_size)?;
             if reverse {
                 unpacked_data.reverse();
             }
             File::create(outfile)?.write_all(&unpacked_data)?;
         }
         if calculate_margin {
-            println!("{}", upkr::calculate_margin(&data, &config));
+            println!("{}", upkr::calculate_margin(&data, &config)?);
         }
     }
 
@@ -138,12 +145,15 @@ fn print_help(exit_code: i32) -> ! {
     eprintln!("  upkr --margin [config options] <infile>");
     eprintln!();
     eprintln!(" -l, --level N       compression level 0-9");
+    eprintln!(" -0, ..., -9         short form for setting compression level");
     eprintln!(" -u, --unpack        unpack infile");
     eprintln!(" --margin            calculate margin for overlapped unpacking of a packed file");
     eprintln!();
     eprintln!("Config presets for specific unpackers:");
-    eprintln!(" --z80               --big-endian-bitstream --invert-bit-encoding --simplified-prob-update");
+    eprintln!(" --z80               --big-endian-bitstream --invert-bit-encoding --simplified-prob-update -9");
-    eprintln!(" --x86               --bitstream --invert-is-match-bit --invert-continue-value-bit");
+    eprintln!(
+        " --x86               --bitstream --invert-is-match-bit --invert-continue-value-bit --invert-new-offset-bit"
+    );
     eprintln!();
     eprintln!("Config options (need to match when packing/unpacking):");
     eprintln!(" -b, --bitstream     bitstream mode");
@@ -158,5 +168,8 @@ fn print_help(exit_code: i32) -> ! {
     eprintln!(" --simplified-prob-update");
     eprintln!(" --big-endian-bitstream   (implies --bitstream)");
     eprintln!(" --no-repeated-offsets");
+    eprintln!(" --eof-in-length");
+    eprintln!(" --max-offset N");
+    eprintln!(" --max-length N");
     process::exit(exit_code);
 }

src/parsing_packer.rs

@@ -105,7 +105,7 @@ fn parse(
         cost_counter: &mut CostCounter,
         pos: usize,
         offset: usize,
-        length: usize,
+        mut length: usize,
         arrival: &Arrival,
         max_arrivals: usize,
         config: &crate::Config,
@@ -113,6 +113,7 @@ fn parse(
         if length < config.min_length() {
             return;
         }
+        length = length.min(config.max_length);
         cost_counter.reset();
         let mut state = arrival.state.clone();
         let op = lz::Op::Match {
@@ -186,6 +187,7 @@ fn parse(
             for m in match_finder.matches(pos) {
                 closest_match = Some(closest_match.unwrap_or(0).max(m.pos));
                 let offset = pos - m.pos;
+                if offset <= encoding_config.max_offset {
                     found_last_offset |= offset as u32 == arrival.state.last_offset();
                     add_match(
                         &mut arrivals,
@@ -201,6 +203,7 @@ fn parse(
                         break 'arrival_loop;
                     }
                 }
+            }
 
             let mut near_matches_left = config.num_near_matches;
             let mut match_pos = last_seen[data[pos] as usize];
@@ -209,6 +212,9 @@ fn parse(
                 && closest_match.iter().all(|p| *p < match_pos)
             {
                 let offset = pos - match_pos;
+                if offset > encoding_config.max_offset {
+                    break;
+                }
                 let length = match_length(offset);
                 assert!(length > 0);
                 add_match(

src/rans.rs

@@ -1,4 +1,5 @@
 use crate::{context_state::Context, Config};
+use thiserror::Error;
 
 pub const PROB_BITS: u32 = 8;
 pub const ONE_PROB: u32 = 1 << PROB_BITS;
@@ -160,6 +161,10 @@ pub struct RansDecoder<'a> {
 
 const PROB_MASK: u32 = ONE_PROB - 1;
 
+#[derive(Debug, Error)]
+#[error("Unexpected end of input")]
+pub struct UnexpectedEOF;
+
 impl<'a> RansDecoder<'a> {
     pub fn new(data: &'a [u8], config: &Config) -> RansDecoder<'a> {
         RansDecoder {
@@ -178,17 +183,20 @@ impl<'a> RansDecoder<'a> {
         self.pos
     }
 
-    pub fn decode_with_context(&mut self, context: &mut Context) -> bool {
+    pub fn decode_with_context(&mut self, context: &mut Context) -> Result<bool, UnexpectedEOF> {
-        let bit = self.decode_bit(context.prob());
+        let bit = self.decode_bit(context.prob())?;
         context.update(bit);
-        bit
+        Ok(bit)
     }
 
-    pub fn decode_bit(&mut self, prob: u16) -> bool {
+    pub fn decode_bit(&mut self, prob: u16) -> Result<bool, UnexpectedEOF> {
         let prob = prob as u32;
         if self.use_bitstream {
             while self.state < 32768 {
                 if self.bits_left == 0 {
+                    if self.pos >= self.data.len() {
+                        return Err(UnexpectedEOF);
+                    }
                     self.byte = self.data[self.pos];
                     self.pos += 1;
                     self.bits_left = 8;
@@ -204,8 +212,11 @@ impl<'a> RansDecoder<'a> {
             }
         } else {
             while self.state < 4096 {
-                self.state = (self.state << 8) | self.data[0] as u32;
+                if self.pos >= self.data.len() {
-                self.data = &self.data[1..];
+                    return Err(UnexpectedEOF);
+                }
+                self.state = (self.state << 8) | self.data[self.pos] as u32;
+                self.pos += 1;
             }
         }
 
@@ -218,6 +229,6 @@ impl<'a> RansDecoder<'a> {
         };
         self.state = prob * (self.state >> PROB_BITS) + (self.state & PROB_MASK) - start;
 
-        bit ^ self.invert_bit_encoding
+        Ok(bit ^ self.invert_bit_encoding)
     }
 }

z80_unpacker/.gitignore

@@ -0,0 +1,3 @@
+*.bin
+*.tap
+*.lst

z80_unpacker/Makefile

@@ -0,0 +1,11 @@
+all: unpack.bin example/example.sna
+
+# binary is positioned from ORG 0, not usable, just assembling to verify the syntax
+unpack.bin: unpack.asm
+	sjasmplus --msg=war --lst --lstlab=sort --raw=unpack.bin unpack.asm
+
+example/example.sna: unpack.asm example/example.asm
+	cd example && sjasmplus --msg=war --lst --lstlab=sort example.asm
+
+clean:
+	$(RM) unpack.bin unpack.lst example/example.sna example/example.lst

z80_unpacker/example/example.asm

@@ -0,0 +1,100 @@
+;; Example using upkr depacker for screens slideshow
+    OPT --syntax=abf
+    DEVICE ZXSPECTRUM48,$8FFF
+
+    ORG     $9000
+  ;; forward example data
+compressed_scr_files.fwd:               ; border color byte + upkr-packed .scr file
+    DB      1
+    INCBIN  "screens/Grongy - ZX Spectrum (2022).scr.upk"
+    DB      7
+    INCBIN  "screens/Schafft - Poison (2017).scr.upk"
+    DB      0
+    INCBIN  "screens/diver - Mercenary 4. The Heaven's Devil (2014) (Forever 2014 Olympic Edition, 1).scr.upk"
+    DB      6
+    INCBIN  "screens/diver - Back to Bjork (2015).scr.upk"
+.e:
+  ;; backward example data (unpacker goes from the end of the data!)
+compressed_scr_files.rwd.e: EQU $-1     ; the final IX will point one byte ahead of "$" here
+    INCBIN  "screens.reversed/diver - Back to Bjork (2015).scr.upk"
+    DB      6
+    INCBIN  "screens.reversed/diver - Mercenary 4. The Heaven's Devil (2014) (Forever 2014 Olympic Edition, 1).scr.upk"
+    DB      0
+    INCBIN  "screens.reversed/Schafft - Poison (2017).scr.upk"
+    DB      7
+    INCBIN  "screens.reversed/Grongy - ZX Spectrum (2022).scr.upk"
+compressed_scr_files.rwd:               ; border color byte + upkr-packed .scr file (backward)
+    DB      1
+
+start:
+    di
+;     OPT --zxnext
+;     nextreg 7,3                       ; ZX Next: switch to 28Mhz
+
+  ;;; FORWARD packed/unpacked data demo
+    ld      ix,compressed_scr_files.fwd
+.slideshow_loop.fwd:
+  ; set BORDER for next image
+    ld      a,(ix)
+    inc     ix
+    out     (254),a
+  ; call unpack of next image directly into VRAM
+    ld      de,$4000                    ; target VRAM
+    exx
+  ; IX = packed data, DE' = destination ($4000)
+  ; returned IX will point right after the packed data
+    call    fwd.upkr.unpack
+  ; do some busy loop with CPU to delay between images
+    call    delay
+  ; check if all images were displayed, loop around from first one then
+    ld      a,ixl
+    cp      low compressed_scr_files.fwd.e
+    jr      nz,.slideshow_loop.fwd
+
+  ;;; BACKWARD packed/unpacked data demo
+    ld      ix,compressed_scr_files.rwd
+.slideshow_loop.rwd:
+  ; set BORDER for next image
+    ld      a,(ix)
+    dec     ix
+    out     (254),a
+  ; call unpack of next image directly into VRAM
+    ld      de,$5AFF                    ; target VRAM
+    exx
+  ; IX = packed data, DE' = destination
+  ; returned IX will point right ahead of the packed data
+    call    rwd.upkr.unpack
+  ; do some busy loop with CPU to delay between images
+    call    delay
+  ; check if all images were displayed, loop around from first one then
+    ld      a,ixl
+    cp      low compressed_scr_files.rwd.e
+    jr      nz,.slideshow_loop.rwd
+
+    jr      start
+
+delay:
+    ld      bc,$AA00
+.delay:
+    .8 ex      (sp),ix
+    dec     c
+    jr      nz,.delay
+    djnz    .delay
+    ret
+
+  ; include the depacker library, optionally putting probs array buffer near end of RAM
+    DEFINE  UPKR_PROBS_ORIGIN $FA00   ; if not defined, array will be put after unpack code
+
+    MODULE fwd
+        INCLUDE "../unpack.asm"
+    ENDMODULE
+
+    MODULE rwd
+        DEFINE BACKWARDS_UNPACK         ; defined to build backwards unpack
+                ; initial IX points at last byte of compressed data
+                ; initial DE' points at last byte of unpacked data
+
+        INCLUDE "../unpack.asm"
+    ENDMODULE
+
+    SAVESNA "example.sna",start

z80_unpacker/readme.txt

@@ -0,0 +1,32 @@
+Z80 asm implementation of C unpacker, code-size focused (not performance).
+
+**ONLY BITSTREAM** variant is currently supported, make sure to use "-b" in packer.
+
+The project is expected to further evolve, including possible changes to binary format, this is
+initial version of Z80 unpacker to explore if/how it works and how it can be improved further.
+
+(copy full packer+depacker source to your project if you plan to use it, as future revisions
+may be incompatible with files you will produce with current version)
+
+Asm syntax is z00m's sjasmplus: https://github.com/z00m128/sjasmplus
+
+Backward direction unpacker added as compile-time option, see example for both forward/backward
+depacker in action.
+
+The packed/unpacked data-overlap has to be tested per-case, in worst case the packed data
+may need even more than 7 bytes to unpack final byte, but usually 1-4 bytes may suffice.
+
+TODO:
+- build bigger corpus of test data to benchmark future changes in algorithm/format (example and zx48.rom was used to do initial tests)
+- maybe try to beat double-loop `decode_number` with different encoding format
+- (@ped7g) Z80N version of unpacker for ZX Next devs
+- (@exoticorn) add Z80 specific packer (to avoid confusion with original MicroW8 variant), and land it all to master branch, maybe in "z80" directory or something? (and overall decide how to organise+merge this upstream into main repo)
+- (@exoticorn) add to packer output with possible packed/unpacked region overlap
+
+DONE:
+* review non-bitstream variant, if it's feasible to try to implement it with Z80
+    - Ped7g: IMHO nope, the 12b x 8b MUL code would probably quickly cancel any gains from the simpler state update
+* review first implementation to identify weak spots where the implementation can be shorter+faster
+with acceptable small changes to the format
+    - Ped7g: the decode_bit settled down and now doesn't feel so confused and redundant, the code seems pretty on point to me, no obvious simplification from format change
+    - Ped7g: the decode_number double-loop is surprisingly resilient, especially in terms of code size I failed to beat it, speed wise only negligible gains

z80_unpacker/unpack.asm

@@ -0,0 +1,381 @@
+;; https://github.com/exoticorn/upkr/blob/z80/c_unpacker/unpack.c - original C implementation
+;; C source in comments ahead of asm - the C macros are removed to keep only bitstream variant
+;;
+;; initial version by Peter "Ped" Helcmanovsky (C) 2022, licensed same as upkr project ("unlicensed")
+;; to assemble use z00m's sjasmplus: https://github.com/z00m128/sjasmplus
+;;
+;; you can define UPKR_PROBS_ORIGIN to specific 256 byte aligned address for probs array (320 bytes),
+;; otherwise it will be positioned after the unpacker code (256 aligned)
+;;
+;; public API:
+;;
+;;     upkr.unpack
+;;         IN: IX = packed data, DE' (shadow DE) = destination
+;;         OUT: IX = after packed data
+;;         modifies: all registers except IY, requires 10 bytes of stack space
+;;
+
+;     DEFINE BACKWARDS_UNPACK         ; uncomment to build backwards depacker (write_ptr--, upkr_data_ptr--)
+            ; initial IX points at last byte of compressed data
+            ; initial DE' points at last byte of unpacked data
+
+;     DEFINE UPKR_UNPACK_SPEED        ; uncomment to get larger but faster unpack routine
+
+; code size hint: if you put probs array just ahead of BASIC entry point, you will get BC
+; initialised to probs.e by BASIC `USR` command and you can remove it from unpack init (-3B)
+
+    OPT push reset --syntax=abf
+    MODULE upkr
+
+NUMBER_BITS     EQU     16+15       ; context-bits per offset/length (16+15 for 16bit offsets/pointers)
+    ; numbers (offsets/lengths) are encoded like: 1a1b1c1d1e0 = 0000'0000'001e'dbca
+
+/*
+u8* upkr_data_ptr;
+u8 upkr_probs[1 + 255 + 1 + 2*32 + 2*32];
+u16 upkr_state;
+u8 upkr_current_byte;
+int upkr_bits_left;
+
+int upkr_unpack(void* destination, void* compressed_data) {
+    upkr_data_ptr = (u8*)compressed_data;
+    upkr_state = 0;
+    upkr_bits_left = 0;
+    for(int i = 0; i < sizeof(upkr_probs); ++i)
+        upkr_probs[i] = 128;
+
+    u8* write_ptr = (u8*)destination;
+
+    int prev_was_match = 0;
+    int offset = 0;
+    for(;;) {
+        if(upkr_decode_bit(0)) {
+            if(prev_was_match || upkr_decode_bit(256)) {
+                offset = upkr_decode_length(257) - 1;
+                if(offset == 0) {
+                    break;
+                }
+            }
+            int length = upkr_decode_length(257 + 64);
+            while(length--) {
+                *write_ptr = write_ptr[-offset];
+                ++write_ptr;
+            }
+            prev_was_match = 1;
+        } else {
+            int byte = 1;
+            while(byte < 256) {
+                int bit = upkr_decode_bit(byte);
+                byte = (byte << 1) + bit;
+            }
+            *write_ptr++ = byte;
+            prev_was_match = 0;
+        }
+    }
+
+    return write_ptr - (u8*)destination;
+}
+*/
+; IN: IX = compressed_data, DE' = destination
+unpack:
+  ; ** reset probs to 0x80, also reset HL (state) to zero, and set BC to probs+context 0
+    ld      hl,probs.c>>1
+    ld      bc,probs.e
+    ld      a,$80
+.reset_probs:
+    dec     bc
+    ld      (bc),a              ; will overwrite one extra byte after the array because of odd length
+    dec     bc
+    ld      (bc),a
+    dec     l
+    jr      nz,.reset_probs
+    exa
+    ; BC = probs (context_index 0), state HL = 0, A' = 0x80 (no source bits left in upkr_current_byte)
+
+  ; ** main loop to decompress data
+    ; D = prev_was_match = uninitialised, literal is expected first => will reset D to "false"
+    ; values for false/true of prev_was_match are: false = high(probs), true = 1 + high(probs)
+.decompress_data:
+    ld      c,0
+    call    decode_bit          ; if(upkr_decode_bit(0))
+    jr      c,.copy_chunk
+
+  ; * extract byte from compressed data (literal)
+    inc     c                   ; C = byte = 1 (and also context_index)
+.decode_byte:
+    call    decode_bit          ; bit = upkr_decode_bit(byte);
+    rl      c                   ; byte = (byte << 1) + bit;
+    jr      nc,.decode_byte     ; while(byte < 256)
+    ld      a,c
+    exx
+    ld      (de),a              ; *write_ptr++ = byte;
+    IFNDEF BACKWARDS_UNPACK : inc de : ELSE : dec de : ENDIF
+    exx
+    ld      d,b                 ; prev_was_match = false
+    jr      .decompress_data
+
+  ; * copy chunk of already decompressed data (match)
+.copy_chunk:
+    ld      a,b
+    inc     b                   ; context_index = 256
+        ;             if(prev_was_match || upkr_decode_bit(256)) {
+        ;                 offset = upkr_decode_length(257) - 1;
+        ;                 if (0 == offset) break;
+        ;             }
+    cp      d                   ; CF = prev_was_match
+    call    nc,decode_bit       ; if not prev_was_match, then upkr_decode_bit(256)
+    jr      nc,.keep_offset     ; if neither, keep old offset
+    call    decode_number       ; context_index is already 257-1 as needed by decode_number
+    dec     de                  ; offset = upkr_decode_length(257) - 1;
+    ld      a,d
+    or      e
+    ret     z                   ; if(offset == 0) break
+    ld      (.offset),de
+.keep_offset:
+        ;             int length = upkr_decode_length(257 + 64);
+        ;             while(length--) {
+        ;                 *write_ptr = write_ptr[-offset];
+        ;                 ++write_ptr;
+        ;             }
+        ;             prev_was_match = 1;
+    ld      c,low(257 + NUMBER_BITS - 1)    ; context_index to second "number" set for lengths decoding
+    call    decode_number       ; length = upkr_decode_length(257 + 64);
+    push    de
+    exx
+    IFNDEF BACKWARDS_UNPACK
+        ; forward unpack (write_ptr++, upkr_data_ptr++)
+        ld      h,d             ; DE = write_ptr
+        ld      l,e
+.offset+*:  ld  bc,0
+        sbc     hl,bc           ; CF=0 from decode_number ; HL = write_ptr - offset
+        pop     bc              ; BC = length
+        ldir
+    ELSE
+        ; backward unpack (write_ptr--, upkr_data_ptr--)
+.offset+*:  ld  hl,0
+        add     hl,de           ; HL = write_ptr + offset
+        pop     bc              ; BC = length
+        lddr
+    ENDIF
+    exx
+    ld      d,b                 ; prev_was_match = true
+    djnz    .decompress_data    ; adjust context_index back to 0..255 range, go to main loop
+
+/*
+int upkr_decode_bit(int context_index) {
+    while(upkr_state < 32768) {
+        if(upkr_bits_left == 0) {
+            upkr_current_byte = *upkr_data_ptr++;
+            upkr_bits_left = 8;
+        }
+        upkr_state = (upkr_state << 1) + (upkr_current_byte >> 7);
+        upkr_current_byte <<= 1;
+        --upkr_bits_left;
+    }
+
+    int prob = upkr_probs[context_index];
+    int bit = (upkr_state & 255) >= prob ? 1 : 0;
+
+    int prob_offset = 16;
+    int state_offset = 0;
+    int state_scale = prob;
+    if(bit) {
+        state_offset = -prob;
+        state_scale = 256 - prob;
+        prob_offset = 0;
+    }
+    upkr_state = state_offset + state_scale * (upkr_state >> 8) + (upkr_state & 255);
+    upkr_probs[context_index] = prob_offset + prob - ((prob + 8) >> 4);
+
+    return bit;
+}
+*/
+inc_c_decode_bit:
+  ; ++low(context_index) before decode_bit (to get -1B by two calls in decode_number)
+    inc     c
+decode_bit:
+  ; HL = upkr_state
+  ; IX = upkr_data_ptr
+  ; BC = probs+context_index
+  ; A' = upkr_current_byte (!!! init to 0x80 at start, not 0x00)
+  ; preserves DE
+  ; ** while (state < 32768) - initial check
+    push    de
+    bit     7,h
+    jr      nz,.state_b15_set
+    exa
+  ; ** while body
+.state_b15_zero:
+  ; HL = upkr_state
+  ; IX = upkr_data_ptr
+  ; A = upkr_current_byte (init to 0x80 at start, not 0x00)
+    add     a,a                     ; upkr_current_byte <<= 1; // and testing if(upkr_bits_left == 0)
+    jr      nz,.has_bit             ; CF=data, ZF=0 -> some bits + stop bit still available
+  ; CF=1 (by stop bit)
+    ld      a,(ix)
+    IFNDEF BACKWARDS_UNPACK : inc ix : ELSE : dec ix : ENDIF    ; upkr_current_byte = *upkr_data_ptr++;
+    adc     a,a                     ; CF=data, b0=1 as new stop bit
+.has_bit:
+    adc     hl,hl                   ; upkr_state = (upkr_state << 1) + (upkr_current_byte >> 7);
+    jp      p,.state_b15_zero       ; while (state < 32768)
+    exa
+  ; ** set "bit"
+.state_b15_set:
+    ld      a,(bc)                  ; A = upkr_probs[context_index]
+    dec     a                       ; prob is in ~7..249 range, never zero, safe to -1
+    cp      l                       ; CF = bit = prob-1 < (upkr_state & 255) <=> prob <= (upkr_state & 255)
+    inc     a
+  ; ** adjust state
+    push    bc
+    ld      c,l                     ; C = (upkr_state & 255); (preserving the value)
+    push    af
+    jr      nc,.bit_is_0
+    neg                             ; A = -prob == (256-prob), CF=1 preserved
+.bit_is_0:
+    ld      d,0
+    ld      e,a                     ; DE = state_scale ; prob || (256-prob)
+    ld      l,d                     ; H:L = (upkr_state>>8) : 0
+
+  IFNDEF UPKR_UNPACK_SPEED
+
+    ;; looped MUL for minimum unpack size
+    ld      b,8                     ; counter
+.mulLoop:
+    add     hl,hl
+    jr      nc,.mul0
+    add     hl,de
+.mul0:
+    djnz    .mulLoop                ; until HL = state_scale * (upkr_state>>8), also BC becomes (upkr_state & 255)
+
+  ELSE
+
+    ;;; unrolled MUL for better performance, +25 bytes unpack size
+    ld      b,d
+    DUP     8
+        add     hl,hl
+        jr      nc,0_f
+        add     hl,de
+0:
+    EDUP
+
+  ENDIF
+
+    add     hl,bc                   ; HL = state_scale * (upkr_state >> 8) + (upkr_state & 255)
+    pop     af                      ; restore prob and CF=bit
+    jr      nc,.bit_is_0_2
+    dec     d                       ; DE = -prob (also D = bit ? $FF : $00)
+    add     hl,de                   ; HL += -prob
+    ; ^ this always preserves CF=1, because (state>>8) >= 128, state_scale: 7..250, prob: 7..250,
+    ; so 7*128 > 250 and thus edge case `ADD hl=(7*128+0),de=(-250)` => CF=1
+.bit_is_0_2:
+ ; *** adjust probs[context_index]
+    rra                             ; + (bit<<4) ; part of -prob_offset, needs another -16
+    and     $FC                     ; clear/keep correct bits to get desired (prob>>4) + extras, CF=0
+    rra
+    rra
+    rra                             ; A = (bit<<4) + (prob>>4), CF=(prob & 8)
+    adc     a,-16                   ; A = (bit<<4) - 16 + ((prob + 8)>>4) ; -prob_offset = (bit<<4) - 16
+    ld      e,a
+    pop     bc
+    ld      a,(bc)                  ; A = prob (cheaper + shorter to re-read again from memory)
+    sub     e                       ; A = 16 - (bit<<4) + prob - ((prob + 8)>>4) ; = prob_offset + prob - ((prob + 8)>>4)
+    ld      (bc),a                  ; probs[context_index] = prob_offset + prob - ((prob + 8) >> 4);
+    add     a,d                     ; restore CF = bit (D = bit ? $FF : $00 && A > 0)
+    pop     de
+    ret
+
+/*
+int upkr_decode_length(int context_index) {
+    int length = 0;
+    int bit_pos = 0;
+    while(upkr_decode_bit(context_index)) {
+        length |= upkr_decode_bit(context_index + 1) << bit_pos++;
+        context_index += 2;
+    }
+    return length | (1 << bit_pos);
+}
+*/
+decode_number:
+  ; HL = upkr_state
+  ; IX = upkr_data_ptr
+  ; BC = probs+context_index-1
+  ; A' = upkr_current_byte (!!! init to 0x80 at start, not 0x00)
+  ; return length in DE, CF=0
+    ld      de,$FFFF            ; length = 0 with positional-stop-bit
+    or      a                   ; CF=0 to skip getting data bit and use only `rr d : rr e` to fix init DE
+.loop:
+    call    c,inc_c_decode_bit  ; get data bit, context_index + 1 / if CF=0 just add stop bit into DE init
+    rr      d
+    rr      e                   ; DE = length = (length >> 1) | (bit << 15);
+    call    inc_c_decode_bit    ; context_index += 2
+    jr      c,.loop
+.fix_bit_pos:
+    ccf                         ; NC will become this final `| (1 << bit_pos)` bit
+    rr      d
+    rr      e
+    jr      c,.fix_bit_pos      ; until stop bit is reached (all bits did land to correct position)
+    ret                         ; return with CF=0 (important for unpack routine)
+
+    DISPLAY "upkr.unpack total size: ",/D,$-unpack
+
+    ; reserve space for probs array without emitting any machine code (using only EQU)
+
+    IFDEF UPKR_PROBS_ORIGIN     ; if specific address is defined by user, move probs array there
+probs:      EQU ((UPKR_PROBS_ORIGIN) + 255) & -$100     ; probs array aligned to 256
+    ELSE
+probs:      EQU ($ + 255) & -$100                       ; probs array aligned to 256
+    ENDIF
+.real_c:    EQU 1 + 255 + 1 + 2*NUMBER_BITS             ; real size of probs array
+.c:         EQU (.real_c + 1) & -2                      ; padding to even size (required by init code)
+.e:         EQU probs + .c
+
+    DISPLAY "upkr.unpack probs array placed at: ",/A,probs,",\tsize: ",/A,probs.c
+
+/*
+ archived: negligibly faster but +6B longer decode_number variant using HL' and BC' to
+ do `number|=(1<<bit_pos);` type of logic in single loop.
+*/
+; decode_number:
+;     exx
+;     ld      bc,1
+;     ld      l,b
+;     ld      h,b                 ; HL = 0
+; .loop
+;     exx
+;     inc     c
+;     call    decode_bit
+;     jr      nc,.done
+;     inc     c
+;     call    decode_bit
+;     exx
+;     jr      nc,.b0
+;     add     hl,bc
+; .b0:
+;     sla     c
+;     rl      b
+;     jr      .loop
+; .done:
+;     exx
+;     add     hl,bc
+;     push    hl
+;     exx
+;     pop     de
+;     ret
+
+/*
+ archived: possible LUT variant of updating probs value, requires 512-aligned 512B table (not tested)
+*/
+; code is replacing decode_bit from "; *** adjust probs[context_index]", followed by `ld (bc),a : add a,d ...`
+;     ld      c,a
+;     ld      a,high(probs_update_table)/2    ; must be 512 aligned
+;     rla
+;     ld      b,a
+;     ld      a,(bc)
+;     pop     bc
+; -------------------------------------------
+; probs_update_table: EQU probs-512
+; -------------------------------------------
+; table generator is not obvious and probably not short either, 20+ bytes almost for sure, maybe even 30-40
+
+    ENDMODULE
+    OPT pop