From 130bf821fa185b54af83d36519b0136f9a934364 Mon Sep 17 00:00:00 2001
From: Dennis Ranke <dennis.ranke@gmail.com>
Date: Tue, 11 Mar 2025 10:52:24 +0100
Subject: [PATCH] implement dictionary support when packing

---
 README.md.upk         | Bin 0 -> 1553 bytes
 src/greedy_packer.rs  |   2 +-
 src/lib.rs            |   5 +++++
 src/main.rs           |  11 +++++++++++
 src/parsing_packer.rs |   4 ++--
 5 files changed, 19 insertions(+), 3 deletions(-)
 create mode 100644 README.md.upk

diff --git a/README.md.upk b/README.md.upk
new file mode 100644
index 0000000000000000000000000000000000000000..73338acc3658760961ecdab170e6e62eb4c8f0a0
GIT binary patch
literal 1553
zcmV+s2JZO^JDD1d9nD>zA*pGOEIl!o?Mf@CR<5EpzRImxqQx4sIltfMROEZVgq|*u
zFd04NY@DnOf_;&zu;B;QFv|mNAmbXf5fOs|&&-r2xfor>U)~?Pr4>&;aaT~fP3!Yf
zu8VdzOqxbT@V`H1{UClCUnCp8nXL}dHuq2CgK1J%<pEHu*B`;o7^qIeF_$102u&hJ
z94S`LxcRIQr_p%(C=eK!rN7pm&TB;&(>sw;T_$s>z;u2>fLtz2a_iF<8&5)PGML!G
zjSC4GHCMU?TvCmY*mj;zF9Vj}?YVjzs_cQ>rB^>Ul*uQTPVvp=z@uEjl+H%^W^SQ)
zvjR$>H6^oUe@nIn{#%Y!?Rq-uFA#Cx1y$)l9{IGVh(cAx<c97nBU5v3Pk1=0YSXfF
z^q+lZ8P*N8LO|19#TgxwGlofij((gatR8mw{C4gwlv}Tk72FN@JO9}(QoJGzS}=m<
z`g+=~4htYTM&~{3#0He~@#Q{^-vp_*n~97?C=FEDxj#eBOyXnk@H<n5lZ6Ps8bt=c
zjZryiwOuE^WPTcL&#8dq!h!EQRIu^Fd4yyLWcb@L4m*u|#$LE0S24XU&ChLO=eFJp
ze}OG8S<?MRrZ`gRJ}VGx$>byb3N^}vREk2cKV7MT*fb&L_l{5wib_`tq`~#Hb8q(=
ziqUgXnfZ!3uwI`p5Es23k4?)yuvuL_!{gzb*VApIR(Y>g2mkn&nAt#qeW+~uIN>tA
z9m5zPyLn7Qnn=2flU9T)K2@_kH5V(BS^D|g<}xtHq(KO92?Wx$G3`j@Ol<N_Wc-N*
zXU@1SY{4o|a{7(E3FI1|b0rOwrv)$nu#GI_1hzk$D6aJ*FW@u)z+9`9_X=2ke+`|g
zZ;bw9=C_XBjmd4wZjv3sZj0#CYh)CqHN_5>yK%F<Dieo+=l>X!o0)uKBZBj^JJ3rI
z@6xhsqM7px*-WuZMhqqj(d3F@{P+9+#9LpI4Hv5!n{k*qI=TL{hzYhqo_Pp)gD}he
z{BlB2+zgl9vP6%KA8HQUKoLSK8L%e<=}fnFwTMmMzChDFRHrK$U>#ddr5ca|Ijn@*
zi@xB}k8wu{y{D_I`l|qCjD)Wp?X>+a8i0)r`%AQTbUuzINUjRAi`7_m7agmqDZt67
z5mqASZ8T$ub86ecSr5@4vg$-a{(hF<PGK<^wSF6>34qaeaw;v_Q~;uDE0agSb{)Ud
zK}?+shjD7GFkim^J>s<5SGCUdjj(+co0V#ujp-YprFEMmXjqZrNobyY2#6J@x?PT@
zpF+=Y<3*ey*lXR+MSrT+2k7Jfay&~q$_1qHYB3RH+7}Siz6T39p93zd;`#EbX{1rp
zX41(Bl09-%WAHHCZL)~{3|y_5F=Q_}gjk0q+PR(#mS;NOFOlNg)>n*c=5a@!GvnSn
zQZZIv!45WyLhF<lc}|k=J!f!nFL<dyE*lPR#Y}hFp23^!>l%-ZTw4oNQRX_@b8wga
z&S1$2u@&Bsmoke{o=g%@k)|^@Oj1La7LTGQvm|X_DS?;flw`!!9JVR562Y5e+WUfu
zME17?#`e;;$&<llaBETxgrlXQzLL`+P89c81KwW!>X*YCH*g>Z@rFv1gg^{%3g<Q2
za~jEjjAGG8zZGcJPQa7ogQk3C22VMI$zsRSwd@zA0L(KZE|EI%#x5|@7tfI%d#;_x
zXPMV-GPcT=RJ;T0@oB1%aKtD`yxyCJw}5Ned{LShgYHT6lIDn;<JU>o=QEDK*xJ?-
za7!D;;#cOtfE5>#-=iHoN7CwK*QAry<oBXL#chx>Et;pl$Ey`0-k|shK^nkPg_)r>
zawN~crUlNXe`~I!DO}N3KdX1;*<b*o;F52gWBbUEwxK%WNg_gFg<|TYZr%D|V%gH!
z2jQ|{s@RQ$F$=p(+1@zQwsb}pU^1U*;@&gnd^(h?$d)5R0W^NV_+{L7(t!7YWVM=}
zb)8eL3gmS>*{7wU7#Ks#fXWx>N01N=-t4v%_B<qJ!Yo_=wfy4XD#XwC9+s@d6C#KV
DLg)i7

literal 0
HcmV?d00001

diff --git a/src/greedy_packer.rs b/src/greedy_packer.rs
index 3b77d13..b1875e7 100644
--- a/src/greedy_packer.rs
+++ b/src/greedy_packer.rs
@@ -12,7 +12,7 @@ pub fn pack(
     let mut rans_coder = RansCoder::new(config);
     let mut state = lz::CoderState::new(config);
 
-    let mut pos = 0;
+    let mut pos = config.dictionary_size;
     while pos < data.len() {
         if let Some(ref mut cb) = progress_callback {
             cb(pos);
diff --git a/src/lib.rs b/src/lib.rs
index c3e4cd5..ec0e591 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -71,6 +71,9 @@ pub struct Config {
     pub max_offset: usize,
     /// The maximum match length value to encode when compressing.
     pub max_length: usize,
+
+    /// Size of dictionary at the beginning of data (how many bytes to skip when compressing.)
+    pub dictionary_size: usize,
 }
 
 impl Default for Config {
@@ -92,6 +95,8 @@ impl Default for Config {
 
             max_offset: usize::MAX,
             max_length: usize::MAX,
+
+            dictionary_size: 0,
         }
     }
 }
diff --git a/src/main.rs b/src/main.rs
index ba4aff7..19460de 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -16,6 +16,7 @@ fn main() -> Result<()> {
     let mut level = 2;
     let mut infile: Option<PathBuf> = None;
     let mut outfile: Option<PathBuf> = None;
+    let mut dictionary: Option<PathBuf> = None;
     let mut max_unpacked_size = 512 * 1024 * 1024;
 
     let mut parser = lexopt::Parser::from_env();
@@ -74,6 +75,7 @@ fn main() -> Result<()> {
                 process::exit(0);
             }
             Long("max-unpacked-size") => max_unpacked_size = parser.value()?.parse()?,
+            Long("dictionary") => dictionary = Some(parser.value()?.try_into()?),
             Value(val) if infile.is_none() => infile = Some(val.try_into()?),
             Value(val) if outfile.is_none() => outfile = Some(val.try_into()?),
             _ => return Err(arg.unexpected().into()),
@@ -94,6 +96,15 @@ fn main() -> Result<()> {
             data.reverse();
         }
 
+        if let Some(dictionary) = dictionary {
+            let mut dict = vec![];
+            File::open(dictionary)?.read_to_end(&mut dict)?;
+            config.dictionary_size = dict.len();
+            // prepend dict
+            dict.append(&mut data);
+            data = dict;
+        }
+
         #[cfg(feature = "terminal")]
         let mut packed_data = {
             let mut pb = pbr::ProgressBar::on(std::io::stderr(), data.len() as u64);
diff --git a/src/parsing_packer.rs b/src/parsing_packer.rs
index 9ab0f75..7769ef7 100644
--- a/src/parsing_packer.rs
+++ b/src/parsing_packer.rs
@@ -137,7 +137,7 @@ fn parse(
     }
     add_arrival(
         &mut arrivals,
-        0,
+        encoding_config.dictionary_size,
         Arrival {
             parse: None,
             state: lz::CoderState::new(encoding_config),
@@ -148,7 +148,7 @@ fn parse(
 
     let cost_counter = &mut CostCounter::new(encoding_config);
     let mut best_per_offset = HashMap::new();
-    for pos in 0..data.len() {
+    for pos in encoding_config.dictionary_size..data.len() {
         let match_length = |offset: usize| {
             data[pos..]
                 .iter()