dranke/upkr
1
0
Fork 0
mirror of https://github.com/exoticorn/upkr.git synced 2026-01-20 11:36:42 +01:00
Code Issues Projects Releases Wiki Activity

implement optional parity contexts

Browse Source
This commit is contained in:
Dennis Ranke
2022-09-20 23:24:19 +02:00
parent cc41feb5cd
commit f5fc9bd005
5 changed files with 84 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/greedy_packer.rs
View File

@@ -6,11 +6,12 @@ use crate::ProgressCallback;
pub fn pack( pub fn pack(
data: &[u8], data: &[u8],
use_bitstream: bool, use_bitstream: bool,
parity_contexts: usize,
mut progress_callback: Option<ProgressCallback>, mut progress_callback: Option<ProgressCallback>,
) -> Vec<u8> { ) -> Vec<u8> {
let mut match_finder = MatchFinder::new(data); let mut match_finder = MatchFinder::new(data);
let mut rans_coder = RansCoder::new(use_bitstream); let mut rans_coder = RansCoder::new(use_bitstream);
let mut state = lz::CoderState::new(); let mut state = lz::CoderState::new(parity_contexts);
let mut pos = 0; let mut pos = 0;
while pos < data.len() { while pos < data.len() {

11
src/lib.rs
View File

@@ -13,12 +13,19 @@ pub fn pack(
data: &[u8], data: &[u8],
level: u8, level: u8,
use_bitstream: bool, use_bitstream: bool,
parity_contexts: usize,
progress_callback: Option<ProgressCallback>, progress_callback: Option<ProgressCallback>,
) -> Vec<u8> { ) -> Vec<u8> {
if level == 0 { if level == 0 {
greedy_packer::pack(data, use_bitstream, progress_callback) greedy_packer::pack(data, use_bitstream, parity_contexts, progress_callback)
} else { } else {
parsing_packer::pack(data, level, use_bitstream, progress_callback) parsing_packer::pack(
data,
level,
use_bitstream,
parity_contexts,
progress_callback,
)
} }
} }

52
src/lz.rs
View File

@@ -9,41 +9,49 @@ pub enum Op {
impl Op { impl Op {
pub fn encode(&self, coder: &mut dyn EntropyCoder, state: &mut CoderState) { pub fn encode(&self, coder: &mut dyn EntropyCoder, state: &mut CoderState) {
let literal_base = state.pos % state.parity_contexts * 256;
match self { match self {
&Op::Literal(lit) => { &Op::Literal(lit) => {
encode_bit(coder, state, 0, false); encode_bit(coder, state, literal_base, false);
let mut context_index = 1; let mut context_index = 1;
for i in (0..8).rev() { for i in (0..8).rev() {
let bit = (lit >> i) & 1 != 0; let bit = (lit >> i) & 1 != 0;
encode_bit(coder, state, context_index, bit); encode_bit(coder, state, literal_base + context_index, bit);
context_index = (context_index << 1) | bit as usize; context_index = (context_index << 1) | bit as usize;
} }
state.prev_was_match = false; state.prev_was_match = false;
state.pos += 1;
} }
&Op::Match { offset, len } => { &Op::Match { offset, len } => {
encode_bit(coder, state, 0, true); encode_bit(coder, state, literal_base, true);
if !state.prev_was_match { if !state.prev_was_match {
encode_bit(coder, state, 256, offset != state.last_offset); encode_bit(
coder,
state,
256 * state.parity_contexts,
offset != state.last_offset,
);
} else { } else {
assert!(offset != state.last_offset); assert!(offset != state.last_offset);
} }
if offset != state.last_offset { if offset != state.last_offset {
encode_length(coder, state, 257, offset + 1); encode_length(coder, state, 256 * state.parity_contexts + 1, offset + 1);
state.last_offset = offset; state.last_offset = offset;
} }
encode_length(coder, state, 257 + 64, len); encode_length(coder, state, 256 * state.parity_contexts + 65, len);
state.prev_was_match = true; state.prev_was_match = true;
state.pos += len as usize;
} }
} }
} }
} }
pub fn encode_eof(coder: &mut dyn EntropyCoder, state: &mut CoderState) { pub fn encode_eof(coder: &mut dyn EntropyCoder, state: &mut CoderState) {
encode_bit(coder, state, 0, true); encode_bit(coder, state, state.pos % state.parity_contexts * 256, true);
if !state.prev_was_match { if !state.prev_was_match {
encode_bit(coder, state, 256, true); encode_bit(coder, state, 256 * state.parity_contexts, true);
} }
encode_length(coder, state, 257, 1); encode_length(coder, state, 256 * state.parity_contexts + 1, 1);
} }
fn encode_bit( fn encode_bit(
@@ -76,16 +84,20 @@ fn encode_length(
#[derive(Clone)] #[derive(Clone)]
pub struct CoderState { pub struct CoderState {
contexts: ContextState, contexts: ContextState,
parity_contexts: usize,
last_offset: u32, last_offset: u32,
prev_was_match: bool, prev_was_match: bool,
pos: usize,
} }
impl CoderState { impl CoderState {
pub fn new() -> CoderState { pub fn new(parity_contexts: usize) -> CoderState {
CoderState { CoderState {
contexts: ContextState::new(1 + 255 + 1 + 64 + 64), contexts: ContextState::new((1 + 255) * parity_contexts + 1 + 64 + 64),
last_offset: 0, last_offset: 0,
parity_contexts,
prev_was_match: false, prev_was_match: false,
pos: 0,
} }
} }
@@ -94,9 +106,9 @@ impl CoderState {
} }
} }
pub fn unpack(packed_data: &[u8], use_bitstream: bool) -> Vec<u8> { pub fn unpack(packed_data: &[u8], use_bitstream: bool, parity_contexts: usize) -> Vec<u8> {
let mut decoder = RansDecoder::new(packed_data, use_bitstream); let mut decoder = RansDecoder::new(packed_data, use_bitstream);
let mut contexts = ContextState::new(1 + 255 + 1 + 64 + 64); let mut contexts = ContextState::new((1 + 255) * parity_contexts + 1 + 64 + 64);
let mut result = vec![]; let mut result = vec![];
let mut offset = 0; let mut offset = 0;
let mut prev_was_match = false; let mut prev_was_match = false;
@@ -119,14 +131,17 @@ pub fn unpack(packed_data: &[u8], use_bitstream: bool) -> Vec<u8> {
} }
loop { loop {
if decoder.decode_with_context(&mut contexts.context_mut(0)) { let literal_base = result.len() % parity_contexts * 256;
if prev_was_match || decoder.decode_with_context(&mut contexts.context_mut(256)) { if decoder.decode_with_context(&mut contexts.context_mut(literal_base)) {
offset = decode_length(&mut decoder, &mut contexts, 257) - 1; if prev_was_match
|| decoder.decode_with_context(&mut contexts.context_mut(256 * parity_contexts))
{
offset = decode_length(&mut decoder, &mut contexts, 256 * parity_contexts + 1) - 1;
if offset == 0 { if offset == 0 {
break; break;
} }
} }
let length = decode_length(&mut decoder, &mut contexts, 257 + 64); let length = decode_length(&mut decoder, &mut contexts, 256 * parity_contexts + 65);
for _ in 0..length { for _ in 0..length {
result.push(result[result.len() - offset]); result.push(result[result.len() - offset]);
} }
@@ -135,7 +150,8 @@ pub fn unpack(packed_data: &[u8], use_bitstream: bool) -> Vec<u8> {
let mut context_index = 1; let mut context_index = 1;
let mut byte = 0; let mut byte = 0;
for i in (0..8).rev() { for i in (0..8).rev() {
let bit = decoder.decode_with_context(&mut contexts.context_mut(context_index)); let bit = decoder
.decode_with_context(&mut contexts.context_mut(literal_base + context_index));
context_index = (context_index << 1) | bit as usize; context_index = (context_index << 1) | bit as usize;
byte |= (bit as u8) << i; byte |= (bit as u8) << i;
} }

27
src/main.rs
View File

@@ -1,5 +1,6 @@
use anyhow::{bail, Result}; use anyhow::{bail, Result};
use std::io::prelude::*; use std::io::prelude::*;
use std::process;
use std::{fs::File, path::PathBuf}; use std::{fs::File, path::PathBuf};
fn main() -> Result<()> { fn main() -> Result<()> {
@@ -10,6 +11,14 @@ fn main() -> Result<()> {
Some("pack") => { Some("pack") => {
let level = args.opt_value_from_str(["-l", "--level"])?.unwrap_or(2u8); let level = args.opt_value_from_str(["-l", "--level"])?.unwrap_or(2u8);
let use_bitstream = args.contains(["-b", "--bitstream"]); let use_bitstream = args.contains(["-b", "--bitstream"]);
let parity_contexts = args
.opt_value_from_str(["-p", "--parity"])?
.unwrap_or(1usize);
if parity_contexts != 1 && parity_contexts != 2 && parity_contexts != 4 {
eprintln!("--parity has to be 1, 2 or 4");
process::exit(1);
}
let infile = args.free_from_os_str::<PathBuf, bool>(|s| Ok(s.into()))?; let infile = args.free_from_os_str::<PathBuf, bool>(|s| Ok(s.into()))?;
let outfile = args.free_from_os_str::<PathBuf, bool>(|s| Ok(s.into()))?; let outfile = args.free_from_os_str::<PathBuf, bool>(|s| Ok(s.into()))?;
@@ -23,6 +32,7 @@ fn main() -> Result<()> {
&data, &data,
level, level,
use_bitstream, use_bitstream,
parity_contexts,
Some(&mut |pos| { Some(&mut |pos| {
pb.set(pos as u64); pb.set(pos as u64);
}), }),
@@ -39,13 +49,21 @@ fn main() -> Result<()> {
} }
Some("unpack") => { Some("unpack") => {
let use_bitstream = args.contains(["-b", "--bitstream"]); let use_bitstream = args.contains(["-b", "--bitstream"]);
let parity_contexts = args
.opt_value_from_str(["-p", "--parity"])?
.unwrap_or(1usize);
if parity_contexts != 1 && parity_contexts != 2 && parity_contexts != 4 {
eprintln!("--parity has to be 1, 2 or 4");
process::exit(1);
}
let infile = args.free_from_os_str::<PathBuf, bool>(|s| Ok(s.into()))?; let infile = args.free_from_os_str::<PathBuf, bool>(|s| Ok(s.into()))?;
let outfile = args.free_from_os_str::<PathBuf, bool>(|s| Ok(s.into()))?; let outfile = args.free_from_os_str::<PathBuf, bool>(|s| Ok(s.into()))?;
let mut data = vec![]; let mut data = vec![];
File::open(infile)?.read_to_end(&mut data)?; File::open(infile)?.read_to_end(&mut data)?;
let packed_data = upkr::unpack(&data, use_bitstream); let packed_data = upkr::unpack(&data, use_bitstream, parity_contexts);
File::create(outfile)?.write_all(&packed_data)?; File::create(outfile)?.write_all(&packed_data)?;
} }
Some(other) => { Some(other) => {
@@ -58,10 +76,11 @@ fn main() -> Result<()> {
fn print_help() { fn print_help() {
eprintln!("Usage:"); eprintln!("Usage:");
eprintln!(" upkr pack [-b] [-l level(0-9)] <infile> <outfile>"); eprintln!(" upkr pack [-b] [-l level(0-9)] [-p N] <infile> <outfile>");
eprintln!(" upkr unpack [-b] <infile> <outfile>"); eprintln!(" upkr unpack [-b] [-p N] <infile> <outfile>");
eprintln!(); eprintln!();
eprintln!(" -b, --bitstream bitstream mode"); eprintln!(" -b, --bitstream bitstream mode");
eprintln!(" -l, --level N compression level 0-9"); eprintln!(" -l, --level N compression level 0-9");
std::process::exit(1); eprintln!(" -p, --parity N use N (2/4) parity contexts");
process::exit(1);
} }

20
src/parsing_packer.rs
View File

@@ -6,14 +6,25 @@ use crate::match_finder::MatchFinder;
use crate::rans::{CostCounter, RansCoder}; use crate::rans::{CostCounter, RansCoder};
use crate::{lz, ProgressCallback}; use crate::{lz, ProgressCallback};
pub fn pack(data: &[u8], level: u8, use_bitstream: bool, progress_cb: Option<ProgressCallback>) -> Vec<u8> { pub fn pack(
let mut parse = parse(data, Config::from_level(level), progress_cb); data: &[u8],
level: u8,
use_bitstream: bool,
parity_contexts: usize,
progress_cb: Option<ProgressCallback>,
) -> Vec<u8> {
let mut parse = parse(
data,
Config::from_level(level),
parity_contexts,
progress_cb,
);
let mut ops = vec![]; let mut ops = vec![];
while let Some(link) = parse { while let Some(link) = parse {
ops.push(link.op); ops.push(link.op);
parse = link.prev.clone(); parse = link.prev.clone();
} }
let mut state = lz::CoderState::new(); let mut state = lz::CoderState::new(parity_contexts);
let mut coder = RansCoder::new(use_bitstream); let mut coder = RansCoder::new(use_bitstream);
for op in ops.into_iter().rev() { for op in ops.into_iter().rev() {
op.encode(&mut coder, &mut state); op.encode(&mut coder, &mut state);
@@ -38,6 +49,7 @@ type Arrivals = HashMap<usize, Vec<Arrival>>;
fn parse( fn parse(
data: &[u8], data: &[u8],
config: Config, config: Config,
parity_contexts: usize,
mut progress_cb: Option<ProgressCallback>, mut progress_cb: Option<ProgressCallback>,
) -> Option<Rc<Parse>> { ) -> Option<Rc<Parse>> {
let mut match_finder = MatchFinder::new(data) let mut match_finder = MatchFinder::new(data)
@@ -129,7 +141,7 @@ fn parse(
0, 0,
Arrival { Arrival {
parse: None, parse: None,
state: lz::CoderState::new(), state: lz::CoderState::new(parity_contexts),
cost: 0.0, cost: 0.0,
}, },
max_arrivals, max_arrivals,