dranke/upkr
1
0
Fork 0
mirror of https://github.com/exoticorn/upkr.git synced 2026-01-20 19:46:42 +01:00
Code Issues Projects Releases Wiki Activity

a few speed and/or compression ratio improvements

Browse Source
This commit is contained in:
Dennis Ranke
2021-11-24 22:35:24 +01:00
parent 11b43f9bc4
commit 5e82c65c18
4 changed files with 120 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
src/match_finder.rs
View File

@@ -6,7 +6,8 @@ pub struct MatchFinder {
rev_suffixes: Vec<u32>,
lcp: Vec<u32>,
max_matches: usize,
max_queue_size: usize,
max_matches_per_length: usize,
patience: usize,
max_length_diff: usize,
}
@@ -42,8 +43,9 @@ impl MatchFinder {
suffixes,
rev_suffixes,
lcp,
max_matches: 10,
patience: 10,
max_queue_size: 100,
max_matches_per_length: 5,
patience: 100,
max_length_diff: 2,
}
}
@@ -57,8 +59,8 @@ impl MatchFinder {
left_length: usize::MAX,
right_index: index,
right_length: usize::MAX,
current_length: 0,
matches_left: self.max_matches,
current_length: usize::MAX,
matches_left: 0,
max_length: 0,
queue: BinaryHeap::new(),
};
@@ -93,15 +95,16 @@ impl<'a> Iterator for Matches<'a> {
type Item = Match;
fn next(&mut self) -> Option<Match> {
if self.queue.is_empty() {
self.current_length = self.left_length.max(self.right_length);
if self.queue.is_empty() || self.matches_left == 0 {
self.queue.clear();
self.current_length = self.current_length.saturating_sub(1).min(self.left_length.max(self.right_length));
self.max_length = self.max_length.max(self.current_length);
if self.current_length < 2
|| self.current_length + self.finder.max_length_diff < self.max_length
{
return None;
}
while self.matches_left > 0
while self.queue.len() < self.finder.max_queue_size
&& (self.left_length == self.current_length
|| self.right_length == self.current_length)
{
@@ -109,13 +112,16 @@ impl<'a> Iterator for Matches<'a> {
self.add_to_queue(self.finder.suffixes[self.left_index]);
self.move_left();
}
if self.right_length == self.current_length && self.matches_left > 0 {
if self.right_length == self.current_length {
self.add_to_queue(self.finder.suffixes[self.right_index]);
self.move_right();
}
}
self.matches_left = self.finder.max_matches_per_length;
}
self.matches_left = self.matches_left.saturating_sub(1);
self.queue.pop().map(|pos| Match {
pos,
length: self.current_length,
@@ -144,7 +150,10 @@ impl<'a> Matches<'a> {
fn move_right(&mut self) {
let mut patience = self.finder.patience;
while self.right_length > 0 && patience > 0 && self.right_index + 1 < self.finder.suffixes.len() {
while self.right_length > 0
&& patience > 0
&& self.right_index + 1 < self.finder.suffixes.len()
{
self.right_index += 1;
self.right_length = self
.right_length
@@ -162,6 +171,5 @@ impl<'a> Matches<'a> {
fn add_to_queue(&mut self, pos: i32) {
self.queue.push(pos as usize);
self.matches_left -= 1;
}
}

77
src/parsing_packer.rs
View File

@@ -1,9 +1,9 @@
use std::collections::HashMap;
use std::rc::Rc;
use crate::match_finder::MatchFinder;
use crate::rans::{RansCoder, CostCounter};
use crate::lz;
use crate::match_finder::MatchFinder;
use crate::rans::{CostCounter, RansCoder};
pub fn pack(data: &[u8]) -> Vec<u8> {
let mut parse = parse(data);
@@ -34,10 +34,12 @@ struct Arrival {
type Arrivals = HashMap<usize, Vec<Arrival>>;
const MAX_ARRIVALS: usize = 16;
const MAX_ARRIVALS: usize = 4;
fn parse(data: &[u8]) -> Option<Rc<Parse>> {
let match_finder = MatchFinder::new(data);
let mut near_matches = [usize::MAX; 1024];
let mut last_seen = [usize::MAX; 256];
let mut arrivals: Arrivals = HashMap::new();
fn add_arrival(arrivals: &mut Arrivals, pos: usize, arrival: Arrival) {
@@ -54,10 +56,19 @@ fn parse(data: &[u8]) -> Option<Rc<Parse>> {
}
}
}
fn add_match(arrivals: &mut Arrivals, pos: usize, offset: usize, length: usize, arrival: &Arrival) {
fn add_match(
arrivals: &mut Arrivals,
pos: usize,
offset: usize,
length: usize,
arrival: &Arrival,
) {
let mut cost_counter = CostCounter(0.);
let mut state = arrival.state.clone();
let op = lz::Op::Match { offset: offset as u32, len: length as u32 };
let op = lz::Op::Match {
offset: offset as u32,
len: length as u32,
};
op.encode(&mut cost_counter, &mut state);
add_arrival(
arrivals,
@@ -81,20 +92,64 @@ fn parse(data: &[u8]) -> Option<Rc<Parse>> {
cost: 0.0,
},
);
let mut best_per_offset = HashMap::new();
for pos in 0..data.len() {
for arrival in arrivals.remove(&pos).unwrap() {
let match_length = |offset: usize| {
data[pos..]
.iter()
.zip(data[(pos - offset)..].iter())
.take_while(|(a, b)| a == b)
.count()
};
let here_arrivals = if let Some(arr) = arrivals.remove(&pos) {
arr
} else {
continue;
};
best_per_offset.clear();
let mut best_cost = f64::MAX;
for arrival in &here_arrivals {
best_cost = best_cost.min(arrival.cost);
let per_offset = best_per_offset
.entry(arrival.state.last_offset())
.or_insert(f64::MAX);
*per_offset = per_offset.min(arrival.cost);
}
for arrival in here_arrivals {
if arrival.cost > (best_cost + 32.0).min(*best_per_offset.get(&arrival.state.last_offset()).unwrap()) {
continue;
}
let mut found_last_offset = false;
let mut closest_match = None;
for m in match_finder.matches(pos) {
closest_match = Some(closest_match.unwrap_or(0).max(m.pos));
let offset = pos - m.pos;
if offset as u32 == arrival.state.last_offset() {
found_last_offset = true;
}
found_last_offset |= offset as u32 == arrival.state.last_offset();
add_match(&mut arrivals, pos, offset, m.length, &arrival);
}
let mut near_matches_left = 4;
let mut match_pos = last_seen[data[pos] as usize];
while near_matches_left > 0
&& match_pos != usize::MAX
&& closest_match.iter().all(|p| *p < match_pos)
{
let offset = pos - match_pos;
let length = match_length(offset);
assert!(length > 0);
add_match(&mut arrivals, pos, offset, length, &arrival);
found_last_offset |= offset as u32 == arrival.state.last_offset();
if offset < near_matches.len() {
match_pos = near_matches[match_pos % near_matches.len()];
}
near_matches_left -= 1;
}
if !found_last_offset && arrival.state.last_offset() > 0 {
let offset = arrival.state.last_offset() as usize;
let length = data[pos..].iter().zip(data[(pos - offset)..].iter()).take_while(|(a, b)| a == b).count();
let length = match_length(offset);
if length > 0 {
add_match(&mut arrivals, pos, offset, length, &arrival);
}
@@ -117,6 +172,8 @@ fn parse(data: &[u8]) -> Option<Rc<Parse>> {
},
);
}
near_matches[pos % near_matches.len()] = last_seen[data[pos] as usize];
last_seen[data[pos] as usize] = pos;
}
arrivals.remove(&data.len()).unwrap()[0].parse.clone()
}