2 Commits

Author SHA1 Message Date
4eab36b9d9 add some api documentation 2022-10-25 23:33:32 +02:00
7cec54f62b make crossterm dependency optional 2022-10-25 22:40:56 +02:00
6 changed files with 98 additions and 243 deletions

245
Cargo.lock generated
View File

@@ -36,34 +36,19 @@ dependencies = [
"sacabase",
]
[[package]]
name = "cfg-if"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
[[package]]
name = "cfg-if"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "cloudabi"
version = "0.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
dependencies = [
"bitflags",
]
[[package]]
name = "crossbeam-channel"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06ed27e177f16d65f0f0c22a213e17c696ace5dd64b14258b52f9417ccb52db4"
dependencies = [
"cfg-if 1.0.0",
"cfg-if",
"crossbeam-utils",
]
@@ -73,26 +58,10 @@ version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d82cfc11ce7f2c3faef78d8a684447b40d503d9681acebed6cb728d45940c4db"
dependencies = [
"cfg-if 1.0.0",
"cfg-if",
"lazy_static",
]
[[package]]
name = "crossterm"
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "207a948d1b4ff59e5aec9bb9426cc4fd3d17b719e5c7b74e27f0a60c4cc2d095"
dependencies = [
"bitflags",
"crossterm_winapi 0.6.2",
"lazy_static",
"libc",
"mio 0.6.23",
"parking_lot 0.10.2",
"signal-hook 0.1.17",
"winapi 0.3.9",
]
[[package]]
name = "crossterm"
version = "0.25.0"
@@ -100,22 +69,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e64e6c0fbe2c17357405f7c758c1ef960fce08bdfb2c03d88d2a18d7e09c4b67"
dependencies = [
"bitflags",
"crossterm_winapi 0.9.0",
"crossterm_winapi",
"libc",
"mio 0.8.4",
"parking_lot 0.12.1",
"signal-hook 0.3.14",
"mio",
"parking_lot",
"signal-hook",
"signal-hook-mio",
"winapi 0.3.9",
]
[[package]]
name = "crossterm_winapi"
version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2265c3f8e080075d9b6417aa72293fc71662f34b4af2612d8d1b074d29510db"
dependencies = [
"winapi 0.3.9",
"winapi",
]
[[package]]
@@ -124,42 +84,7 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ae1b35a484aa10e07fe0638d02301c5ad24de82d310ccbd2f3693da5f09bf1c"
dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "fuchsia-zircon"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82"
dependencies = [
"bitflags",
"fuchsia-zircon-sys",
]
[[package]]
name = "fuchsia-zircon-sys"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
[[package]]
name = "iovec"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e"
dependencies = [
"libc",
]
[[package]]
name = "kernel32-sys"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"
dependencies = [
"winapi 0.2.8",
"winapi-build",
"winapi",
]
[[package]]
@@ -180,15 +105,6 @@ version = "0.2.135"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68783febc7782c6c5cb401fbda4de5a9898be1762314da0bb2c10ced61f18b0c"
[[package]]
name = "lock_api"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75"
dependencies = [
"scopeguard",
]
[[package]]
name = "lock_api"
version = "0.4.9"
@@ -205,26 +121,7 @@ version = "0.4.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
dependencies = [
"cfg-if 1.0.0",
]
[[package]]
name = "mio"
version = "0.6.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4"
dependencies = [
"cfg-if 0.1.10",
"fuchsia-zircon",
"fuchsia-zircon-sys",
"iovec",
"kernel32-sys",
"libc",
"log",
"miow",
"net2",
"slab",
"winapi 0.2.8",
"cfg-if",
]
[[package]]
@@ -239,29 +136,6 @@ dependencies = [
"windows-sys 0.36.1",
]
[[package]]
name = "miow"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d"
dependencies = [
"kernel32-sys",
"net2",
"winapi 0.2.8",
"ws2_32-sys",
]
[[package]]
name = "net2"
version = "0.2.38"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "74d0df99cfcd2530b2e694f6e17e7f37b8e26bb23983ac530c0c97408837c631"
dependencies = [
"cfg-if 0.1.10",
"libc",
"winapi 0.3.9",
]
[[package]]
name = "num-traits"
version = "0.2.14"
@@ -271,38 +145,14 @@ dependencies = [
"autocfg",
]
[[package]]
name = "parking_lot"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e"
dependencies = [
"lock_api 0.3.4",
"parking_lot_core 0.7.2",
]
[[package]]
name = "parking_lot"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
dependencies = [
"lock_api 0.4.9",
"parking_lot_core 0.9.4",
]
[[package]]
name = "parking_lot_core"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3"
dependencies = [
"cfg-if 0.1.10",
"cloudabi",
"libc",
"redox_syscall 0.1.57",
"smallvec",
"winapi 0.3.9",
"lock_api",
"parking_lot_core",
]
[[package]]
@@ -311,9 +161,9 @@ version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4dc9e0dc2adc1c69d09143aff38d3d30c5c3f0df0dad82e6d25547af174ebec0"
dependencies = [
"cfg-if 1.0.0",
"cfg-if",
"libc",
"redox_syscall 0.2.16",
"redox_syscall",
"smallvec",
"windows-sys 0.42.0",
]
@@ -327,7 +177,7 @@ dependencies = [
"crossbeam-channel",
"libc",
"time",
"winapi 0.3.9",
"winapi",
]
[[package]]
@@ -348,12 +198,6 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "redox_syscall"
version = "0.1.57"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce"
[[package]]
name = "redox_syscall"
version = "0.2.16"
@@ -378,17 +222,6 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]]
name = "signal-hook"
version = "0.1.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e31d442c16f047a671b5a71e2161d6e68814012b7f5379d269ebd915fac2729"
dependencies = [
"libc",
"mio 0.6.23",
"signal-hook-registry",
]
[[package]]
name = "signal-hook"
version = "0.3.14"
@@ -406,8 +239,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af"
dependencies = [
"libc",
"mio 0.8.4",
"signal-hook 0.3.14",
"mio",
"signal-hook",
]
[[package]]
@@ -419,15 +252,6 @@ dependencies = [
"libc",
]
[[package]]
name = "slab"
version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef"
dependencies = [
"autocfg",
]
[[package]]
name = "smallvec"
version = "1.10.0"
@@ -445,16 +269,6 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "terminal"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9cd146db3fbe90911d0bb5498a393abda0af71a2c11e287ecb9ae30cf5d43667"
dependencies = [
"bitflags",
"crossterm 0.15.0",
]
[[package]]
name = "thiserror"
version = "1.0.36"
@@ -483,7 +297,7 @@ checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255"
dependencies = [
"libc",
"wasi 0.10.0+wasi-snapshot-preview1",
"winapi 0.3.9",
"winapi",
]
[[package]]
@@ -498,10 +312,9 @@ version = "0.2.0"
dependencies = [
"anyhow",
"cdivsufsort",
"crossterm 0.25.0",
"crossterm",
"lexopt",
"pbr",
"terminal",
"thiserror",
]
@@ -517,12 +330,6 @@ version = "0.11.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "winapi"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
[[package]]
name = "winapi"
version = "0.3.9"
@@ -533,12 +340,6 @@ dependencies = [
"winapi-x86_64-pc-windows-gnu",
]
[[package]]
name = "winapi-build"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"
[[package]]
name = "winapi-i686-pc-windows-gnu"
version = "0.4.0"
@@ -650,13 +451,3 @@ name = "windows_x86_64_msvc"
version = "0.42.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5"
[[package]]
name = "ws2_32-sys"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e"
dependencies = [
"winapi 0.2.8",
"winapi-build",
]

View File

@@ -12,5 +12,4 @@ lexopt = "0.2.1"
anyhow = "1"
thiserror = "1.0.36"
pbr = "1"
crossterm = { version = "0.25.0", default-features = false }
terminal = { version = "0.2.1", default-features = false, features = ["crossterm-backend"] }
crossterm = { version = "0.25.0", default-features = false, optional = true }

View File

@@ -23,12 +23,12 @@ upkr-windows-$(VERSION).zip: upkr-windows/upkr.exe PHONY
zip -r -9 $@ upkr-windows
upkr-linux/upkr:
cargo build --target x86_64-unknown-linux-musl --release
cargo build --target x86_64-unknown-linux-musl --release -F crossterm
mkdir -p upkr-linux
cp ../target/x86_64-unknown-linux-musl/release/upkr upkr-linux/
upkr-windows/upkr.exe:
cargo build --target x86_64-pc-windows-gnu --release
cargo build --target x86_64-pc-windows-gnu --release -F crossterm
mkdir -p upkr-windows
cp ../target/x86_64-pc-windows-gnu/release/upkr.exe upkr-windows/

View File

@@ -72,6 +72,7 @@ impl Heatmap {
self.data[index]
}
#[cfg(feature = "crossterm")]
pub fn print_as_hex(&self) -> std::io::Result<()> {
use crossterm::{
style::{Attribute, Color, Print, SetAttribute, SetBackgroundColor},

View File

@@ -1,3 +1,12 @@
#![warn(missing_docs)]
//! Compression and decompression of the upkr format and variants.
//!
//! Upkr is a compression format initially designed for the MicroW8 fantasy console,
//! with design goals being a competitive compression ratio, reasonable fast
//! decompression, low memory overhead and very small decompression code
//! when handoptimized in assembler. (An optimized DOS execuable decompressor is <140 bytes.)
mod context_state;
mod greedy_packer;
mod heatmap;
@@ -9,25 +18,58 @@ mod rans;
pub use heatmap::Heatmap;
pub use lz::{calculate_margin, create_heatmap, unpack, UnpackError};
/// The type of a callback function to be given to the `pack` function.
///
/// It will be periodically called with the number of bytes of the input already processed.
pub type ProgressCallback<'a> = &'a mut dyn FnMut(usize);
/// A configuration of which compression format variation to use.
///
/// Use `Config::default()` for the standard upkr format.
///
/// Compression format variants exist to help with micro-optimizations in uncompression
/// code on specific platforms.
#[derive(Debug)]
pub struct Config {
/// Shift in bits from a bitstream into the rANS state, rather than whole bytes.
/// This decreases the size of the rNAS state to 16 bits which is very useful on
/// 8 bit platforms.
pub use_bitstream: bool,
/// The number of parity contexts (usually 1, 2 or 4). This can improve compression
/// on data that consists of regular groups of 2 or 4 bytes. One example is 32bit ARM
/// code, where each instruction is 4 bytes, so `parity_contexts = 4` improves compression
/// quite a bit. Defaults to `1`.
pub parity_contexts: usize,
/// Invert the encoding of bits in the rANS coder. `bit = state_lo >= prob` instead of
/// `bit = state_lo < prob`.
pub invert_bit_encoding: bool,
/// The boolean value which encodes a match. Defaults to `true`.
pub is_match_bit: bool,
/// The boolean value which encodes a new offset (rather than re-using the previous offset).
/// Defaults to `true`.
pub new_offset_bit: bool,
/// The boolean value which encodes that there are more bits comming for length/offset values.
/// Defaults to `true`.
pub continue_value_bit: bool,
/// Reverses the bits in the bitstream.
pub bitstream_is_big_endian: bool,
/// A slightly less accurate, but slightly simpler variation of the prob update in the
/// rANS coder, Used for the z80 uncompressor.
pub simplified_prob_update: bool,
/// Disables support for re-using the last offset in the compression format.
/// This might save a few bytes when working with very small data.
pub no_repeated_offsets: bool,
/// Standard upkr encodes the EOF marker in the offset. This encodes it in the match length
/// instead.
pub eof_in_length: bool,
/// The maximum match offset value to encode when compressing.
pub max_offset: usize,
/// The maximum match length value to encode when compressing.
pub max_length: usize,
}
@@ -55,7 +97,7 @@ impl Default for Config {
}
impl Config {
pub fn min_length(&self) -> usize {
fn min_length(&self) -> usize {
if self.eof_in_length {
2
} else {
@@ -64,6 +106,21 @@ impl Config {
}
}
/// Compresses the given data.
///
/// # Arguments
/// - `data`: The data to compress
/// - `level`: The compression level (0-9). Increasing the level by one roughly halves the
/// compression speed.
/// - `config`: The compression format variant to use.
/// - `progress_callback`: An optional callback which will periodically be called with
/// the number of bytes already processed.
///
/// # Example
/// ```rust
/// let compressed_data = upkr::pack(b"Hello, World! Yellow world!", 0, &upkr::Config::default(), None);
/// assert!(compressed_data.len() < 27);
/// ```
pub fn pack(
data: &[u8],
level: u8,
@@ -77,6 +134,9 @@ pub fn pack(
}
}
/// Estimate the exact (fractional) size of upkr compressed data.
///
/// Note that this currently does NOT work for the bitstream variant.
pub fn compressed_size(mut data: &[u8]) -> f32 {
let mut state = 0;
while state < 4096 {

View File

@@ -10,6 +10,7 @@ fn main() -> Result<()> {
let mut unpack = false;
let mut calculate_margin = false;
let mut create_heatmap = false;
#[allow(unused_mut)]
let mut do_hexdump = false;
let mut level = 2;
let mut infile: Option<PathBuf> = None;
@@ -61,6 +62,7 @@ fn main() -> Result<()> {
Short('u') | Long("unpack") => unpack = true,
Long("margin") => calculate_margin = true,
Long("heatmap") => create_heatmap = true,
#[cfg(feature = "crossterm")]
Long("hexdump") => do_hexdump = true,
Short('l') | Long("level") => level = parser.value()?.parse()?,
Short(n) if n.is_ascii_digit() => level = n as u8 - b'0',
@@ -162,18 +164,20 @@ fn main() -> Result<()> {
if reverse {
heatmap.reverse();
}
if do_hexdump {
heatmap.print_as_hex()?;
} else {
let mut heatmap_bin = Vec::with_capacity(heatmap.len());
for i in 0..heatmap.len() {
let cost = (heatmap.cost(i).log2() * 8. + 64.)
.round()
.max(0.)
.min(127.) as u8;
heatmap_bin.push((cost << 1) | heatmap.is_literal(i) as u8);
match do_hexdump {
#[cfg(feature = "crossterm")]
true => heatmap.print_as_hex()?,
_ => {
let mut heatmap_bin = Vec::with_capacity(heatmap.len());
for i in 0..heatmap.len() {
let cost = (heatmap.cost(i).log2() * 8. + 64.)
.round()
.max(0.)
.min(127.) as u8;
heatmap_bin.push((cost << 1) | heatmap.is_literal(i) as u8);
}
File::create(outfile(OutFileType::Heatmap))?.write_all(&heatmap_bin)?;
}
File::create(outfile(OutFileType::Heatmap))?.write_all(&heatmap_bin)?;
}
}
if calculate_margin {