better parity contexts integration for literal length

Cargo.lock

@@ -172,7 +172,7 @@ checksum = "dcc811dc4066ac62f84f11307873c4850cb653bfa9b1719cee2bd2204a4bc5dd"
 
 [[package]]
 name = "upkr"
-version = "0.2.0"
+version = "0.2.0-pre3"
 dependencies = [
  "anyhow",
  "cdivsufsort",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "upkr"
-version = "0.2.0"
+version = "0.2.0-pre3"
 edition = "2021"
 
 [profile.release]

README.md

@@ -2,57 +2,13 @@
 
 Upkr is a simple general purpose lz packer designed to be used in the [MicroW8](https://github.com/exoticorn/microw8) platform.
 The compressed format is losely based on [Shrinkler](https://github.com/askeksa/Shrinkler) with the main difference being that
-Upkr doesn't differentiate between literals at odd or even addresses (by default) and that I went with rANS/rABS instead of a range coder.
+Upkr doesn't differnetiate between literals at odd or even addresses and that I went with rANS/rABS instead of a range coder.
 
-Compression rate is on par with Shrinkler.
+At this point, Upkr should still be considered unstable - the compressed format is not very likely to change but I still want
-
+to keep that option open a little longer.
-The differences compare to Shrinkler also makes it interesting on 8bit platforms. The z80 unpacker included in the release
-is both about twice as fast and smaller than the Shrinkler unpacker.
 
 ## Inspirations:
 
 * Ferris' blog about his [C64 intro packer](https://yupferris.github.io/blog/2020/08/31/c64-4k-intro-packer-deep-dive.html)
 * [Shrinkler](https://github.com/askeksa/Shrinkler)
-* Ryg's [sample rANS implementation](https://github.com/rygorous/ryg_rans)
+* Ryg's [sample rANS implementation](https://github.com/rygorous/ryg_rans)
-
-## Unpackers
-
-The release includes a reference c unpacker, as well as some optimized asm unpackers (arm and riscv). The unpckers in
-c_unpacker and asm_unpackers unpack the default upkr compressed format. The z80_unpacker
-is based on some variations to the compressed format. (Use `upkr --z80` to select those variations.)
-The 16 bit dos unpacker also uses some variations. (`upkr --x86`)
-
-## Usage
-
-```
-  upkr [-l level(0-9)] [config options] <infile> [<outfile>]
-  upkr -u [config options] <infile> [<outfile>]
-  upkr --margin [config options] <infile>
-
- -l, --level N       compression level 0-9
- -0, ..., -9         short form for setting compression level
- -u, --unpack        unpack infile
- --margin            calculate margin for overlapped unpacking of a packed file
-
-Config presets for specific unpackers:
- --z80               --big-endian-bitstream --invert-bit-encoding --simplified-prob-update -9
- --x86               --bitstream --invert-is-match-bit --invert-continue-value-bit --invert-new-offset-bit
- --x86b              --bitstream --invert-continue-value-bit --no-repeated-offsets -9
-
-Config options (need to match when packing/unpacking):
- -b, --bitstream     bitstream mode
- -p, --parity N      use N (2/4) parity contexts
- -r, --reverse       reverse input & output
-
-Config options to tailor output to specific optimized unpackers:
- --invert-is-match-bit
- --invert-new-offset-bit
- --invert-continue-value-bit
- --invert-bit-encoding
- --simplified-prob-update
- --big-endian-bitstream   (implies --bitstream)
- --no-repeated-offsets
- --eof-in-length
- --max-offset N
- --max-length N
-```

asm_unpackers/unpack_arm32.S

@@ -17,7 +17,7 @@
 upkr_unpack:
     push { r3-r11, lr }
 
-    mov r2, #384
+	mov r2, #384
     mov r3, #128
 .Lclear:
     subs r2, r2, #1
@@ -29,7 +29,7 @@ upkr_unpack:
     bl upkr_decode_bit
     bcc .Ldata
 .Lmatch:
-    mov r5, #256
+	mov r5, #256
     rsbs r6, r4, #0
     blcc upkr_decode_bit
     bcc .Lskip_offset
@@ -39,7 +39,7 @@ upkr_unpack:
     popeq { r3-r11, pc }
 .Lskip_offset:
 
-    mov r5, #256+64
+	mov r5, #256+64
     bl upkr_decode_length
 .Lcopy_loop:
     ldrb r5, [r0, r3]
@@ -55,46 +55,46 @@ upkr_unpack:
 .Ldata_loop:
     bl upkr_decode_bit
     adc r5, r5, r5
-    movs r4, r5, lsr #8
+	movs r4, r5, lsr #8
     beq .Ldata_loop
-    b .Lstore
+	b .Lstore
 
 .type upkr_decode_length, %function
 upkr_decode_length:
-    mov r12, lr
+	mov r12, lr
 
     mov r4, #0
-    mvn r6, #0
+	mvn r6, #0
 .Lbit_loop:
     bl upkr_decode_bit_inc
-    addcc r4, r4, r6
+	addcc r4, r4, r6
-    movcc pc, r12
+	movcc pc, r12
 
     bl upkr_decode_bit_inc
     addcs r4, r4, r6
-    mov r6, r6, lsl #1
+	mov r6, r6, lsl #1
     b .Lbit_loop
 
 .type upkr_decode_bit, %function
 upkr_decode_bit_inc:
-    add r5, r5, #1
+	add r5, r5, #1
 upkr_decode_bit:
-    cmp r2, #4096
+	cmp r2, #4096
-    ldrltb r8, [r1], #1
+	ldrltb r8, [r1], #1
-    orrlt r2, r8, r2, lsl#8
+	orrlt r2, r8, r2, lsl#8
-    blt upkr_decode_bit
+	blt upkr_decode_bit
 
     ldrb r8, [sp, -r5]
-    and r9, r2, #255
+	and r9, r2, #255
-    add r9, r9, #1
+	add r9, r9, #1
-    cmp r8, r9
+	cmp r8, r9
-    rsbcs r8, r8, #256
+	rsbcs r8, r8, #256
-    mvn r9, r2, lsr#8
+	mvn r9, r2, lsr#8
-    addcs r9, r9, #1
+	addcs r9, r9, #1
-    mla r2, r8, r9, r2
+	mla r2, r8, r9, r2
-    add r9, r8, #8
+	add r9, r8, #8
-    sub r8, r8, r9, lsr#4
+	sub r8, r8, r9, lsr#4
-    rsbcs r8, r8, #256
+	rsbcs r8, r8, #256
-    strb r8, [sp, -r5]
+	strb r8, [sp, -r5]
-    mov pc, r14
+	mov pc, r14
 

dos_unpacker/readme.txt

@@ -1,13 +0,0 @@
-16 bit DOS executable stubs
----------------------------
-
-by pestis and TomCat
-
-unpack_x86_16_DOS.asm:
-  maximum compatibility, relocates unpacked code to normal start address
-unpack_x86_16_DOS_no_relocation.asm:
-  saves some bytes by not relocating, unpacked code needs to be assembled to
-  start at 0x3FFE
-unpack_x86_16_DOS_no_repeated_offset.asm:
-  removes support for repeated offsets, potentially at the cost of some compression ratio.
-  most likely only a win in very narrow circumstances around the 1kb mark

dos_unpacker/unpack_x86_16_DOS.asm

@@ -1,161 +0,0 @@
-; Contributions from pestis, TomCat and exoticorn
-;
-; This is the 16-bit DOS x86 decompression stub for upkr, which is designed for
-; maximum compatibility: it relocates the compressed data so it can be
-; decompressed starting at the normal .COM starting address. In other words,
-; many of the already existing .COM files should be compressable using this
-; stub.
-;
-; How to use:
-;   1) Pack your intro using upkr into data.bin with the --x86 command line
-;      argument:
-;
-;           $ upkr --x86 intro.com data.bin
-;
-;   2) Compile this .asm file using nasm (or any compatible assembler):
-;
-;           $ nasm unpack_x86_16_DOS.asm -fbin -o intropck.com
-;
-; The packed size of the intro+stub is limited by max_len (see below) bytes.
-;
-; In specific cases, the unpacker stub can be further optimized to save a byte
-; or two:
-;   1) You can remove CLC before RET, if you don't mind carry being set upon
-;      program entry
-;   2) You can also move PUSHA before PUSH SI and put POPA as the first
-;      operation of the compressed code.
-max_len     equ 16384
-prog_start  equ (0x100+max_len+510+relocation-upkr_unpack)
-probs       equ (((prog_start+max_len+510)+255)/256)*256
-
-org 0x100
-
-; This is will be loaded at 0x100, but relocates the code and data to prog_start
-relocation:
-    push    si                  ; si = 0x100 at DOS start, so save it for later ret
-    pusha                       ; pusha to recall all registers before starting intro
-    push    si                  ; for pop di to start writing the output
-    mov     di, prog_start      ; the depacker & data are relocated from 0x100 to prog_start
-    mov     ch, max_len/512
-    rep     movsw
-    jmp     si                  ; jump to relocated upkr_unpack
-
-
-; upkr_unpack unpacks the code to 0x100 and runs it when done.
-upkr_unpack:
-    xchg    ax, bp              ; position in input bitstream (bp) = 0
-    cwd                         ; upkr_state (dx) = 0;
-    xchg    ax, cx              ; cx = 0x9XX
-    mov     al, 128             ; for(int i = 0; i < sizeof(upkr_probs); ++i) upkr_probs[i] = 128;
-    rep     stosb
-    pop     di                  ; u8* write_ptr = (u8*)destination;
-.mainloop:
-    mov     bx, probs
-    call    upkr_decode_bit
-    jc      .else               ; if(upkr_decode_bit(0)) {
-    mov     bh, (probs+256)/256
-    jcxz    .skip_call
-    call    upkr_decode_bit
-    jc      .skipoffset
-.skip_call:
-    stc
-    call    upkr_decode_number  ; offset = upkr_decode_length(258) - 1;
-    loop    .notdone            ; if(offset == 0)
-    popa
-    clc
-    ret
-.notdone:
-    mov     si, di
-.sub:
-    dec     si
-    loop    .sub
-.skipoffset:
-    mov     bl, 128             ; int length = upkr_decode_length(384);
-    call    upkr_decode_number
-    rep     movsb               ; *write_ptr = write_ptr[-offset];
-    jmp     .mainloop
-.byteloop:
-    call    upkr_decode_bit     ; int bit = upkr_decode_bit(byte);
-.else:
-    adc     bl, bl              ; byte = (byte << 1) + bit;
-    jnc     .byteloop
-    xchg    ax, bx
-    stosb
-    inc     si
-    mov     cl, 1
-    jmp     .mainloop           ;  prev_was_match = 0;
-
-
-; upkr_decode_bit decodes one bit from the rANS entropy encoded bit stream.
-; parameters:
-;    bx = memory address of the context probability
-;    dx = decoder state
-;    bp = bit position in input stream
-; returns:
-;    dx = new decoder state
-;    bp = new bit position in input stream
-;    carry = bit
-; trashes ax
-upkr_load_bit:
-    bt      [compressed_data-relocation+prog_start], bp
-    inc     bp
-    adc     dx, dx
-upkr_decode_bit:
-    inc     dx              ; inc dx, dec dx is used to test the top (sign) bit of dx
-    dec     dx
-    jns     upkr_load_bit
-    movzx   ax, byte [bx]   ; u16 prob = upkr_probs[context_index]
-    push    ax              ; save prob, tmp = prob
-    cmp     dl, al          ; int bit = (upkr_state & 255) < prob ? 1 : 0; (carry = bit)
-    pushf                   ; save bit flags
-    jc      .bit            ; (skip if bit)
-    neg     al              ;   tmp = 256 - tmp;
-.bit:
-    mov     [bx], al        ; upkr_probs[context_index] = tmp + (256 - tmp + 8) >> 4;
-    neg     byte [bx]
-    shr     byte [bx], 4
-    adc     [bx], al
-    mul     dh              ; upkr_state = tmp * (upkr_state >> 8) + (upkr_state & 255);
-    mov     dh, 0
-    add     dx, ax
-    popf
-    pop     ax
-    jc      .bit2           ; (skip if bit)
-    neg     byte [bx]       ;   tmp = 256 - tmp;
-    sub     dx, ax          ;   upkr_state -= prob; note that this will also leave carry always unset, which is what we want
-.bit2:
-    ret                     ; return the bit in carry
-
-
-; upkr_decode_number loads a variable length encoded number (up to 16 bits) from
-; the compressed stream. Only numbers 1..65535 can be encoded. If the encoded
-; number has 4 bits and is 1ABC, it is encoded using a kind of an "interleaved
-; elias code": 0A0B0C1. The 1 in the end implies that no more bits are coming.
-; parameters:
-;   cx = must be 0
-;   bx = memory address of the context probability
-;   dx = decoder state
-;   bp = bit position in input stream
-;   carry = must be 1
-; returns:
-;   cx = length
-;   dx = new decoder state
-;   bp = new bit position in input stream
-;   carry = 1
-; trashes bl, ax
-upkr_decode_number_loop:
-    inc     bx
-    call    upkr_decode_bit
-upkr_decode_number:
-    rcr     cx, 1
-    inc     bx
-    call    upkr_decode_bit
-    jnc     upkr_decode_number_loop     ; 0 = there's more bits coming, 1 = no more bits
-.loop2:
-    rcr     cx, 1
-    jnc     .loop2
-    ret
-
-
-compressed_data:
-   incbin   "data.bin"

dos_unpacker/unpack_x86_16_DOS_no_relocation.asm

@@ -1,152 +0,0 @@
-; Contributions from pestis, TomCat and exoticorn
-;
-; This is the 16-bit DOS x86 decompression stub for upkr, which decompresses the
-; code starting at address 0x3FFE (or whatever is defined by the entrypoint
-; below). Thus, the packed code needs to be assembled with org 0x3FFE to work.
-;
-; How to use:
-;   1) Put POPA as the first instruction of your compiled code and use org
-;      0x3FFE
-;   2) Pack your intro using upkr into data.bin with the --x86 command line
-;      argument:
-;
-;           $ upkr --x86 intro.com data.bin
-;
-;   2) Compile this .asm file using nasm (or any compatible assembler) e.g.
-;
-;           $ nasm unpack_x86_16_DOS_no_relocation.asm -fbin -o intropck.com
-;
-; In specific cases, the unpacker stub can be further optimized to save a byte
-; or two:
-;   1) If your stub+compressed code is 2k or smaller, you can save 1 byte by
-;      putting probs at 0x900 and initializing DI with SALC; XCHG AX, DI instead
-;      of MOV DI, probs
-;   2) If you remove the PUSHA (and POPA in the compressed code), then you can
-;      assume the registers as follows: AX = 0x00XX, BX = probs + 0x1XX, CX = 0
-;      DX = (trash), SI = DI = right after your program, SP = as it was when the
-;      program started, flags = carry set
-;
-; Note that even with the PUSHA / POPA, carry will be set (!) unlike normal dos
-; program.
-entry       equ 0x3FFE
-probs       equ entry - 0x1FE   ; must be aligned to 256
-
-org 0x100
-
-
-; This is will be loaded at 0x100, but relocates the code and data to prog_start
-upkr_unpack:
-    pusha
-    xchg    ax, bp              ; position in bitstream = 0
-    cwd                         ; upkr_state = 0;
-    mov     di, probs
-    mov     ax, 0x8080          ; for(int i = 0; i < sizeof(upkr_probs); ++i) upkr_probs[i] = 128;
-    rep     stosw
-    push    di
-.mainloop:
-    mov     bx, probs
-    call    upkr_decode_bit
-    jc      .else               ; if(upkr_decode_bit(0)) {
-    mov     bh, (probs+256)/256
-    jcxz    .skip_call          ; if(prev_was_match || upkr_decode_bit(257)) {
-    call    upkr_decode_bit
-    jc      .skipoffset
-.skip_call:
-    stc
-    call    upkr_decode_number  ;  offset = upkr_decode_number(258) - 1;
-    mov     si, di
-    loop    .sub                ; if(offset == 0)
-    ret
-.sub:
-    dec     si
-    loop    .sub
-.skipoffset:
-    mov     bl, 128             ; int length = upkr_decode_number(384);
-    call    upkr_decode_number
-    rep     movsb               ; *write_ptr = write_ptr[-offset];
-    jmp     .mainloop
-.byteloop:
-    call    upkr_decode_bit     ; int bit = upkr_decode_bit(byte);
-.else:
-    adc     bl, bl              ; byte = (byte << 1) + bit;
-    jnc     .byteloop
-    xchg    ax, bx
-    stosb
-    inc     si
-    mov     cl, 1
-    jmp     .mainloop           ;  prev_was_match = 0;
-
-
-; upkr_decode_bit decodes one bit from the rANS entropy encoded bit stream.
-; parameters:
-;    bx = memory address of the context probability
-;    dx = decoder state
-;    bp = bit position in input stream
-; returns:
-;    dx = new decoder state
-;    bp = new bit position in input stream
-;    carry = bit
-; trashes ax
-upkr_load_bit:
-    bt      [compressed_data], bp
-    inc     bp
-    adc     dx, dx
-upkr_decode_bit:
-    inc     dx
-    dec     dx              ; inc dx, dec dx is used to test the top (sign) bit of dx
-    jns     upkr_load_bit
-    movzx   ax, byte [bx]   ; int prob = upkr_probs[context_index]
-    push    ax              ; save prob
-    cmp     dl, al          ; int bit = (upkr_state & 255) < prob ? 1 : 0; (carry = bit)
-    pushf                   ; save bit flags
-    jc      .bit            ; (skip if bit)
-    neg     al              ;   tmp = 256 - tmp;
-.bit:
-    mov     [bx], al        ; tmp_new = tmp + (256 - tmp + 8) >> 4;
-    neg     byte [bx]
-    shr     byte [bx], 4
-    adc     [bx], al
-    mul     dh              ; upkr_state = tmp * (upkr_state >> 8) + (upkr_state & 255);
-    mov     dh, 0
-    add     dx, ax
-    popf
-    pop     ax
-    jc      .bit2           ; (skip if bit)
-    neg     byte [bx]       ;   tmp = 256 - tmp;
-    sub     dx, ax          ;    upkr_state -= prob; note that this will also leave carry always unset, which is what we want
-.bit2:
-    ret                     ; flags = bit
-
-
-; upkr_decode_number loads a variable length encoded number (up to 16 bits) from
-; the compressed stream. Only numbers 1..65535 can be encoded. If the encoded
-; number has 4 bits and is 1ABC, it is encoded using a kind of an "interleaved
-; elias code": 0A0B0C1. The 1 in the end implies that no more bits are coming.
-; parameters:
-;   cx = must be 0
-;   bx = memory address of the context probability
-;   dx = decoder state
-;   bp = bit position in input stream
-;   carry = must be 1
-; returns:
-;   cx = length
-;   dx = new decoder state
-;   bp = new bit position in input stream
-;   carry = 1
-; trashes bl, ax
-upkr_decode_number_loop:
-    inc     bx
-    call    upkr_decode_bit
-upkr_decode_number:
-    rcr     cx, 1
-    inc     bx
-    call    upkr_decode_bit
-    jnc     upkr_decode_number_loop ; while(upkr_decode_bit(context_index)) {
-.loop2:
-    rcr     cx, 1
-    jnc     .loop2
-    ret
-
-
-compressed_data:
-    incbin  "data.bin"

dos_unpacker/unpack_x86_16_DOS_no_repeated_offset.asm

@@ -1,155 +0,0 @@
-; Contributions from pestis, TomCat and exoticorn
-;
-; This is the 16-bit DOS x86 decompression stub for upkr, which is designed for
-; the --no-repeated-offsets option of upkr. The decompression stub is slightly
-; smaller, but the compressed data might be bigger, so you have to test if
-; --no-repeated-offsets pays off in the end. This stub relocates the compressed
-; data so it can be decompressed starting at the normal .COM starting address.
-;
-; How to use:
-;   1) Pack your intro using upkr into data.bin with the --x86b command line
-;      argument: (notice the --x86b, not --x86!)
-;
-;           $ upkr --x86b intro.com data.bin
-;
-;   2) Compile this .asm file using nasm (or any compatible assembler):
-;
-;           $ nasm unpack_x86_16_DOS_no_repeated_offsets.asm -fbin -o intropck.com
-;
-; The packed size of the intro+stub is limited by max_len (see below) bytes.
-;
-; In specific cases, the unpacker stub can be further optimized to save a byte
-; or two:
-;   1) You can remove CLC before RET, if you don't mind carry being set upon
-;      program entry
-;   2) You can also move PUSHA before PUSH SI and put POPA as the first
-;      operation of the compressed code.
-max_len     equ 16384
-prog_start  equ (0x100+max_len+510+relocation-upkr_unpack)
-probs       equ (((prog_start+max_len+510)+255)/256)*256
-
-org 0x100
-
-
-; This is will be loaded at 0x100, but relocates the code and data to prog_start
-relocation:
-    push    si                  ; si = 0x100 at DOS start, so save it for later ret
-    pusha                       ; pusha to recall all registers before starting intro
-    push    si                  ; for pop di to start writing the output
-    mov     di, prog_start      ; the depacker & data are relocated from 0x100 to prog_start
-    mov     ch, max_len/512
-    rep     movsw
-    jmp     si                  ; jump to relocated upkr_unpack
-
-
-; upkr_unpack unpacks the code to 0x100 and runs it when done.
-upkr_unpack:
-    xchg    ax, bp              ; position in bitstream = 0
-    cwd                         ; upkr_state = 0;
-    xchg    cx, ax              ; cx > 0x0200
-    mov     al, 128             ; for(int i = 0; i < sizeof(upkr_probs); ++i) upkr_probs[i] = 128;
-    rep     stosb
-    pop     di                  ; u8* write_ptr = (u8*)destination;
-.mainloop:
-    mov     bx, probs
-    call    upkr_decode_bit
-    jnc     .else               ; if(upkr_decode_bit(0)) {
-    inc     bh
-    call    upkr_decode_number  ;  offset = upkr_decode_number(258) - 1;
-    loop    .notdone            ; if(offset == 0)
-    popa
-    clc
-    ret
-.notdone:
-    mov     si, di
-.sub:
-    dec     si
-    loop    .sub
-    mov     bl, 128             ; int length = upkr_decode_number(384);
-    call    upkr_decode_number
-    rep     movsb               ; *write_ptr = write_ptr[-offset];
-    jmp     .mainloop
-.else:
-    inc     bx
-.byteloop:
-    call    upkr_decode_bit     ; int bit = upkr_decode_bit(byte);
-    adc     bl, bl              ; byte = (byte << 1) + bit;
-    jnc     .byteloop
-    xchg    ax, bx
-    stosb
-    jmp     .mainloop           ;  prev_was_match = 0;
-
-
-; upkr_decode_bit decodes one bit from the rANS entropy encoded bit stream.
-; parameters:
-;    bx = memory address of the context probability
-;    dx = decoder state
-;    bp = bit position in input stream
-; returns:
-;    dx = new decoder state
-;    bp = new bit position in input stream
-;    carry = bit
-; trashes ax
-upkr_load_bit:
-    bt      [compressed_data-relocation+prog_start], bp
-    inc     bp
-    adc     dx, dx
-upkr_decode_bit:
-    inc     dx
-    dec     dx              ; or whatever other test for the top bit there is
-    jns     upkr_load_bit
-    movzx   ax, byte [bx]   ; int prob = upkr_probs[context_index]
-    push    ax              ; save prob
-    cmp     dl, al          ; int bit = (upkr_state & 255) < prob ? 1 : 0; (carry = bit)
-    pushf                   ; save bit flags
-    jc      .bit            ; (skip if bit)
-    neg     al              ;   tmp = 256 - tmp;
-.bit:
-    mov     [bx], al        ; tmp += (256 - tmp + 8) >> 4;
-    neg     byte [bx]
-    shr     byte [bx],4
-    adc     [bx], al        ; upkr_probs[context_index] = tmp;
-    mul     dh              ; upkr_state = tmp * (upkr_state >> 8) + (upkr_state & 255);
-    mov     dh, 0
-    add     dx, ax
-    popf
-    pop     ax
-    jc      .bit2           ; (skip if bit)
-    neg     byte [bx]       ;   tmp = 256 - tmp;
-    sub     dx, ax          ;   upkr_state -= prob; note that this will also leave carry always unset, which is what we want
-.bit2:
-    ret                     ; flags = bit
-
-
-; upkr_decode_number loads a variable length encoded number (up to 16 bits) from
-; the compressed stream. Only numbers 1..65535 can be encoded. If the encoded
-; number has 4 bits and is 1ABC, it is encoded using a kind of an "interleaved
-; elias code": 0A0B0C1. The 1 in the end implies that no more bits are coming.
-; parameters:
-;   cx = must be 0
-;   bx = memory address of the context probability
-;   dx = decoder state
-;   bp = bit position in input stream
-;   carry = must be 1
-; returns:
-;   cx = length
-;   dx = new decoder state
-;   bp = new bit position in input stream
-;   carry = 1
-; trashes bl, ax
-upkr_decode_number_loop:
-    inc     bx
-    call    upkr_decode_bit
-upkr_decode_number:
-    rcr     cx, 1
-    inc     bx
-    call    upkr_decode_bit
-    jnc     upkr_decode_number_loop     ; 0 = there's more bits coming, 1 = no more bits
-.loop2:
-    rcr     cx, 1
-    jnc     .loop2
-    ret
-
-
-compressed_data:
-    incbin "data.bin"

release/.gitignore

@@ -1,4 +0,0 @@
-*.zip
-*.tgz
-upkr-linux/
-upkr-windows/

release/Makefile

@@ -1,35 +0,0 @@
-VERSION := $(shell cargo run --release -- --version)
-
-all: clean upkr-linux-$(VERSION).tgz upkr-windows-$(VERSION).zip
-
-clean:
-	rm -rf upkr-linux
-	rm -f upkr-linux*.tgz
-	rm -rf upkr-windows
-	rm -f upkr-windows*.zip
-
-upkr-linux-$(VERSION).tgz: upkr-linux/upkr PHONY
-	cp ../README.md upkr-linux
-	cd .. && git archive HEAD c_unpacker | tar -xC release/upkr-linux
-	cd .. && git archive HEAD z80_unpacker | tar -xC release/upkr-linux
-	cd .. && git archive HEAD asm_unpackers | tar -xC release/upkr-linux
-	tar czf $@ upkr-linux
-
-upkr-windows-$(VERSION).zip: upkr-windows/upkr.exe PHONY
-	cp ../README.md upkr-windows/
-	cd .. && git archive HEAD c_unpacker | tar -xC release/upkr-windows
-	cd .. && git archive HEAD z80_unpacker | tar -xC release/upkr-windows
-	cd .. && git archive HEAD asm_unpackers | tar -xC release/upkr-windows
-	zip -r -9 $@ upkr-windows
-
-upkr-linux/upkr:
-	cargo build --target x86_64-unknown-linux-musl --release
-	mkdir -p upkr-linux
-	cp ../target/x86_64-unknown-linux-musl/release/upkr upkr-linux/
-
-upkr-windows/upkr.exe:
-	cargo build --target x86_64-pc-windows-gnu --release
-	mkdir -p upkr-windows
-	cp ../target/x86_64-pc-windows-gnu/release/upkr.exe upkr-windows/
-	
-PHONY:

src/greedy_packer.rs

@@ -12,6 +12,8 @@ pub fn pack(
     let mut rans_coder = RansCoder::new(config);
     let mut state = lz::CoderState::new(config);
 
+    let mut literal = vec![];
+
     let mut pos = 0;
     while pos < data.len() {
         if let Some(ref mut cb) = progress_callback {
@@ -22,6 +24,10 @@ pub fn pack(
             let max_offset = config.max_offset.min(1 << (m.length * 3 - 1).min(31));
             let offset = pos - m.pos;
             if offset < max_offset && m.length >= config.min_length() {
+                if !literal.is_empty() {
+                    lz::Op::Literal(literal).encode(&mut rans_coder, &mut state, config);
+                    literal = vec![];
+                }
                 let length = m.length.min(config.max_length);
                 lz::Op::Match {
                     offset: offset as u32,
@@ -43,6 +49,10 @@ pub fn pack(
                     .count()
                     .min(config.max_length);
                 if length >= config.min_length() {
+                    if !literal.is_empty() {
+                        lz::Op::Literal(literal).encode(&mut rans_coder, &mut state, config);
+                        literal = vec![];
+                    }
                     lz::Op::Match {
                         offset: offset as u32,
                         len: length as u32,
@@ -55,11 +65,14 @@ pub fn pack(
         }
 
         if !encoded_match {
-            lz::Op::Literal(data[pos]).encode(&mut rans_coder, &mut state, config);
+            literal.push(data[pos]);
             pos += 1;
         }
     }
 
+    if !literal.is_empty() {
+        lz::Op::Literal(literal).encode(&mut rans_coder, &mut state, config);
+    }
     lz::encode_eof(&mut rans_coder, &mut state, config);
     rans_coder.finish()
 }

src/lz.rs

@@ -3,36 +3,53 @@ use crate::rans::{EntropyCoder, RansDecoder};
 use crate::Config;
 use thiserror::Error;
 
-#[derive(Copy, Clone, Debug)]
+#[derive(Clone, Debug)]
 pub enum Op {
-    Literal(u8),
+    Literal(Vec<u8>),
     Match { offset: u32, len: u32 },
 }
 
 impl Op {
     pub fn encode(&self, coder: &mut dyn EntropyCoder, state: &mut CoderState, config: &Config) {
-        let literal_base = state.pos % state.parity_contexts * 256;
         match self {
-            &Op::Literal(lit) => {
+            &Op::Literal(ref lit) => {
-                encode_bit(coder, state, literal_base, !config.is_match_bit);
+                assert!(state.prev_was_match);
-                let mut context_index = 1;
+                encode_length(
-                for i in (0..8).rev() {
+                    coder,
-                    let bit = (lit >> i) & 1 != 0;
+                    state,
-                    encode_bit(coder, state, literal_base + context_index, bit);
+                    256 + state.pos % state.parity_contexts * 320,
-                    context_index = (context_index << 1) | bit as usize;
+                    lit.len() as u32 + 1,
+                    config,
+                );
+                for lit in lit {
+                    let literal_base = state.pos % state.parity_contexts * 320;
+                    let mut context_index = 1;
+                    for i in (0..8).rev() {
+                        let bit = (lit >> i) & 1 != 0;
+                        encode_bit(coder, state, literal_base + context_index, bit);
+                        context_index = (context_index << 1) | bit as usize;
+                    }
+                    state.pos += 1;
                 }
                 state.prev_was_match = false;
-                state.pos += 1;
             }
             &Op::Match { offset, len } => {
-                encode_bit(coder, state, literal_base, config.is_match_bit);
+                if state.prev_was_match {
+                    encode_length(
+                        coder,
+                        state,
+                        256 + state.pos % state.parity_contexts * 320,
+                        1,
+                        config,
+                    );
+                }
                 let mut new_offset = true;
                 if !state.prev_was_match && !config.no_repeated_offsets {
                     new_offset = offset != state.last_offset;
                     encode_bit(
                         coder,
                         state,
-                        256 * state.parity_contexts,
+                        320 * state.parity_contexts,
                         new_offset == config.new_offset_bit,
                     );
                 }
@@ -41,14 +58,14 @@ impl Op {
                     encode_length(
                         coder,
                         state,
-                        256 * state.parity_contexts + 1,
+                        320 * state.parity_contexts + 1,
                         offset + if config.eof_in_length { 0 } else { 1 },
                         config,
                     );
                     state.last_offset = offset;
                 }
                 assert!(len as usize >= config.min_length() && len as usize <= config.max_length);
-                encode_length(coder, state, 256 * state.parity_contexts + 65, len, config);
+                encode_length(coder, state, 320 * state.parity_contexts + 65, len, config);
                 state.prev_was_match = true;
                 state.pos += len as usize;
             }
@@ -57,25 +74,28 @@ impl Op {
 }
 
 pub fn encode_eof(coder: &mut dyn EntropyCoder, state: &mut CoderState, config: &Config) {
-    encode_bit(
+    if state.prev_was_match {
-        coder,
+        encode_length(
-        state,
+            coder,
-        state.pos % state.parity_contexts * 256,
+            state,
-        config.is_match_bit,
+            256 + state.pos % state.parity_contexts * 320,
-    );
+            1,
+            config,
+        );
+    }
     if !state.prev_was_match && !config.no_repeated_offsets {
         encode_bit(
             coder,
             state,
-            256 * state.parity_contexts,
+            320 * state.parity_contexts,
             config.new_offset_bit ^ config.eof_in_length,
         );
     }
     if !config.eof_in_length || state.prev_was_match || config.no_repeated_offsets {
-        encode_length(coder, state, 256 * state.parity_contexts + 1, 1, config);
+        encode_length(coder, state, 320 * state.parity_contexts + 1, 1, config);
     }
     if config.eof_in_length {
-        encode_length(coder, state, 256 * state.parity_contexts + 65, 1, config);
+        encode_length(coder, state, 320 * state.parity_contexts + 65, 1, config);
     }
 }
 
@@ -119,9 +139,9 @@ pub struct CoderState {
 impl CoderState {
     pub fn new(config: &Config) -> CoderState {
         CoderState {
-            contexts: ContextState::new((1 + 255) * config.parity_contexts + 1 + 64 + 64, config),
+            contexts: ContextState::new((64 + 256) * config.parity_contexts + 1 + 64 + 64, config),
             last_offset: 0,
-            prev_was_match: false,
+            prev_was_match: true,
             pos: 0,
             parity_contexts: config.parity_contexts,
         }
@@ -168,7 +188,8 @@ pub fn unpack_internal(
     max_size: usize,
 ) -> Result<isize, UnpackError> {
     let mut decoder = RansDecoder::new(packed_data, &config);
-    let mut contexts = ContextState::new((1 + 255) * config.parity_contexts + 1 + 64 + 64, &config);
+    let mut contexts =
+        ContextState::new((64 + 256) * config.parity_contexts + 1 + 64 + 64, &config);
     let mut offset = usize::MAX;
     let mut position = 0usize;
     let mut prev_was_match = false;
@@ -199,50 +220,14 @@ pub fn unpack_internal(
 
     loop {
         margin = margin.max(position as isize - decoder.pos() as isize);
-        let literal_base = position % config.parity_contexts * 256;
+        let literal_length = decode_length(
-        if decoder.decode_with_context(&mut contexts.context_mut(literal_base))?
+            &mut decoder,
-            == config.is_match_bit
+            &mut contexts,
-        {
+            256 + position % config.parity_contexts * 320,
-            if config.no_repeated_offsets
+            config,
-                || prev_was_match
+        )? - 1;
-                || decoder
+        for _ in 0..literal_length {
-                    .decode_with_context(&mut contexts.context_mut(256 * config.parity_contexts))?
+            let literal_base = position % config.parity_contexts * 320;
-                    == config.new_offset_bit
-            {
-                offset = decode_length(
-                    &mut decoder,
-                    &mut contexts,
-                    256 * config.parity_contexts + 1,
-                    &config,
-                )? - if config.eof_in_length { 0 } else { 1 };
-                if offset == 0 {
-                    break;
-                }
-            }
-            let length = decode_length(
-                &mut decoder,
-                &mut contexts,
-                256 * config.parity_contexts + 65,
-                &config,
-            )?;
-            if config.eof_in_length && length == 1 {
-                break;
-            }
-            if offset > position {
-                return Err(UnpackError::OffsetOutOfRange { offset, position });
-            }
-            if let Some(ref mut result) = result {
-                for _ in 0..length {
-                    if result.len() < max_size {
-                        result.push(result[result.len() - offset]);
-                    } else {
-                        break;
-                    }
-                }
-            }
-            position += length;
-            prev_was_match = true;
-        } else {
             let mut context_index = 1;
             let mut byte = 0;
             for i in (0..8).rev() {
@@ -259,6 +244,46 @@ pub fn unpack_internal(
             position += 1;
             prev_was_match = false;
         }
+
+        if config.no_repeated_offsets
+            || prev_was_match
+            || decoder
+                .decode_with_context(&mut contexts.context_mut(320 * config.parity_contexts))?
+                == config.new_offset_bit
+        {
+            offset = decode_length(
+                &mut decoder,
+                &mut contexts,
+                320 * config.parity_contexts + 1,
+                &config,
+            )? - if config.eof_in_length { 0 } else { 1 };
+            if offset == 0 {
+                break;
+            }
+        }
+        let length = decode_length(
+            &mut decoder,
+            &mut contexts,
+            320 * config.parity_contexts + 65,
+            &config,
+        )?;
+        if config.eof_in_length && length == 1 {
+            break;
+        }
+        if offset > position {
+            return Err(UnpackError::OffsetOutOfRange { offset, position });
+        }
+        if let Some(ref mut result) = result {
+            for _ in 0..length {
+                if result.len() < max_size {
+                    result.push(result[result.len() - offset]);
+                } else {
+                    break;
+                }
+            }
+        }
+        position += length;
+        prev_was_match = true;
     }
 
     if position > max_size {

src/main.rs

@@ -61,10 +61,6 @@ fn main() -> Result<()> {
             Short('l') | Long("level") => level = parser.value()?.parse()?,
             Short(n) if n.is_ascii_digit() => level = n as u8 - b'0',
             Short('h') | Long("help") => print_help(0),
-            Long("version") => {
-                println!("{}", env!("CARGO_PKG_VERSION"));
-                process::exit(0);
-            }
             Long("max-unpacked-size") => max_unpacked_size = parser.value()?.parse()?,
             Value(val) if infile.is_none() => infile = Some(val.try_into()?),
             Value(val) if outfile.is_none() => outfile = Some(val.try_into()?),
@@ -159,8 +155,6 @@ fn print_help(exit_code: i32) -> ! {
     eprintln!(" -u, --unpack        unpack infile");
     eprintln!(" --margin            calculate margin for overlapped unpacking of a packed file");
     eprintln!();
-    eprintln!("Version: {}", env!("CARGO_PKG_VERSION"));
-    eprintln!();
     eprintln!("Config presets for specific unpackers:");
     eprintln!(" --z80               --big-endian-bitstream --invert-bit-encoding --simplified-prob-update -9");
     eprintln!(

src/parsing_packer.rs

@@ -15,7 +15,7 @@ pub fn pack(
     let mut parse = parse(data, Config::from_level(level), config, progress_cb);
     let mut ops = vec![];
     while let Some(link) = parse {
-        ops.push(link.op);
+        ops.push(link.op.clone());
         parse = link.prev.clone();
     }
     let mut state = lz::CoderState::new(config);
@@ -32,9 +32,15 @@ struct Parse {
     op: lz::Op,
 }
 
+struct LiteralPrefix {
+    arrival: Arrival,
+    prefix: Vec<u8>,
+}
+
 struct Arrival {
     parse: Option<Rc<Parse>>,
     state: lz::CoderState,
+    literal_prefix: Option<Box<LiteralPrefix>>,
     cost: f64,
 }
 
@@ -130,6 +136,7 @@ fn parse(
                     op,
                 })),
                 state,
+                literal_prefix: None,
                 cost: arrival.cost + cost_counter.cost(),
             },
             max_arrivals,
@@ -141,6 +148,7 @@ fn parse(
         Arrival {
             parse: None,
             state: lz::CoderState::new(encoding_config),
+            literal_prefix: None,
             cost: 0.0,
         },
         max_arrivals,
@@ -252,19 +260,26 @@ fn parse(
             }
 
             cost_counter.reset();
-            let mut state = arrival.state;
+            let (arrival, mut prefix) = if let Some(prefix) = arrival.literal_prefix {
-            let op = lz::Op::Literal(data[pos]);
+                (prefix.arrival, prefix.prefix)
+            } else {
+                (arrival, vec![])
+            };
+            let mut state = arrival.state.clone();
+            prefix.push(data[pos]);
+            let op = lz::Op::Literal(prefix.clone());
             op.encode(cost_counter, &mut state, encoding_config);
             add_arrival(
                 &mut arrivals,
                 pos + 1,
                 Arrival {
                     parse: Some(Rc::new(Parse {
-                        prev: arrival.parse,
+                        prev: arrival.parse.clone(),
                         op,
                     })),
                     state,
                     cost: arrival.cost + cost_counter.cost(),
+                    literal_prefix: Some(Box::new(LiteralPrefix { arrival, prefix })),
                 },
                 max_arrivals,
             );