add highlevel description of compressed format in unpack.c

README.md

@@ -27,6 +27,7 @@ The 16 bit dos unpacker also uses some variations. (`upkr --x86`)
 * [Atari Lynx](https://github.com/42Bastian/new_bll/blob/master/demos/depacker/unupkr.asm)
 * [Atari Jaguar](https://github.com/42Bastian/new_bjl/blob/main/exp/depacker/unupkr.js)
 * [8080, R800](https://github.com/ivagorRetrocomp/DeUpkr)
+* [6502](https://github.com/pfusik/upkr6502)
 
 ## Usage
 

asm_unpackers/unpack_jagrisc.js

@@ -0,0 +1,190 @@
+;;; -*-asm-*-
+;;; ukpr unpacker for Atari Jaguar RISC.
+
+;;; lyxass syntax
+
+
+; input:
+;;; R20 : packed buffer
+;;; R21 : output buffer
+;;; r30 : return address
+;;;
+;;; Register usage (destroyed!)
+;;; r0-r17,r20,r21
+;;;
+
+DST		REG 21
+SRC		REG 20
+
+	REGTOP 16
+LR_save		REG 99
+LR_save2	REG 99
+GETBIT		REG 99
+GETLENGTH	REG 99
+LITERAL		REG 99
+LOOP		REG 99
+index		REG 99
+bit_pos		REG 99
+state		REG 99
+prev_was_match	REG 99
+offset		REG 99
+prob		reg 99
+byte		REG 99
+PROBS		reg 99
+tmp2		reg 2
+tmp1		REG 1
+tmp0		REG 0
+
+	REGMAP
+
+upkr_probs	equ $200
+
+SIZEOF_PROBS	EQU 1+255+1+2*32+2*32
+
+unupkr::
+	move	LR,LR_save
+	moveq	#0,tmp0
+	movei	#upkr_probs,PROBS
+	bset	#7,tmp0
+	movei	#SIZEOF_PROBS,tmp2
+	move	PROBS,tmp1
+.init	storeb	tmp0,(tmp1)
+	subq	#1,tmp2
+	jr	pl,.init
+	addq	#1,tmp1
+
+	moveq	#0,offset
+	moveq	#0,state
+	movei	#getlength,GETLENGTH
+	movei	#getbit,GETBIT
+.looppc	move	PC,LOOP
+	addq	#.loop-.looppc,LOOP
+	move	pc,LITERAL
+	jr	.start
+	addq	#6,LITERAL
+
+.literal
+	moveq	#1,byte
+	move	pc,LR
+	jr	.into
+	addq	#6,LR		; LR = .getbit
+.getbit
+	addc	byte,byte
+.into
+	btst	#8,byte
+	jump	eq,(GETBIT)
+	move	byte,index
+
+	storeb	byte,(DST)
+	addq	#1,DST
+.start
+	moveq	#0,prev_was_match
+
+.loop
+	moveq	#0,index
+	BL	(GETBIT)
+	jump	cc,(LITERAL)
+	addq	#14,LR
+	cmpq	#1,prev_was_match
+	jr	eq,.newoff
+	shlq	#8,r0
+	jump	(GETBIT)
+	move	r0,index
+	jr	cc,.oldoff
+	shlq	#8,r0
+.newoff
+	addq	#1,r0		; r0 = 257
+	BL	(GETLENGTH)
+	subq	#1,r0
+	jump	eq,(LR_save)
+	move	r0,offset
+
+.oldoff
+	movei	#257+64,r0
+	BL	(GETLENGTH)
+
+	move	DST,r1
+	sub	offset,r1
+.cpymatch1
+	loadb	(r1),r2
+	subq	#1,r0
+	addqt	#1,r1
+	storeb	r2,(DST)
+	jr	ne,.cpymatch1
+	addq	#1,DST
+
+	jump	(LOOP)
+	moveq	#1,prev_was_match
+
+getlength:
+	move	LR,LR_save2
+	moveq	#0,byte
+	move	r0,index
+	moveq	#0,bit_pos
+	move	pc,LR
+	jump	(GETBIT)
+	addq	#6,LR
+.gl
+	jr	cc,.exit
+	addq	#8,LR		; => return to "sh ..."
+	jump	(GETBIT)
+	nop
+	sh	bit_pos,r0
+	subq	#1,bit_pos	; sh < 0 => shift left!
+	or	r0,byte
+	jump	(GETBIT)
+	subq	#8,LR
+.exit
+	moveq	#1,r0
+	sh	bit_pos,r0
+	jump	(LR_save2)
+	or	byte,r0
+
+.newbyte:
+	loadb	(SRC),r2
+	shlq	#8,state
+	addq	#1,SRC
+	or	r2,state
+getbit
+	move	state,r2
+	move	PROBS,r1
+	add	index,r1		; r1 = &probs[index]
+	shrq	#12,r2
+	loadb	(r1),prob
+	jr	eq,.newbyte
+	move	state,r2
+	move	state,r0
+	shlq	#24,r2
+	shrq	#8,r0		; sh
+	shrq	#24,r2		; sl
+	cmp	prob,r2
+	addqt	#1,index
+	jr	cs,.one
+	mult	prob,r0
+
+	;; state -= ((state >> 8) + 1)*prob
+	;; prob -= (prob+8)>>4
+	move	prob,r2
+	add	prob,r0
+	addq	#8,r2
+	sub	r0,state
+	shrq	#4,r2
+	moveq	#0,r0
+	jr	.ret
+	sub	r2,prob
+
+.one
+	;; state = (state >> 8)*prob+(state & 0xff)
+	;; prob += (256 + 8 - prob) >> 4
+	move	r2,state
+	movei	#256+8,r2
+	add	r0,state
+	sub	prob,r2		; 256-prob+8
+	shrq	#4,r2
+	add	r2,prob
+
+	moveq	#3,r0
+.ret
+	storeb	prob,(r1)
+	jump	(LR)
+	shrq	#1,r0		; C = 0, r0 = 1

asm_unpackers/unpack_jagrisc_fast.js

@@ -0,0 +1,217 @@
+;;; -*-asm-*-
+;;; ukpr unpacker for Atari Jaguar RISC. (quick version)
+
+;;; lyxass syntax
+
+
+; input:
+;;; R20 : packed buffer
+;;; R21 : output buffer
+;;; r30 : return address
+;;;
+;;; Register usage (destroyed!)
+;;; r0-r17,r20,r21
+;;;
+
+DST		REG 21
+SRC		REG 20
+
+	REGTOP 17
+LR_save		REG 99
+LR_save2	REG 99
+GETBIT		REG 99
+GETLENGTH	REG 99
+LITERAL		REG 99
+LOOP		REG 99
+index		REG 99
+bit_pos		REG 99
+state		REG 99
+prev_was_match	REG 99
+offset		REG 99
+prob		reg 99
+byte		REG 99
+ndata		reg 99
+PROBS		reg 99
+tmp2		reg 2
+tmp1		REG 1
+tmp0		REG 0
+
+	REGMAP
+
+upkr_probs	equ $200
+
+SIZEOF_PROBS	EQU 1+255+1+2*32+2*32
+
+unupkr::
+	move	LR,LR_save
+	movei	#$80808080,tmp0
+	movei	#upkr_probs,PROBS
+	movei	#SIZEOF_PROBS,tmp2
+	move	PROBS,tmp1
+.init	store	tmp0,(tmp1)
+	subq	#4,tmp2
+	jr	pl,.init
+	addq	#4,tmp1
+
+	loadb	(SRC),ndata
+	addq	#1,SRC
+	moveq	#0,offset
+	moveq	#0,state
+	movei	#getlength,GETLENGTH
+	movei	#getbit,GETBIT
+.looppc	move	PC,LOOP
+	addq	#.loop-.looppc,LOOP
+	move	pc,LITERAL
+	jr	.start
+	addq	#6,LITERAL
+
+.literal
+	moveq	#1,byte
+	move	pc,LR
+	jr	.into
+	addq	#6,LR		; LR = .getbit
+.getbit
+	addc	byte,byte
+.into
+	btst	#8,byte
+	jump	eq,(GETBIT)
+	move	byte,index
+
+	storeb	byte,(DST)
+	addq	#1,DST
+.start
+	moveq	#0,prev_was_match
+
+.loop
+	moveq	#0,index
+	BL	(GETBIT)
+	jump	cc,(LITERAL)
+	addq	#14,LR
+	cmpq	#1,prev_was_match
+	jr	eq,.newoff
+	shlq	#8,r0
+	jump	(GETBIT)
+	move	r0,index
+	jr	cc,.oldoff
+	shlq	#8,r0
+.newoff
+	addq	#1,r0		; r0 = 257
+	BL	(GETLENGTH)
+	subq	#1,r0
+	move	r0,offset
+	jump	eq,(LR_save)
+	nop
+.oldoff
+	movei	#257+64,r0
+	BL	(GETLENGTH)
+
+	move	DST,r2
+	move	DST,r1
+	or	offset,r2
+	btst	#0,r2
+	moveq	#1,prev_was_match
+	jr	ne,.cpymatch1
+	sub	offset,r1
+.cpymatch2
+	loadw	(r1),r2
+	addqt	#2,r1
+	subq	#2,r0
+	storew	r2,(DST)
+	jump	eq,(LOOP)
+	addqt	#2,DST
+	jr	pl,.cpymatch2
+	nop
+	jump	(LOOP)
+	subq	#1,DST
+
+.cpymatch1
+	loadb	(r1),r2
+	subq	#1,r0
+	addqt	#1,r1
+	storeb	r2,(DST)
+	jr	ne,.cpymatch1
+	addq	#1,DST
+
+	jump	(LOOP)
+//->	nop
+
+getlength:
+	move	LR,LR_save2
+	moveq	#0,byte
+	move	r0,index
+	moveq	#0,bit_pos
+	move	pc,LR
+	jump	(GETBIT)
+	addq	#6,LR
+.gl
+	jr	cc,.exit
+	addq	#8,LR		; => return to "sh ..."
+	jump	(GETBIT)
+	nop
+	sh	bit_pos,r0
+	subq	#1,bit_pos	; sh < 0 => shift left!
+	or	r0,byte
+	jump	(GETBIT)
+	subq	#8,LR
+.exit
+	moveq	#1,r0
+	sh	bit_pos,r0
+	jump	(LR_save2)
+	or	byte,r0
+
+.newbyte:
+	move	ndata,r2
+	shlq	#8,state
+	loadb	(SRC),ndata
+	or	r2,state
+	addq	#1,SRC
+	move	state,r2
+	shrq	#12,r2
+	jr	ne,.done
+	move	state,r2
+	jr	.newbyte
+getbit
+	move	state,r2
+	move	PROBS,r1
+	add	index,r1		; r1 = &probs[index]
+	shrq	#12,r2
+	loadb	(r1),prob
+	jr	eq,.newbyte
+	move	state,r2
+.done
+	move	state,r0
+	shlq	#24,r2
+	shrq	#8,r0		; sh
+	shrq	#24,r2		; sl
+	cmp	prob,r2
+	addqt	#1,index
+	jr	cs,.one
+	mult	prob,r0
+
+	;; state -= ((state >> 8) + 1)*prob
+	;; prob -= (prob+8)>>4
+	move	prob,r2
+	add	prob,r0
+	addq	#8,r2
+	sub	r0,state
+	shrq	#4,r2
+	moveq	#0,r0
+	sub	r2,prob
+	shrq	#1,r0		; C = 0, r0 = 0
+	jump	(LR)
+	storeb	prob,(r1)
+
+.one
+	;; state = (state >> 8)*prob+(state & 0xff)
+	;; prob += (256 + 8 - prob) >> 4
+	move	r2,state
+	movei	#256+8,r2
+	add	r0,state
+	sub	prob,r2		; 256-prob+8
+	shrq	#4,r2
+	add	r2,prob
+
+	moveq	#3,r0
+	storeb	prob,(r1)
+	jump	(LR)
+	shrq	#1,r0		; C = 0, r0 = 1

c_unpacker/unpack.c

@@ -1,3 +1,56 @@
+/*
+    A simple C unpacker for upkr compressed data.
+
+    This implements two variants, selected by the UPKR_BITSTREAM define:
+    - normal: faster and smaller on modern hardware as whole bytes are shifted into
+              the rANS state at a time, but requires 20bits for the state
+    - bitstream: only single bits are shifted into the rANS state at a time
+                 which allows the state to always fit in 16bits which is a boon
+                 on very old CPUs.
+    The encoder and decoder need to be configured to use the same varianet.
+
+    upkr compressed data is a rANS byte-/bit-stream encoding a series of literal
+    byte values and back-references as probability encoded bits.
+
+    upkr_decode_bit reads one bit from the rANS stream, taking a probability context
+    as parameter. The probability context is a byte estimating the probability of
+    a bit encoded in this context being set. It is updated by upkr_decode_bit
+    after each decoded bit to reflect the observed past frequencies of on/off bits.
+
+    There are a number of different contexts used in the compressed format. The order in the
+    upkr_probs array is arbitrary, the only requirement for the unpacker is that all bits
+    that shared the same context while encoding also share the same context while decoding.
+    The contexts are:
+    - is match
+    - has offset
+    - literal bit N (0-7) with already decoded highest bits of literal == M (255 total)
+    - offset bit N (one less than max offset bits)
+    - has offset bit N (max offset bits)
+    - length bit N (one less then max length bits)
+    - has length bit N (max length bits)
+
+    Literal bytes are encoded from highest to lowest bit, with the bit position and
+    the already decoded bits as context.
+
+    Offst and Length are encoded in an interlaced variant of elias gamma coding. They
+    are encoded from lowest to highest bits. For each bit, first one bit is read in the
+    "has offset/length bit N)". If this is set, offset/length bit N is read in it's context
+    and the decoding continues with the next bit. If the "has bit N" is read as false, a
+    fixed 1 bit is added as the top bit at this position.
+
+    The highlevel decode loop then looks like this:
+    loop:
+        if read_bit(IS_MATCH):
+            if prev_was_match || read_bit(HAS_OFFSET):
+                offset = read_length_or_offset(OFFSET) - 1
+                if offset == 0:
+                    break
+            length = read_length_or_offset(LENGTH)
+            copy_bytes_from_offset(length, offset)
+        else:
+            read_and_push(literal)
+*/
+
 typedef unsigned char u8;
 typedef unsigned short u16;
 typedef unsigned long u32;
@@ -14,6 +67,7 @@ u32 upkr_state;
 
 int upkr_decode_bit(int context_index) {
 #ifdef UPKR_BITSTREAM
+    // shift in single bits until rANS state is >= 32768
     while(upkr_state < 32768) {
         if(upkr_bits_left == 0) {
             upkr_current_byte = *upkr_data_ptr++;
@@ -24,6 +78,7 @@ int upkr_decode_bit(int context_index) {
         --upkr_bits_left;
     }
 #else
+    // shift in a full byte until rANS state is >= 4096
     while(upkr_state < 4096) {
         upkr_state = (upkr_state << 8) | *upkr_data_ptr++;
     }
@@ -32,6 +87,8 @@ int upkr_decode_bit(int context_index) {
     int prob = upkr_probs[context_index];
     int bit = (upkr_state & 255) < prob ? 1 : 0;
     
+    // rANS state and context probability update
+    // for the later, add 1/16th (rounded) of difference from either 0 or 256
     if(bit) {
         upkr_state = prob * (upkr_state >> 8) + (upkr_state & 255);
         prob += (256 - prob + 8) >> 4;
@@ -60,6 +117,7 @@ void* upkr_unpack(void* destination, void* compressed_data) {
 #ifdef UPKR_BITSTREAM
     upkr_bits_left = 0;
 #endif
+    // all contexts are initialized to 128 = equal probability of 0 and 1
     for(int i = 0; i < sizeof(upkr_probs); ++i)
         upkr_probs[i] = 128;
     
@@ -68,10 +126,13 @@ void* upkr_unpack(void* destination, void* compressed_data) {
     int prev_was_match = 0;
     int offset = 0;
     for(;;) {
+        // is match
         if(upkr_decode_bit(0)) {
+            // has offset
             if(prev_was_match || upkr_decode_bit(256)) {
                 offset = upkr_decode_length(257) - 1;
                 if(offset == 0) {
+                    // a 0 offset signals the end of the compressed data
                     break;
                 }
             }
@@ -82,6 +143,9 @@ void* upkr_unpack(void* destination, void* compressed_data) {
             }
             prev_was_match = 1;
         } else {
+            // byte contains the previously read bits and indicates the number of
+            // read bits by the set top bit. Therefore it can be directly used as the
+            // context index. The set top bit ends up at bit position 8 and is not stored.
             int byte = 1;
             while(byte < 256) {
                 int bit = upkr_decode_bit(byte);