1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
|
// Copyright 2019+ Klaus Post. All rights reserved.
// License information can be found in the LICENSE file.
// Based on work by Yann Collet, released under BSD License.
package zstd
import (
"fmt"
"math"
"sync"
)
var (
// fsePredef are the predefined fse tables as defined here:
// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#default-distributions
// These values are already transformed.
fsePredef [3]fseDecoder
// fsePredefEnc are the predefined encoder based on fse tables as defined here:
// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#default-distributions
// These values are already transformed.
fsePredefEnc [3]fseEncoder
// symbolTableX contain the transformations needed for each type as defined in
// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#the-codes-for-literals-lengths-match-lengths-and-offsets
symbolTableX [3][]baseOffset
// maxTableSymbol is the biggest supported symbol for each table type
// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#the-codes-for-literals-lengths-match-lengths-and-offsets
maxTableSymbol = [3]uint8{tableLiteralLengths: maxLiteralLengthSymbol, tableOffsets: maxOffsetLengthSymbol, tableMatchLengths: maxMatchLengthSymbol}
// bitTables is the bits table for each table.
bitTables = [3][]byte{tableLiteralLengths: llBitsTable[:], tableOffsets: nil, tableMatchLengths: mlBitsTable[:]}
)
type tableIndex uint8
const (
// indexes for fsePredef and symbolTableX
tableLiteralLengths tableIndex = 0
tableOffsets tableIndex = 1
tableMatchLengths tableIndex = 2
maxLiteralLengthSymbol = 35
maxOffsetLengthSymbol = 30
maxMatchLengthSymbol = 52
)
// baseOffset is used for calculating transformations.
type baseOffset struct {
baseLine uint32
addBits uint8
}
// fillBase will precalculate base offsets with the given bit distributions.
func fillBase(dst []baseOffset, base uint32, bits ...uint8) {
if len(bits) != len(dst) {
panic(fmt.Sprintf("len(dst) (%d) != len(bits) (%d)", len(dst), len(bits)))
}
for i, bit := range bits {
if base > math.MaxInt32 {
panic(fmt.Sprintf("invalid decoding table, base overflows int32"))
}
dst[i] = baseOffset{
baseLine: base,
addBits: bit,
}
base += 1 << bit
}
}
var predef sync.Once
func initPredefined() {
predef.Do(func() {
// Literals length codes
tmp := make([]baseOffset, 36)
for i := range tmp[:16] {
tmp[i] = baseOffset{
baseLine: uint32(i),
addBits: 0,
}
}
fillBase(tmp[16:], 16, 1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16)
symbolTableX[tableLiteralLengths] = tmp
// Match length codes
tmp = make([]baseOffset, 53)
for i := range tmp[:32] {
tmp[i] = baseOffset{
// The transformation adds the 3 length.
baseLine: uint32(i) + 3,
addBits: 0,
}
}
fillBase(tmp[32:], 35, 1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16)
symbolTableX[tableMatchLengths] = tmp
// Offset codes
tmp = make([]baseOffset, maxOffsetBits+1)
tmp[1] = baseOffset{
baseLine: 1,
addBits: 1,
}
fillBase(tmp[2:], 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30)
symbolTableX[tableOffsets] = tmp
// Fill predefined tables and transform them.
// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#default-distributions
for i := range fsePredef[:] {
f := &fsePredef[i]
switch tableIndex(i) {
case tableLiteralLengths:
// https://github.com/facebook/zstd/blob/ededcfca57366461021c922720878c81a5854a0a/lib/decompress/zstd_decompress_block.c#L243
f.actualTableLog = 6
copy(f.norm[:], []int16{4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1,
2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1,
-1, -1, -1, -1})
f.symbolLen = 36
case tableOffsets:
// https://github.com/facebook/zstd/blob/ededcfca57366461021c922720878c81a5854a0a/lib/decompress/zstd_decompress_block.c#L281
f.actualTableLog = 5
copy(f.norm[:], []int16{
1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1})
f.symbolLen = 29
case tableMatchLengths:
//https://github.com/facebook/zstd/blob/ededcfca57366461021c922720878c81a5854a0a/lib/decompress/zstd_decompress_block.c#L304
f.actualTableLog = 6
copy(f.norm[:], []int16{
1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1,
-1, -1, -1, -1, -1})
f.symbolLen = 53
}
if err := f.buildDtable(); err != nil {
panic(fmt.Errorf("building table %v: %v", tableIndex(i), err))
}
if err := f.transform(symbolTableX[i]); err != nil {
panic(fmt.Errorf("building table %v: %v", tableIndex(i), err))
}
f.preDefined = true
// Create encoder as well
enc := &fsePredefEnc[i]
copy(enc.norm[:], f.norm[:])
enc.symbolLen = f.symbolLen
enc.actualTableLog = f.actualTableLog
if err := enc.buildCTable(); err != nil {
panic(fmt.Errorf("building encoding table %v: %v", tableIndex(i), err))
}
enc.setBits(bitTables[i])
enc.preDefined = true
}
})
}
|
