# changelog

* Jan 11, 2022 (v1.14.1)

* s2: Add stream index in [#462](https://github.com/klauspost/compress/pull/462)

* flate: Speed and efficiency improvements in [#439](https://github.com/klauspost/compress/pull/439) [#461](https://github.com/klauspost/compress/pull/461) [#455](https://github.com/klauspost/compress/pull/455) [#452](https://github.com/klauspost/compress/pull/452) [#458](https://github.com/klauspost/compress/pull/458)

* zstd: Detect short invalid signatures [#382](https://github.com/klauspost/compress/pull/382)

* zstd: Spawn decoder goroutine only if needed. [#380](https://github.com/klauspost/compress/pull/380)

* May 25, 2021 (v1.12.3)

* deflate: Better/faster Huffman encoding [#374](https://github.com/klauspost/compress/pull/374)

* deflate: Allocate less for history. [#375](https://github.com/klauspost/compress/pull/375)

* s2c/s2d/s2sx: Always truncate when writing files [#352](https://github.com/klauspost/compress/pull/352)

* zstd: Reduce memory usage further when using [WithLowerEncoderMem](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithLowerEncoderMem) [#346](https://github.com/klauspost/compress/pull/346)

* s2: Fix potential problem with amd64 assembly and profilers [#349](https://github.com/klauspost/compress/pull/349)

<details>

* Mar 26, 2021 (v1.11.13)

* zstd: Big speedup on small dictionary encodes [#344](https://github.com/klauspost/compress/pull/344) [#345](https://github.com/klauspost/compress/pull/345)

</details>

<details>

* July 8, 2020 (v1.10.11)

* zstd: Fix extra block when compressing with ReadFrom. [#278](https://github.com/klauspost/compress/pull/278)

# deflate usage

The packages are drop-in replacements for standard libraries. Simply replace the import path to use them:

| old import | new import | Documentation

If you expect to have a lot of concurrently allocated Writers consider using

the stateless compress described below.

# Stateless compression

This package offers stateless compression as a special option for gzip/deflate.

// matchlenLong will return the match length between offsets and t in src.

// It is assumed that s > t, that t >=0 and s < len(src).

func (e *fastGen) matchlenLong(s, t int32, src []byte) int32 {

if t >= s {

panic(fmt.Sprint("t >=s:", t, s))

}

"encoding/binary"

"fmt"

"io"

)

const (

codegenCodeCount = 19

badCode = 255

// bufferFlushSize indicates the buffer size

// after which bytes are flushed to the writer.

// Should preferably be a multiple of 6, since

bufferSize = bufferFlushSize + 8

)

// The number of extra bits needed by length code X - LENGTH_CODES_START.

/* 257 */ 0, 0, 0,

/* 260 */ 0, 0, 0, 0, 0, 1, 1, 1, 1, 2,

/* 270 */ 2, 2, 2, 3, 3, 3, 3, 4, 4, 4,

64, 80, 96, 112, 128, 160, 192, 224, 255,

}

// offset code word extra bits.

var offsetExtraBits = [32]int8{

0, 0, 0, 0, 1, 1, 2, 2, 3, 3,

for i := range offsetCombined[:] {

// Don't use extended window values...

continue

}

}

}

// Data waiting to be written is bytes[0:nbytes]

// and then the low nbits of bits.

bits uint64

nbytes uint8

lastHuffMan bool

literalEncoding *huffmanEncoder

_, w.err = w.writer.Write(b)

}

w.bits |= uint64(b) << (w.nbits & 63)

w.nbits += nb

if w.nbits >= 48 {

// Fixed Huffman baseline.

var literalEncoding = fixedLiteralEncoding

var offsetEncoding = fixedOffsetEncoding

// Dynamic Huffman?

var numCodegens int

size = reuseSize

}

return

}

}

// Check if we get a reasonable size decrease.

if storable && ssize <= size {

size, numCodegens = w.dynamicSize(w.literalEncoding, w.offsetEncoding, extraBits)

// Store predefined, if we don't get a reasonable improvement.

return

}

}

if storable && ssize <= size {

bits, nbits, nbytes := w.bits, w.nbits, w.nbytes

for _, t := range tokens {

//w.writeCode(lits[t.literal()])

bits |= uint64(c.code) << (nbits & 63)

nbits += c.len

if nbits >= 48 {

// Write the length

length := t.length()

if false {

} else {

// inlined

bits |= uint64(c.code) << (nbits & 63)

nbits += c.len

if nbits >= 48 {

}

}

//w.writeBits(extraLength, extraLengthBits)

bits |= uint64(extraLength) << (nbits & 63)

nbits += extraLengthBits

if nbits >= 48 {

}

// Write the offset

offset := t.offset()

if false {

} else {

// inlined

c := offs[offsetCode]

}

}

}

//w.writeBits(extraOffset, extraOffsetBits)

if nbits >= 48 {

binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits)

//*(*uint64)(unsafe.Pointer(&w.bytes[nbytes])) = bits

// https://stackoverflow.com/a/25454430

const guessHeaderSizeBits = 70 * 8

histogram(input, w.literalFreq[:numLiterals], fill)

w.literalFreq[endBlockMarker] = 1

w.tmpLitEncoding.generate(w.literalFreq[:numLiterals], 15)

if fill {

estBits += estBits >> w.logNewTablePenalty

// Store bytes, if we don't get a reasonable improvement.

if storable && ssize <= estBits {

w.writeStoredHeader(len(input), eof)

w.writeBytes(input)

return

if estBits < reuseSize {

if debugDeflate {

}

// We owe an EOB

w.writeCode(w.literalEncoding.codes[endBlockMarker])

// Go 1.16 LOVES having these on stack. At least 1.5x the speed.

bits, nbits, nbytes := w.bits, w.nbits, w.nbytes

// Unroll, write 3 codes/loop.

// Fastest number of unrolls.

for len(input) > 3 {

binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits)

bits >>= (n * 8) & 63

nbits -= n * 8

}

if nbytes >= bufferFlushSize {

if w.err != nil {

nbytes = 0

return

}

_, w.err = w.writer.Write(w.bytes[:nbytes])

nbytes = 0

}

// Remaining...

for _, t := range input {

if nbits >= 48 {

binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits)

//*(*uint64)(unsafe.Pointer(&w.bytes[nbytes])) = bits

nbytes = 0

return

}

_, w.err = w.writer.Write(w.bytes[:nbytes])

nbytes = 0

}

}

}

// Restore...

w.bits, w.nbits, w.nbytes = bits, nbits, nbytes

if debugDeflate {

}

if eof || sync {

w.writeCode(w.literalEncoding.codes[endBlockMarker])

w.lastHeader = 0

// hcode is a huffman code with a bit code and bit length.

type hcode struct {

}

type huffmanEncoder struct {

}

// set sets the code and length of an hcode.

h.len = length

h.code = code

}

var ch uint16

for ch = 0; ch < literalCount; ch++ {

var bits uint16

switch {

case ch < 144:

// size 8, 000110000 .. 10111111

bits = ch + 192 - 280

size = 8

}

}

return h

}

// of the level j ancestor.

var leafCounts [maxBitsLimit][maxBitsLimit]int32

for level := int32(1); level <= maxBits; level++ {

// For every level, the first two items are the first two characters.

// We initialize the levels as if we had already figured this out.

levels[level] = levelInfo{

level: level,

}

leafCounts[level][level] = 2

if level == 1 {

// We need a total of 2*n - 2 items at top level and have already generated 2.

levels[maxBits].needed = 2*n - 4

l := &levels[level]

if l.nextPairFreq == math.MaxInt32 && l.nextCharFreq == math.MaxInt32 {

// We've run out of both leafs and pairs.

// more values in the level below

l.lastFreq = l.nextPairFreq

// Take leaf counts from the lower level, except counts[level] remains the same.

levels[l.level-1].needed = 2

}

sortByLiteral(chunk)

for _, node := range chunk {

code++

}

list = list[0 : len(list)-int(bits)]

// maxBits The maximum number of bits to use for any literal.

func (h *huffmanEncoder) generate(freq []uint16, maxBits int32) {

list := h.freqcache[:len(freq)+1]

// Number of non-zero literals

count := 0

// Set list to be the set of all non-zero literals and their frequencies

list[count] = literalNode{uint16(i), f}

count++

} else {

}

}

list = list[:count]

if count <= 2 {

var decCodeToLen = [32]lengthExtra{{length: 0x0, extra: 0x0}, {length: 0x1, extra: 0x0}, {length: 0x2, extra: 0x0}, {length: 0x3, extra: 0x0}, {length: 0x4, extra: 0x0}, {length: 0x5, extra: 0x0}, {length: 0x6, extra: 0x0}, {length: 0x7, extra: 0x0}, {length: 0x8, extra: 0x1}, {length: 0xa, extra: 0x1}, {length: 0xc, extra: 0x1}, {length: 0xe, extra: 0x1}, {length: 0x10, extra: 0x2}, {length: 0x14, extra: 0x2}, {length: 0x18, extra: 0x2}, {length: 0x1c, extra: 0x2}, {length: 0x20, extra: 0x3}, {length: 0x28, extra: 0x3}, {length: 0x30, extra: 0x3}, {length: 0x38, extra: 0x3}, {length: 0x40, extra: 0x4}, {length: 0x50, extra: 0x4}, {length: 0x60, extra: 0x4}, {length: 0x70, extra: 0x4}, {length: 0x80, extra: 0x5}, {length: 0xa0, extra: 0x5}, {length: 0xc0, extra: 0x5}, {length: 0xe0, extra: 0x5}, {length: 0xff, extra: 0x0}, {length: 0x0, extra: 0x0}, {length: 0x0, extra: 0x0}, {length: 0x0, extra: 0x0}}

// Initialize the fixedHuffmanDecoder only once upon first use.

var fixedOnce sync.Once

var fixedHuffmanDecoder huffmanDecoder

return nil

}

// Copy a single uncompressed data block from input to output.

func (f *decompressor) dataBlock() {

// Uncompressed.

)

fr := f.r.(*bytes.Buffer)

switch f.stepState {

case stateInit:

goto readLiteral

// cases, the chunks slice will be 0 for the invalid sequence, leading it

// satisfy the n == 0 check below.

n := uint(f.hl.maxRead)

for {

c, err := fr.ReadByte()

if err != nil {

f.err = noEOF(err)

return

}

f.roffset++

}

n = uint(chunk & huffmanCountMask)

if n > huffmanChunkBits {

n = uint(chunk & huffmanCountMask)

}

if n == 0 {

if debugDecode {

fmt.Println("huffsym: n==0")

}

f.err = CorruptInputError(f.roffset)

return

}

v = int(chunk >> huffmanValueShift)

break

}

f.toRead = f.dict.readFlush()

f.step = (*decompressor).huffmanBytesBuffer

f.stepState = stateInit

return

}

goto readLiteral

case v == 256:

f.finishBlock()

return

// otherwise, reference to older data

val := decCodeToLen[(v - 257)]

length = int(val.length) + 3

n := uint(val.extra)

c, err := fr.ReadByte()

if err != nil {

if debugDecode {

fmt.Println("morebits n>0:", err)

}

return

}

f.roffset++

}

default:

if debugDecode {

fmt.Println(v, ">= maxNumLit")

}

f.err = CorruptInputError(f.roffset)

return

}

var dist uint32

if f.hd == nil {

c, err := fr.ReadByte()

if err != nil {

if debugDecode {

fmt.Println("morebits f.nb<5:", err)

}

return

}

f.roffset++

}

} else {

// Since a huffmanDecoder can be empty or be composed of a degenerate tree

// with single element, huffSym must error on these two edge cases. In both

// Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers,

// but is smart enough to keep local variables in registers, so use nb and b,

// inline call to moreBits and reassign b,nb back to f on return.

for {

c, err := fr.ReadByte()

if err != nil {

f.err = noEOF(err)

return

}

f.roffset++

}

n = uint(chunk & huffmanCountMask)

if n > huffmanChunkBits {

n = uint(chunk & huffmanCountMask)

}

if n == 0 {

if debugDecode {

fmt.Println("huffsym: n==0")

}

f.err = CorruptInputError(f.roffset)

return

}

dist = uint32(chunk >> huffmanValueShift)

break

}

nb := uint(dist-2) >> 1

// have 1 bit in bottom of dist, need nb more.

extra := (dist & 1) << (nb & regSizeMaskUint32)

c, err := fr.ReadByte()

if err != nil {

if debugDecode {

fmt.Println("morebits f.nb<nb:", err)

}

return

}

f.roffset++

}

dist = 1<<((nb+1)&regSizeMaskUint32) + 1 + extra

default:

if debugDecode {

fmt.Println("dist too big:", dist, maxNumDist)

}

// No check on length; encoding can be prescient.

if dist > uint32(f.dict.histSize()) {

if debugDecode {

fmt.Println("dist > f.dict.histSize():", dist, f.dict.histSize())

}

f.toRead = f.dict.readFlush()

f.step = (*decompressor).huffmanBytesBuffer // We need to continue this work

f.stepState = stateDict

return

}

goto readLiteral

}

}

// Decode a single Huffman block from f.

)

fr := f.r.(*bytes.Reader)

switch f.stepState {

case stateInit:

goto readLiteral

// cases, the chunks slice will be 0 for the invalid sequence, leading it

// satisfy the n == 0 check below.

n := uint(f.hl.maxRead)

for {

c, err := fr.ReadByte()

if err != nil {

f.err = noEOF(err)

return

}

f.roffset++

}

n = uint(chunk & huffmanCountMask)

if n > huffmanChunkBits {

n = uint(chunk & huffmanCountMask)

}

if n == 0 {

if debugDecode {

fmt.Println("huffsym: n==0")

}

f.err = CorruptInputError(f.roffset)

return

}

v = int(chunk >> huffmanValueShift)

break

}

f.toRead = f.dict.readFlush()

f.step = (*decompressor).huffmanBytesReader

f.stepState = stateInit

return

}

goto readLiteral

case v == 256:

f.finishBlock()

return

// otherwise, reference to older data

val := decCodeToLen[(v - 257)]

length = int(val.length) + 3

n := uint(val.extra)

c, err := fr.ReadByte()

if err != nil {

if debugDecode {

fmt.Println("morebits n>0:", err)

}

return

}

f.roffset++

}

default:

if debugDecode {

fmt.Println(v, ">= maxNumLit")

}

f.err = CorruptInputError(f.roffset)

return

}

var dist uint32

if f.hd == nil {

c, err := fr.ReadByte()

if err != nil {

if debugDecode {

fmt.Println("morebits f.nb<5:", err)

}

return

}

f.roffset++

}

} else {

// Since a huffmanDecoder can be empty or be composed of a degenerate tree

// with single element, huffSym must error on these two edge cases. In both

// Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers,

// but is smart enough to keep local variables in registers, so use nb and b,

// inline call to moreBits and reassign b,nb back to f on return.

for {

c, err := fr.ReadByte()

if err != nil {

f.err = noEOF(err)

return

}

f.roffset++

}

n = uint(chunk & huffmanCountMask)

if n > huffmanChunkBits {

n = uint(chunk & huffmanCountMask)

}

if n == 0 {

if debugDecode {

fmt.Println("huffsym: n==0")

}

f.err = CorruptInputError(f.roffset)

return

}

dist = uint32(chunk >> huffmanValueShift)

break

}

nb := uint(dist-2) >> 1

// have 1 bit in bottom of dist, need nb more.

extra := (dist & 1) << (nb & regSizeMaskUint32)

c, err := fr.ReadByte()

if err != nil {

if debugDecode {

fmt.Println("morebits f.nb<nb:", err)

}

return

}

f.roffset++

}

dist = 1<<((nb+1)&regSizeMaskUint32) + 1 + extra

default:

if debugDecode {

fmt.Println("dist too big:", dist, maxNumDist)

}

// No check on length; encoding can be prescient.

if dist > uint32(f.dict.histSize()) {

if debugDecode {

fmt.Println("dist > f.dict.histSize():", dist, f.dict.histSize())

}

f.toRead = f.dict.readFlush()

f.step = (*decompressor).huffmanBytesReader // We need to continue this work

f.stepState = stateDict

return

}

goto readLiteral

}

}

// Decode a single Huffman block from f.

)

fr := f.r.(*bufio.Reader)

switch f.stepState {

case stateInit:

goto readLiteral

// cases, the chunks slice will be 0 for the invalid sequence, leading it

// satisfy the n == 0 check below.

n := uint(f.hl.maxRead)

for {

c, err := fr.ReadByte()

if err != nil {

f.err = noEOF(err)

return

}

f.roffset++

}

n = uint(chunk & huffmanCountMask)

if n > huffmanChunkBits {

n = uint(chunk & huffmanCountMask)

}

if n == 0 {

if debugDecode {

fmt.Println("huffsym: n==0")

}

f.err = CorruptInputError(f.roffset)

return

}

v = int(chunk >> huffmanValueShift)

break

}

f.toRead = f.dict.readFlush()

f.step = (*decompressor).huffmanBufioReader

f.stepState = stateInit

return

}

goto readLiteral

case v == 256:

f.finishBlock()

return

// otherwise, reference to older data

val := decCodeToLen[(v - 257)]

length = int(val.length) + 3

n := uint(val.extra)

c, err := fr.ReadByte()

if err != nil {

if debugDecode {

fmt.Println("morebits n>0:", err)

}

return

}

f.roffset++

}

default:

if debugDecode {

fmt.Println(v, ">= maxNumLit")

}

f.err = CorruptInputError(f.roffset)

return

}

var dist uint32

if f.hd == nil {

c, err := fr.ReadByte()

if err != nil {

if debugDecode {

fmt.Println("morebits f.nb<5:", err)

}

return

}

f.roffset++

}

} else {

// Since a huffmanDecoder can be empty or be composed of a degenerate tree

// with single element, huffSym must error on these two edge cases. In both

// Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers,

// but is smart enough to keep local variables in registers, so use nb and b,

// inline call to moreBits and reassign b,nb back to f on return.

for {

c, err := fr.ReadByte()

if err != nil {

f.err = noEOF(err)

return

}

f.roffset++

}

n = uint(chunk & huffmanCountMask)

if n > huffmanChunkBits {

n = uint(chunk & huffmanCountMask)

}

if n == 0 {

if debugDecode {

fmt.Println("huffsym: n==0")

}

f.err = CorruptInputError(f.roffset)

return

}

dist = uint32(chunk >> huffmanValueShift)

break

}

nb := uint(dist-2) >> 1

// have 1 bit in bottom of dist, need nb more.

extra := (dist & 1) << (nb & regSizeMaskUint32)

c, err := fr.ReadByte()

if err != nil {

if debugDecode {

fmt.Println("morebits f.nb<nb:", err)

}

return

}

f.roffset++

}

dist = 1<<((nb+1)&regSizeMaskUint32) + 1 + extra

default:

if debugDecode {

fmt.Println("dist too big:", dist, maxNumDist)

}

// No check on length; encoding can be prescient.

if dist > uint32(f.dict.histSize()) {

if debugDecode {

fmt.Println("dist > f.dict.histSize():", dist, f.dict.histSize())

}

f.toRead = f.dict.readFlush()

f.step = (*decompressor).huffmanBufioReader // We need to continue this work

f.stepState = stateDict

return

}

goto readLiteral

}

}

// Decode a single Huffman block from f.

)

fr := f.r.(*strings.Reader)

switch f.stepState {

case stateInit:

goto readLiteral

// cases, the chunks slice will be 0 for the invalid sequence, leading it

// satisfy the n == 0 check below.

n := uint(f.hl.maxRead)

// Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers,

// but is smart enough to keep local variables in registers, so use nb and b,

// inline call to moreBits and reassign b,nb back to f on return.

for {

c, err := fr.ReadByte()

if err != nil {

f.err = noEOF(err)

return

}

f.roffset++

}

n = uint(chunk & huffmanCountMask)

if n > huffmanChunkBits {

n = uint(chunk & huffmanCountMask)

}

if n == 0 {

if debugDecode {

fmt.Println("huffsym: n==0")

}

f.err = CorruptInputError(f.roffset)

return

}

v = int(chunk >> huffmanValueShift)

break

}

f.dict.writeByte(byte(v))

if f.dict.availWrite() == 0 {

f.toRead = f.dict.readFlush()

f.stepState = stateInit

return

}

goto readLiteral

case v == 256:

f.finishBlock()

return

// otherwise, reference to older data

val := decCodeToLen[(v - 257)]

length = int(val.length) + 3

n := uint(val.extra)

c, err := fr.ReadByte()

if err != nil {

if debugDecode {

fmt.Println("morebits n>0:", err)

}

return

}

f.roffset++

}

default:

if debugDecode {

fmt.Println(v, ">= maxNumLit")

}

f.err = CorruptInputError(f.roffset)

return

}

var dist uint32

if f.hd == nil {

c, err := fr.ReadByte()

if err != nil {

if debugDecode {

fmt.Println("morebits f.nb<5:", err)

}

return

}

f.roffset++

}

} else {

// Since a huffmanDecoder can be empty or be composed of a degenerate tree

// with single element, huffSym must error on these two edge cases. In both

// Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers,

// but is smart enough to keep local variables in registers, so use nb and b,

// inline call to moreBits and reassign b,nb back to f on return.

for {

c, err := fr.ReadByte()

if err != nil {

f.err = noEOF(err)

return

}

f.roffset++

}

n = uint(chunk & huffmanCountMask)

if n > huffmanChunkBits {

n = uint(chunk & huffmanCountMask)

}

if n == 0 {

if debugDecode {

fmt.Println("huffsym: n==0")

}

f.err = CorruptInputError(f.roffset)

return

}

dist = uint32(chunk >> huffmanValueShift)

break

}

nb := uint(dist-2) >> 1

// have 1 bit in bottom of dist, need nb more.

extra := (dist & 1) << (nb & regSizeMaskUint32)

c, err := fr.ReadByte()

if err != nil {

if debugDecode {

fmt.Println("morebits f.nb<nb:", err)

}

return

}

f.roffset++

}

dist = 1<<((nb+1)&regSizeMaskUint32) + 1 + extra

default:

if debugDecode {

fmt.Println("dist too big:", dist, maxNumDist)

}

// No check on length; encoding can be prescient.

if dist > uint32(f.dict.histSize()) {

if debugDecode {

fmt.Println("dist > f.dict.histSize():", dist, f.dict.histSize())

}

if f.dict.availWrite() == 0 || f.copyLen > 0 {

f.toRead = f.dict.readFlush()

f.stepState = stateDict

return

}

goto readLiteral

}

}

func (f *decompressor) huffmanBlockDecoder() func() {

return f.huffmanBufioReader

case *strings.Reader:

return f.huffmanStringsReader

default:

}

}

package flate

// fastGen maintains the table for matches,

// and the previous byte block for level 2.

// Extend the 4-byte match as long as possible.

t := candidate.offset - e.cur

// Extend backwards

for t > 0 && s > nextEmit && src[t-1] == src[s-1] {

l++

}

if nextEmit < s {

}

// Save the match found

s += l

nextEmit = s

if nextS >= s {

l++

}

if nextEmit < s {

}

dst.AddMatchLong(l, uint32(s-t-baseMatchOffset))

// fastEncL3

type fastEncL3 struct {

fastGen

}

// Encode uses a similar algorithm to level 2, will check up to two candidates.

const (

inputMargin = 8 - 1

minNonLiteralBlockSize = 1 + 1 + inputMargin

)

if debugDeflate && e.cur < 0 {

nextS := s

var candidate tableEntry

for {

s = nextS

nextS = s + 1 + (s-nextEmit)>>skipLog

if nextS > sLimit {

l++

}

if nextEmit < s {

}

dst.AddMatchLong(l, uint32(s-t-baseMatchOffset))

// Index first pair after match end.

if int(t+4) < len(src) && t > 0 {

cv := load3232(src, t)

e.table[nextHash] = tableEntryPrev{

Prev: e.table[nextHash].Cur,

Cur: tableEntry{offset: e.cur + t},

goto emitRemainder

}

}

e.table[prevHash] = tableEntryPrev{

Prev: e.table[prevHash].Cur,

Cur: tableEntry{offset: e.cur + s - 2},

}

x >>= 8

e.table[prevHash] = tableEntryPrev{

Prev: e.table[prevHash].Cur,

Cur: tableEntry{offset: e.cur + s - 1},

}

x >>= 8

candidates := e.table[currHash]

cv = uint32(x)

e.table[currHash] = tableEntryPrev{

candidate = candidates.Cur

minOffset := e.cur + s - (maxMatchOffset - 4)

candidate = candidates.Prev

if candidate.offset > minOffset && cv == load3232(src, candidate.offset-e.cur) {

}

}

cv = uint32(x >> 8)

l++

}

if nextEmit < s {

}

if debugDeflate {

if t >= s {

l++

}

if nextEmit < s {

}

if debugDeflate {

if t >= s {

l++

}

if nextEmit < s {

}

if false {

if t >= s {

l++

}

if nextEmit < s {

}

// Save the match found

)

const (

lengthShift = 22

offsetMask = 1<<lengthShift - 1

typeMask = 3 << 30

// emitLiteral writes a literal chunk and returns the number of bytes written.

func emitLiteral(dst *tokens, lit []byte) {

dst.litHist[v]++

}

}

func (t *tokens) AddLiteral(lit byte) {

xoffset |= oCode << 16

t.extraHist[lengthCodes1[uint8(xlength)]]++

t.tokens[t.n] = token(matchType | xlength<<lengthShift | xoffset)

t.n++

}

xl := xlength

if xl > 258 {

// We need to have at least baseMatchLength left over for next loop.

}

xlength -= xl

xl -= baseMatchLength

t.extraHist[lengthCodes1[uint8(xl)]]++

t.tokens[t.n] = token(matchType | uint32(xl)<<lengthShift | xoffset)

t.n++

}

func (t token) length() uint8 { return uint8(t >> lengthShift) }

// Returns the offset code corresponding to a specific offset

func offsetCode(off uint32) uint32 {

import (

"encoding/binary"

"errors"

"io"

)

// bitReader reads a bitstream in reverse.

// The last set bit indicates the start of the stream and is used

// for aligning the input.

type bitReaderBytes struct {

in []byte

off uint // next byte to read is at in[off - 1]

return b.off == 0 && b.bitsRead >= 64

}

// close the bitstream and returns an error if out-of-buffer reads occurred.

func (b *bitReaderBytes) close() error {

// Release reference.

b.in = nil

if b.bitsRead > 64 {

return io.ErrUnexpectedEOF

}

return uint16(b.value >> ((64 - n) & 63))

}

func (b *bitReaderShifted) advance(n uint8) {

b.bitsRead += n

b.value <<= n & 63

return b.off == 0 && b.bitsRead >= 64

}

// close the bitstream and returns an error if out-of-buffer reads occurred.

func (b *bitReaderShifted) close() error {

// Release reference.

b.in = nil

if b.bitsRead > 64 {

return io.ErrUnexpectedEOF

}

import (

"fmt"

"runtime"

"sync"

)

if err != nil {

return nil, err

}

// Write compressed length as little endian before block.

if i < 3 {

// Last length is not written.

return nil, errs[i]

}

o := s.tmpOut[i]

// Write compressed length as little endian before block.

if i < 3 {

// Last length is not written.

"errors"

"fmt"

"io"

"github.com/klauspost/compress/fse"

)

return &Decoder{

dt: s.dt,

actualTableLog: s.actualTableLog,

}

}

type Decoder struct {

dt dTable

actualTableLog uint8

}

// Decompress1X will decompress a 1X encoded stream.

dt := d.dt.single[:tlSize]

// Use temp table to avoid bound checks/append penalty.

var off uint8

for br.off >= 8 {

if off == 0 {

if len(dst)+256 > maxDecodedSize {

br.close()

return nil, ErrMaxDecodedSizeExceeded

}

dst = append(dst, buf[:]...)

}

if len(dst)+int(off) > maxDecodedSize {

br.close()

return nil, ErrMaxDecodedSizeExceeded

}

}

}

if len(dst) >= maxDecodedSize {

br.close()

return nil, ErrMaxDecodedSizeExceeded

}

bitsLeft -= nBits

dst = append(dst, uint8(v.entry>>8))

}

return dst, br.close()

}

dt := d.dt.single[:256]

// Use temp table to avoid bound checks/append penalty.

var off uint8

switch d.actualTableLog {

if off == 0 {

if len(dst)+256 > maxDecodedSize {

br.close()

return nil, ErrMaxDecodedSizeExceeded

}

dst = append(dst, buf[:]...)

if off == 0 {

if len(dst)+256 > maxDecodedSize {

br.close()

return nil, ErrMaxDecodedSizeExceeded

}

dst = append(dst, buf[:]...)

off += 4

if off == 0 {

if len(dst)+256 > maxDecodedSize {

br.close()

return nil, ErrMaxDecodedSizeExceeded

}

off += 4

if off == 0 {

if len(dst)+256 > maxDecodedSize {

br.close()

return nil, ErrMaxDecodedSizeExceeded

}

off += 4

if off == 0 {

if len(dst)+256 > maxDecodedSize {

br.close()

return nil, ErrMaxDecodedSizeExceeded

}

off += 4

if off == 0 {

if len(dst)+256 > maxDecodedSize {

br.close()

return nil, ErrMaxDecodedSizeExceeded

}

off += 4

if off == 0 {

if len(dst)+256 > maxDecodedSize {

br.close()

return nil, ErrMaxDecodedSizeExceeded

}

off += 4

if off == 0 {

if len(dst)+256 > maxDecodedSize {

br.close()

return nil, ErrMaxDecodedSizeExceeded

}

}

}

default:

return nil, fmt.Errorf("invalid tablelog: %d", d.actualTableLog)

}

if len(dst)+int(off) > maxDecodedSize {

br.close()

return nil, ErrMaxDecodedSizeExceeded

}

}

if len(dst) >= maxDecodedSize {

br.close()

return nil, ErrMaxDecodedSizeExceeded

}

v := dt[br.peekByteFast()>>shift]

bitsLeft -= int8(nBits)

dst = append(dst, uint8(v.entry>>8))

}

return dst, br.close()

}

dt := d.dt.single[:256]

// Use temp table to avoid bound checks/append penalty.

var off uint8

const shift = 56

off += 4

if off == 0 {

if len(dst)+256 > maxDecodedSize {

br.close()

return nil, ErrMaxDecodedSizeExceeded

}

}

if len(dst)+int(off) > maxDecodedSize {

br.close()

return nil, ErrMaxDecodedSizeExceeded

}

}

}

if len(dst) >= maxDecodedSize {

br.close()

return nil, ErrMaxDecodedSizeExceeded

}

bitsLeft -= int8(nBits)

dst = append(dst, uint8(v.entry>>8))

}

return dst, br.close()

}

// The length of the supplied input must match the end of a block exactly.

// The *capacity* of the dst slice must match the destination size of

// the uncompressed data exactly.

func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) {

if d.actualTableLog == 8 {

return d.decompress4X8bitExactly(dst, src)

single := d.dt.single[:tlSize]

// Use temp table to avoid bound checks/append penalty.

var off uint8

var decoded int

// Decode 4 values from each decoder/loop.

for {

if br[0].off < 4 || br[1].off < 4 || br[2].off < 4 || br[3].off < 4 {

break

br1.value <<= v & 63

br2.bitsRead += uint8(v2)

br2.value <<= v2 & 63

v = single[uint8(br1.value>>shift)].entry

v2 = single[uint8(br2.value>>shift)].entry

br1.value <<= v & 63

br2.bitsRead += uint8(v2)

br2.value <<= v2 & 63

v = single[uint8(br1.value>>shift)].entry

v2 = single[uint8(br2.value>>shift)].entry

br1.value <<= v & 63

br2.bitsRead += uint8(v2)

br2.value <<= v2 & 63

v = single[uint8(br1.value>>shift)].entry

v2 = single[uint8(br2.value>>shift)].entry

br1.value <<= v & 63

br2.bitsRead += uint8(v2)

br2.value <<= v2 & 63

}

{

br1.value <<= v & 63

br2.bitsRead += uint8(v2)

br2.value <<= v2 & 63

v = single[uint8(br1.value>>shift)].entry

v2 = single[uint8(br2.value>>shift)].entry

br1.value <<= v & 63

br2.bitsRead += uint8(v2)

br2.value <<= v2 & 63

v = single[uint8(br1.value>>shift)].entry

v2 = single[uint8(br2.value>>shift)].entry

br1.value <<= v & 63

br2.bitsRead += uint8(v2)

br2.value <<= v2 & 63

v = single[uint8(br1.value>>shift)].entry

v2 = single[uint8(br2.value>>shift)].entry

br1.value <<= v & 63

br2.bitsRead += uint8(v2)

br2.value <<= v2 & 63

}

off += 4

if bufoff > dstEvery {

return nil, errors.New("corruption detected: stream overrun 1")

}

out = out[bufoff:]

// There must at least be 3 buffers left.

if len(out) < dstEvery*3 {

return nil, errors.New("corruption detected: stream overrun 2")

}

}

if off > 0 {

ioff := int(off)

if len(out) < dstEvery*3+ioff {

return nil, errors.New("corruption detected: stream overrun 3")

}

decoded += int(off) * 4

out = out[off:]

}

// Decode remaining.

for i := range br {

offset := dstEvery * i

br := &br[i]

for bitsLeft > 0 {

if br.finished() {

return nil, io.ErrUnexpectedEOF

}

if br.bitsRead >= 56 {

}

}

// end inline...

return nil, errors.New("corruption detected: stream overrun 4")

}

v := single[uint8(br.value>>shift)].entry

nBits := uint8(v)

br.advance(nBits)

out[offset] = uint8(v >> 8)

offset++

}

decoded += offset - dstEvery*i

err = br.close()

if err != nil {

return nil, err

}

}

if dstSize != decoded {

return nil, errors.New("corruption detected: short output block")

}

single := d.dt.single[:tlSize]

// Use temp table to avoid bound checks/append penalty.

var off uint8

var decoded int

// Decode 4 values from each decoder/loop.

for {

if br[0].off < 4 || br[1].off < 4 || br[2].off < 4 || br[3].off < 4 {

break

// Interleave 2 decodes.

const stream = 0

const stream2 = 1

}

{

const stream = 2

const stream2 = 3

}

off += 4

if bufoff > dstEvery {

return nil, errors.New("corruption detected: stream overrun 1")

}

out = out[bufoff:]

// There must at least be 3 buffers left.

if len(out) < dstEvery*3 {

return nil, errors.New("corruption detected: stream overrun 2")

}

}

if len(out) < dstEvery*3+ioff {

return nil, errors.New("corruption detected: stream overrun 3")

}

decoded += int(off) * 4

out = out[off:]

}

// Decode remaining.

for i := range br {

offset := dstEvery * i

br := &br[i]

for bitsLeft > 0 {

if br.finished() {

return nil, io.ErrUnexpectedEOF

}

if br.bitsRead >= 56 {

}

}

// end inline...

return nil, errors.New("corruption detected: stream overrun 4")

}

v := single[br.peekByteFast()].entry

nBits := uint8(v)

br.advance(nBits)

out[offset] = uint8(v >> 8)

offset++

}

decoded += offset - dstEvery*i

err = br.close()

if err != nil {

return nil, err

}

}

if dstSize != decoded {

return nil, errors.New("corruption detected: short output block")

}

"fmt"

"math"

"math/bits"

"github.com/klauspost/compress/fse"

)

nodes []nodeElt

tmpOut [4][]byte

fse *fse.Scratch

huffWeight [maxSymbolValue + 1]byte

}

in the future. So if you want to limit concurrency for future updates, specify the concurrency

you would like.

You can specify your desired compression level using `WithEncoderLevel()` option. Currently only pre-defined

compression settings can be specified.

For compressing small blocks, the returned encoder has a function called `EncodeAll(src, dst []byte) []byte`.

`EncodeAll` will encode all input in src and append it to dst.

Encoded blocks can be concatenated and the result will be the combined input stream.

Data compressed with EncodeAll can be decoded with the Decoder, using either a stream or `DecodeAll`.

This package:

file out level insize outsize millis mb/s

cgo zstd:

silesia.tar zstd 1 211947520 73605392 543 371.56

silesia.tar zstd 9 211947520 60212393 5063 39.92

gzip, stdlib/this package:

GOB stream of binary data. Highly compressible.

https://files.klauspost.com/compress/gob-stream.7z

file out level insize outsize millis mb/s

gob-stream zstd 1 1911399616 249810424 2637 691.26

gob-stream zstd 3 1911399616 208192146 3490 522.31

gob-stream zstd 6 1911399616 193632038 6687 272.56

gob-stream zstd 9 1911399616 177620386 16175 112.70

The test data for the Large Text Compression Benchmark is the first

10^9 bytes of the English Wikipedia dump on Mar. 3, 2006.

http://mattmahoney.net/dc/textdata.html

file out level insize outsize millis mb/s

enwik9 zstd 1 1000000000 358072021 3110 306.65

enwik9 zstd 3 1000000000 313734672 4784 199.35

enwik9 zstd 6 1000000000 295138875 10290 92.68

enwik9 zstd 9 1000000000 278348700 28549 33.40

Highly compressible JSON file.

https://files.klauspost.com/compress/github-june-2days-2019.json.zst

file out level insize outsize millis mb/s

github-june-2days-2019.json zstd 1 6273951764 766284037 8450 708.00

github-june-2days-2019.json zstd 3 6273951764 661889476 10927 547.57

github-june-2days-2019.json zstd 6 6273951764 642756859 22996 260.18

github-june-2days-2019.json zstd 9 6273951764 601974523 52413 114.16

VM Image, Linux mint with a few installed applications:

https://files.klauspost.com/compress/rawstudio-mint14.7z

file out level insize outsize millis mb/s

rawstudio-mint14.tar zstd 1 8558382592 3609250104 17136 476.27

rawstudio-mint14.tar zstd 3 8558382592 3341679997 29262 278.92

rawstudio-mint14.tar zstd 6 8558382592 3235846406 77904 104.77

rawstudio-mint14.tar zstd 9 8558382592 3160778861 140946 57.91

CSV data:

https://files.klauspost.com/compress/nyc-taxi-data-10M.csv.zst

file out level insize outsize millis mb/s

nyc-taxi-data-10M.csv zstd 1 3325605752 687399637 8233 385.18

nyc-taxi-data-10M.csv zstd 3 3325605752 598514411 10065 315.07

nyc-taxi-data-10M.csv zstd 6 3325605752 570522953 20038 158.27

nyc-taxi-data-10M.csv zstd 9 3325605752 517554797 64565 49.12

```

## Decompressor

}

```

For decoding buffers, it could look something like this:

// Create a reader that caches decompressors.

// For this operation type we supply a nil Reader.

// Decompress a buffer. We don't supply a destination buffer,

// so it will be allocated by the decoder.

```

Both of these cases should provide the functionality needed.

It will only allow a certain number of concurrent operations to run.

### Dictionaries

The buffer decoder does everything on the same goroutine and does nothing concurrently.

It can however decode several buffers concurrently. Use `WithDecoderConcurrency(n)` to limit that.

So effectively this also means the decoder will "read ahead" and prepare data to always be available for output.

Since "blocks" are quite dependent on the output of the previous block stream decoding will only have limited concurrency.

### Benchmarks

These are some examples of performance compared to [datadog cgo library](https://github.com/DataDog/zstd).

import (

"encoding/binary"

"errors"

"io"

"math/bits"

)

func (b *bitReader) close() error {

// Release reference.

b.in = nil

if b.bitsRead > 64 {

return io.ErrUnexpectedEOF

}

// Window size of the block.

WindowSize uint64

// Frame to use for singlethreaded decoding.

// Should not be used by the decoder itself since parent may be another frame.

localFrame *frameDec

// Block is RLE, this is the size.

RLESize uint32

tmp [4]byte

func newBlockDec(lowMem bool) *blockDec {

b := blockDec{

}

return &b

}

case blockTypeReserved:

return ErrReservedBlockType

case blockTypeRLE:

b.RLESize = uint32(cSize)

if b.lowMem {

maxSize = cSize

}

return ErrCompressedSizeTooBig

}

case blockTypeRaw:

b.RLESize = 0

// We do not need a destination for raw blocks.

maxSize = -1

b.Last = true

b.Type = blockTypeReserved

b.err = err

}

// Close will release resources.

// Closed blockDec cannot be reset.

func (b *blockDec) Close() {

}

func (b *blockDec) decodeBuf(hist *history) error {

switch b.Type {

case blockTypeRLE:

return nil

case blockTypeCompressed:

saved := b.dst

err := b.decodeCompressed(hist)

if debugDecoder {

println("Decompressed to total", len(b.dst), "bytes, hash:", xxhash.Sum64(b.dst), "error:", err)

}

return err

case blockTypeReserved:

// Used for returning errors.

}

}

// There must be at least one byte for Literals_Block_Type and one for Sequences_Section_Header

if len(in) < 2 {

}

litType := literalsBlockType(in[0] & 3)

var litRegenSize int

var litCompSize int

sizeFormat := (in[0] >> 2) & 3

var fourStreams bool

switch litType {

case literalsBlockRaw, literalsBlockRLE:

switch sizeFormat {

// Regenerated_Size uses 20 bits (0-1048575). Literals_Section_Header uses 3 bytes.

if len(in) < 3 {

println("too small: litType:", litType, " sizeFormat", sizeFormat, len(in))

}

litRegenSize = int(in[0]>>4) + (int(in[1]) << 4) + (int(in[2]) << 12)

in = in[3:]

// Both Regenerated_Size and Compressed_Size use 10 bits (0-1023).

if len(in) < 3 {

println("too small: litType:", litType, " sizeFormat", sizeFormat, len(in))

}

n := uint64(in[0]>>4) + (uint64(in[1]) << 4) + (uint64(in[2]) << 12)

litRegenSize = int(n & 1023)

fourStreams = true

if len(in) < 4 {

println("too small: litType:", litType, " sizeFormat", sizeFormat, len(in))

}

n := uint64(in[0]>>4) + (uint64(in[1]) << 4) + (uint64(in[2]) << 12) + (uint64(in[3]) << 20)

litRegenSize = int(n & 16383)

fourStreams = true

if len(in) < 5 {

println("too small: litType:", litType, " sizeFormat", sizeFormat, len(in))

}

n := uint64(in[0]>>4) + (uint64(in[1]) << 4) + (uint64(in[2]) << 12) + (uint64(in[3]) << 20) + (uint64(in[4]) << 28)

litRegenSize = int(n & 262143)

if debugDecoder {

println("literals type:", litType, "litRegenSize:", litRegenSize, "litCompSize:", litCompSize, "sizeFormat:", sizeFormat, "4X:", fourStreams)

}

switch litType {

case literalsBlockRaw:

if len(in) < litRegenSize {

println("too small: litType:", litType, " sizeFormat", sizeFormat, "remain:", len(in), "want:", litRegenSize)

}

literals = in[:litRegenSize]

in = in[litRegenSize:]

case literalsBlockRLE:

if len(in) < 1 {

println("too small: litType:", litType, " sizeFormat", sizeFormat, "remain:", len(in), "want:", 1)

}

if cap(b.literalBuf) < litRegenSize {

if b.lowMem {

b.literalBuf = make([]byte, litRegenSize)

} else {

b.literalBuf = make([]byte, litRegenSize, maxCompressedLiteralSize)

}

}

}

case literalsBlockTreeless:

if len(in) < litCompSize {

println("too small: litType:", litType, " sizeFormat", sizeFormat, "remain:", len(in), "want:", litCompSize)

}

// Store compressed literals, so we defer decoding until we get history.

literals = in[:litCompSize]

if debugDecoder {

printf("Found %d compressed literals\n", litCompSize)

}

case literalsBlockCompressed:

if len(in) < litCompSize {

println("too small: litType:", litType, " sizeFormat", sizeFormat, "remain:", len(in), "want:", litCompSize)

}

literals = in[:litCompSize]

in = in[litCompSize:]

// Ensure we have space to store it.

if cap(b.literalBuf) < litRegenSize {

if b.lowMem {

b.literalBuf = make([]byte, 0, litRegenSize)

} else {

}

}

}

huff, literals, err = huff0.ReadTable(literals, huff)

if err != nil {

println("reading huffman table:", err)

}

// Use our out buffer.

if fourStreams {

literals, err = huff.Decoder().Decompress4X(b.literalBuf[:0:litRegenSize], literals)

}

if err != nil {

println("decoding compressed literals:", err)

}

// Make sure we don't leak our literals buffer

if len(literals) != litRegenSize {

}

if debugDecoder {

printf("Decompressed %d literals into %d bytes\n", litCompSize, litRegenSize)

}

}

// Decode Sequences

// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#sequences-section

if len(in) < 1 {

return ErrBlockTooSmall

}

seqHeader := in[0]

switch {

case seqHeader < 128:

nSeqs = int(seqHeader)

in = in[1:]

nSeqs = 0x7f00 + int(in[1]) + (int(in[2]) << 8)

in = in[3:]

}

if nSeqs > 0 {

if len(in) < 1 {

return ErrBlockTooSmall

}

switch mode {

case compModePredefined:

seq.fse = &fsePredef[i]

case compModeRLE:

if br.remain() < 1 {

}

v := br.Uint8()

br.advance(1)

symb, err := decSymbolValue(v, symbolTableX[i])

if err != nil {

printf("RLE Transform table (%v) error: %v", tableIndex(i), err)

return err

}

if debugDecoder {

printf("RLE set to %+v, code: %v", symb, v)

}

case compModeFSE:

println("Reading table for", tableIndex(i))

if err != nil {

println("Read table error:", err)

return err

}

if err != nil {

println("Transform table error:", err)

return err

}

if debugDecoder {

}

case compModeRepeat:

seq.repeat = true

}

}

in = br.unread()

}

if debugDecoder {

}

if nSeqs == 0 {

}

return nil

}

}

if err := br.init(in); err != nil {

return err

}

hbytes := hist.b

if len(hbytes) > hist.windowSize {

hbytes = hbytes[len(hbytes)-hist.windowSize:]

if hist.dict != nil {

hist.dict.content = nil

}

}

if err != nil {

return err

}

if len(b.data) > maxCompressedBlockSize {

return fmt.Errorf("compressed block size too large (%d)", len(b.data))

}

// Set output and release references.

if b.Last {

// if last block we don't care about history.

println("Last block, no history returned")

hist.b = hist.b[:0]

return nil

}

return nil

}

func (r *readerWrapper) readByte() (byte, error) {

n2, err := r.r.Read(r.tmp[:1])

if err != nil {

return 0, err

}

if n2 != 1 {

package zstd

import (

"io"

"sync"

)

// Decoder provides decoding of zstandard streams.

// Unreferenced decoders, ready for use.

decoders chan *blockDec

// Current read position used for Reader functionality.

current decoderState

// Custom dictionaries.

// Always uses copies.

dicts map[uint32]dict

output chan decodeOutput

// cancel remaining output.

flushed bool

}

return nil, err

}

}

d.current.flushed = true

if r == nil {

break

}

if !d.nextBlock(n == 0) {

}

}

}

d.drainOutput()

if r == nil {

d.current.err = ErrDecoderNilInput

if len(d.current.b) > 0 {

}

return nil

}

// Remove current block.

d.current.decodeOutput = decodeOutput{}

d.current.err = nil

d.current.flushed = false

d.current.d = nil

}

return nil

}

// drainOutput will drain the output until errEndOfStream is sent.

func (d *Decoder) drainOutput() {

if d.current.cancel != nil {

d.current.cancel = nil

}

if d.current.d != nil {

}

d.decoders <- v.d

}

}

}

// WriteTo writes data to w until there's no more data to write or when an error occurs.

// DecodeAll can be used concurrently.

// The Decoder concurrency limits will be respected.

func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) {

return dst, ErrDecoderClosed

}

}

frame.rawInput = nil

frame.bBuf = nil

d.decoders <- block

}()

frame.bBuf = input

for {

frame.history.reset()

err := frame.reset(&frame.bBuf)

}

}

if frame.DictionaryID != nil {

dict, ok := d.dicts[*frame.DictionaryID]

if !ok {

return nil, ErrUnknownDictionary

}

frame.history.setDict(&dict)

}

return dst, ErrDecoderSizeExceeded

}

if cap(dst)-len(dst) < int(frame.FrameContentSize) {

dst2 := make([]byte, len(dst), len(dst)+int(frame.FrameContentSize))

copy(dst2, dst)

// If non-blocking mode is used the returned boolean will be false

// if no data was available without blocking.

func (d *Decoder) nextBlock(blocking bool) (ok bool) {

if d.current.err != nil {

// Keep error state.

}

if blocking {

} else {

select {

default:

return false

}

}

if debugDecoder {

}

return true

}

// Close will release all resources.

// It is NOT possible to reuse the decoder after this.

func (d *Decoder) Close() {

return

}

d.drainOutput()

d.streamWg.Wait()

}

if d.decoders != nil {

close(d.decoders)

err error

}

}

// Create Decoder:

defer d.streamWg.Done()

}

}

}

}

}

}

}

}

}

}

}

}

}

concurrent: runtime.GOMAXPROCS(0),

maxWindowSize: MaxWindowSize,

}

o.maxDecodedSize = 1 << 63

}

return func(o *decoderOptions) error { o.lowMem = b; return nil }

}

func WithDecoderConcurrency(n int) DOption {

return func(o *decoderOptions) error {

return errors.New("concurrency must be at least 1")

}

return nil

}

}

// TEMPLATE

const hashLog = tableBits

// seems global, but would be nice to tweak.

// nextEmit is where in src the next emitLiteral should start from.

nextEmit := s

// TEMPLATE

const hashLog = tableBits

// seems global, but would be nice to tweak.

// nextEmit is where in src the next emitLiteral should start from.

nextEmit := s

if cap(s.filling) == 0 {

s.filling = make([]byte, 0, e.o.blockSize)

}

}

if s.encoder == nil {

s.encoder = e.o.encoder()

}

s.filling = s.filling[:0]

s.encoder.Reset(e.o.dict, false)

s.headerWritten = false

s.eofWritten = false

return s.err

}

// Move blocks forward.

s.filling, s.current, s.previous = s.previous[:0], s.filling, s.current

s.nInput += int64(len(s.current))

// WithEncoderConcurrency will set the concurrency,

// meaning the maximum number of encoders to run concurrently.

// The value supplied must be at least 1.