diff options
Diffstat (limited to 'vendor/github.com/klauspost/compress/zstd')
9 files changed, 94 insertions, 59 deletions
diff --git a/vendor/github.com/klauspost/compress/zstd/blockdec.go b/vendor/github.com/klauspost/compress/zstd/blockdec.go index ed670bcc7..63062ffa6 100644 --- a/vendor/github.com/klauspost/compress/zstd/blockdec.go +++ b/vendor/github.com/klauspost/compress/zstd/blockdec.go @@ -75,21 +75,25 @@ type blockDec struct { // Window size of the block. WindowSize uint64 - Type blockType - RLESize uint32 - // Is this the last block of a frame? - Last bool - - // Use less memory - lowMem bool history chan *history input chan struct{} result chan decodeOutput sequenceBuf []seq - tmp [4]byte err error decWG sync.WaitGroup + + // Block is RLE, this is the size. + RLESize uint32 + tmp [4]byte + + Type blockType + + // Is this the last block of a frame? + Last bool + + // Use less memory + lowMem bool } func (b *blockDec) String() string { diff --git a/vendor/github.com/klauspost/compress/zstd/decoder.go b/vendor/github.com/klauspost/compress/zstd/decoder.go index 86553c2c3..234025505 100644 --- a/vendor/github.com/klauspost/compress/zstd/decoder.go +++ b/vendor/github.com/klauspost/compress/zstd/decoder.go @@ -169,7 +169,12 @@ func (d *Decoder) Reset(r io.Reader) error { println("*bytes.Buffer detected, doing sync decode, len:", bb.Len()) } b := bb.Bytes() - dst, err := d.DecodeAll(b, nil) + var dst []byte + if cap(d.current.b) > 0 { + dst = d.current.b + } + + dst, err := d.DecodeAll(b, dst[:0]) if err == nil { err = io.EOF } diff --git a/vendor/github.com/klauspost/compress/zstd/enc_better.go b/vendor/github.com/klauspost/compress/zstd/enc_better.go index 4375e08b4..c120d9054 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_better.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_better.go @@ -104,10 +104,7 @@ func (e *betterFastEncoder) Encode(blk *blockEnc, src []byte) { sLimit := int32(len(src)) - inputMargin // stepSize is the number of bytes to skip on every main loop iteration. // It should be >= 1. - stepSize := int32(e.o.targetLength) - if stepSize == 0 { - stepSize++ - } + const stepSize = 1 const kSearchStrength = 9 diff --git a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go index d640e6a9f..5ebead9dc 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go @@ -80,10 +80,7 @@ func (e *doubleFastEncoder) Encode(blk *blockEnc, src []byte) { sLimit := int32(len(src)) - inputMargin // stepSize is the number of bytes to skip on every main loop iteration. // It should be >= 1. - stepSize := int32(e.o.targetLength) - if stepSize == 0 { - stepSize++ - } + const stepSize = 1 const kSearchStrength = 8 @@ -401,10 +398,7 @@ func (e *doubleFastEncoder) EncodeNoHist(blk *blockEnc, src []byte) { sLimit := int32(len(src)) - inputMargin // stepSize is the number of bytes to skip on every main loop iteration. // It should be >= 1. - stepSize := int32(e.o.targetLength) - if stepSize == 0 { - stepSize++ - } + const stepSize = 1 const kSearchStrength = 8 diff --git a/vendor/github.com/klauspost/compress/zstd/enc_fast.go b/vendor/github.com/klauspost/compress/zstd/enc_fast.go index 1387b8082..d1d3658e6 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_fast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_fast.go @@ -25,7 +25,6 @@ type tableEntry struct { } type fastBase struct { - o encParams // cur is the offset at the start of hist cur int32 // maximum offset. Should be at least 2x block size. @@ -117,11 +116,7 @@ func (e *fastEncoder) Encode(blk *blockEnc, src []byte) { sLimit := int32(len(src)) - inputMargin // stepSize is the number of bytes to skip on every main loop iteration. // It should be >= 2. - stepSize := int32(e.o.targetLength) - if stepSize == 0 { - stepSize++ - } - stepSize++ + const stepSize = 2 // TEMPLATE const hashLog = tableBits diff --git a/vendor/github.com/klauspost/compress/zstd/enc_params.go b/vendor/github.com/klauspost/compress/zstd/enc_params.go index b6779ecb6..d874116f7 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_params.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_params.go @@ -4,6 +4,8 @@ package zstd +/* +// encParams are not really used, just here for reference. type encParams struct { // largest match distance : larger == more compression, more memory needed during decompression windowLog uint8 @@ -152,3 +154,4 @@ var defEncParams = [4][]encParams{ {14, 15, 15, 10, 3, 999, strategyBtultra2}, // level 22. }, } +*/ diff --git a/vendor/github.com/klauspost/compress/zstd/encoder.go b/vendor/github.com/klauspost/compress/zstd/encoder.go index 67d45efb9..af4f00b73 100644 --- a/vendor/github.com/klauspost/compress/zstd/encoder.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder.go @@ -39,17 +39,18 @@ type encoder interface { } type encoderState struct { - w io.Writer - filling []byte - current []byte - previous []byte - encoder encoder - writing *blockEnc - err error - writeErr error - nWritten int64 - headerWritten bool - eofWritten bool + w io.Writer + filling []byte + current []byte + previous []byte + encoder encoder + writing *blockEnc + err error + writeErr error + nWritten int64 + headerWritten bool + eofWritten bool + fullFrameWritten bool // This waitgroup indicates an encode is running. wg sync.WaitGroup @@ -114,6 +115,7 @@ func (e *Encoder) Reset(w io.Writer) { s.encoder.Reset() s.headerWritten = false s.eofWritten = false + s.fullFrameWritten = false s.w = w s.err = nil s.nWritten = 0 @@ -172,6 +174,22 @@ func (e *Encoder) nextBlock(final bool) error { return fmt.Errorf("block > maxStoreBlockSize") } if !s.headerWritten { + // If we have a single block encode, do a sync compression. + if final && len(s.filling) > 0 { + s.current = e.EncodeAll(s.filling, s.current[:0]) + var n2 int + n2, s.err = s.w.Write(s.current) + if s.err != nil { + return s.err + } + s.nWritten += int64(n2) + s.current = s.current[:0] + s.filling = s.filling[:0] + s.headerWritten = true + s.fullFrameWritten = true + return nil + } + var tmp [maxHeaderSize]byte fh := frameHeader{ ContentSize: 0, @@ -294,7 +312,9 @@ func (e *Encoder) ReadFrom(r io.Reader) (n int64, err error) { src := e.state.filling for { n2, err := r.Read(src) - _, _ = e.state.encoder.CRC().Write(src[:n2]) + if e.o.crc { + _, _ = e.state.encoder.CRC().Write(src[:n2]) + } // src is now the unfilled part... src = src[n2:] n += int64(n2) @@ -359,6 +379,9 @@ func (e *Encoder) Close() error { if err != nil { return err } + if e.state.fullFrameWritten { + return s.err + } s.wg.Wait() s.wWg.Wait() diff --git a/vendor/github.com/klauspost/compress/zstd/encoder_options.go b/vendor/github.com/klauspost/compress/zstd/encoder_options.go index 0ff970dac..3fc03097a 100644 --- a/vendor/github.com/klauspost/compress/zstd/encoder_options.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder_options.go @@ -12,15 +12,16 @@ type EOption func(*encoderOptions) error // options retains accumulated state of multiple options. type encoderOptions struct { - concurrent int - crc bool - single *bool - pad int - blockSize int - windowSize int - level EncoderLevel - fullZero bool - noEntropy bool + concurrent int + level EncoderLevel + single *bool + pad int + blockSize int + windowSize int + crc bool + fullZero bool + noEntropy bool + customWindow bool } func (o *encoderOptions) setDefault() { @@ -30,7 +31,7 @@ func (o *encoderOptions) setDefault() { crc: true, single: nil, blockSize: 1 << 16, - windowSize: 1 << 22, + windowSize: 8 << 20, level: SpeedDefault, } } @@ -85,6 +86,7 @@ func WithWindowSize(n int) EOption { } o.windowSize = n + o.customWindow = true if o.blockSize > o.windowSize { o.blockSize = o.windowSize } @@ -195,6 +197,16 @@ func WithEncoderLevel(l EncoderLevel) EOption { return fmt.Errorf("unknown encoder level") } o.level = l + if !o.customWindow { + switch o.level { + case SpeedFastest: + o.windowSize = 4 << 20 + case SpeedDefault: + o.windowSize = 8 << 20 + case SpeedBetterCompression: + o.windowSize = 16 << 20 + } + } return nil } } diff --git a/vendor/github.com/klauspost/compress/zstd/framedec.go b/vendor/github.com/klauspost/compress/zstd/framedec.go index cda590b5f..e38f34a9b 100644 --- a/vendor/github.com/klauspost/compress/zstd/framedec.go +++ b/vendor/github.com/klauspost/compress/zstd/framedec.go @@ -16,16 +16,11 @@ import ( ) type frameDec struct { - o decoderOptions - crc hash.Hash64 - frameDone sync.WaitGroup - offset int64 + o decoderOptions + crc hash.Hash64 + offset int64 - WindowSize uint64 - DictionaryID uint32 - FrameContentSize uint64 - HasCheckSum bool - SingleSegment bool + WindowSize uint64 // maxWindowSize is the maximum windows size to support. // should never be bigger than max-int. @@ -42,9 +37,16 @@ type frameDec struct { // Byte buffer that can be reused for small input blocks. bBuf byteBuf + FrameContentSize uint64 + frameDone sync.WaitGroup + + DictionaryID uint32 + HasCheckSum bool + SingleSegment bool + // asyncRunning indicates whether the async routine processes input on 'decoding'. - asyncRunning bool asyncRunningMu sync.Mutex + asyncRunning bool } const ( |