diff options
Diffstat (limited to 'vendor/github.com/klauspost')
16 files changed, 174 insertions, 110 deletions
diff --git a/vendor/github.com/klauspost/compress/flate/deflate.go b/vendor/github.com/klauspost/compress/flate/deflate.go index 2b101d26b..25dbe3e15 100644 --- a/vendor/github.com/klauspost/compress/flate/deflate.go +++ b/vendor/github.com/klauspost/compress/flate/deflate.go @@ -80,9 +80,7 @@ type advancedState struct { // deflate state length int offset int - hash uint32 maxInsertIndex int - ii uint16 // position of last match, intended to overflow to reset. // Input hash chains // hashHead[hashValue] contains the largest inputIndex with the specified hash value @@ -97,6 +95,9 @@ type advancedState struct { // input window: unprocessed data is window[index:windowEnd] index int hashMatch [maxMatchLength + minMatchLength]uint32 + + hash uint32 + ii uint16 // position of last match, intended to overflow to reset. } type compressor struct { @@ -107,18 +108,19 @@ type compressor struct { // compression algorithm fill func(*compressor, []byte) int // copy data to window step func(*compressor) // process window - sync bool // requesting flush - window []byte - windowEnd int - blockStart int // window index where current tokens start - byteAvailable bool // if true, still need to process window[index-1]. - err error + window []byte + windowEnd int + blockStart int // window index where current tokens start + err error // queued output tokens tokens tokens fast fastEnc state *advancedState + + sync bool // requesting flush + byteAvailable bool // if true, still need to process window[index-1]. } func (d *compressor) fillDeflate(b []byte) int { diff --git a/vendor/github.com/klauspost/compress/flate/inflate.go b/vendor/github.com/klauspost/compress/flate/inflate.go index 7f175a4ec..3e4259f15 100644 --- a/vendor/github.com/klauspost/compress/flate/inflate.go +++ b/vendor/github.com/klauspost/compress/flate/inflate.go @@ -295,10 +295,6 @@ type decompressor struct { r Reader roffset int64 - // Input bits, in top of b. - b uint32 - nb uint - // Huffman decoders for literal/length, distance. h1, h2 huffmanDecoder @@ -309,19 +305,24 @@ type decompressor struct { // Output history, buffer. dict dictDecoder - // Temporary buffer (avoids repeated allocation). - buf [4]byte - // Next step in the decompression, // and decompression state. step func(*decompressor) stepState int - final bool err error toRead []byte hl, hd *huffmanDecoder copyLen int copyDist int + + // Temporary buffer (avoids repeated allocation). + buf [4]byte + + // Input bits, in top of b. + b uint32 + + nb uint + final bool } func (f *decompressor) nextBlock() { diff --git a/vendor/github.com/klauspost/compress/fse/fse.go b/vendor/github.com/klauspost/compress/fse/fse.go index 075357b5b..535cbadfd 100644 --- a/vendor/github.com/klauspost/compress/fse/fse.go +++ b/vendor/github.com/klauspost/compress/fse/fse.go @@ -44,18 +44,14 @@ var ( // Scratch provides temporary storage for compression and decompression. type Scratch struct { // Private - count [maxSymbolValue + 1]uint32 - norm [maxSymbolValue + 1]int16 - symbolLen uint16 // Length of active part of the symbol table. - actualTableLog uint8 // Selected tablelog. - br byteReader - bits bitReader - bw bitWriter - ct cTable // Compression tables. - decTable []decSymbol // Decompression table. - zeroBits bool // no bits has prob > 50%. - clearCount bool // clear count - maxCount int // count of the most probable symbol + count [maxSymbolValue + 1]uint32 + norm [maxSymbolValue + 1]int16 + br byteReader + bits bitReader + bw bitWriter + ct cTable // Compression tables. + decTable []decSymbol // Decompression table. + maxCount int // count of the most probable symbol // Per block parameters. // These can be used to override compression parameters of the block. @@ -68,17 +64,22 @@ type Scratch struct { // and allocation will be avoided. Out []byte - // MaxSymbolValue will override the maximum symbol value of the next block. - MaxSymbolValue uint8 - - // TableLog will attempt to override the tablelog for the next block. - TableLog uint8 - // DecompressLimit limits the maximum decoded size acceptable. // If > 0 decompression will stop when approximately this many bytes // has been decoded. // If 0, maximum size will be 2GB. DecompressLimit int + + symbolLen uint16 // Length of active part of the symbol table. + actualTableLog uint8 // Selected tablelog. + zeroBits bool // no bits has prob > 50%. + clearCount bool // clear count + + // MaxSymbolValue will override the maximum symbol value of the next block. + MaxSymbolValue uint8 + + // TableLog will attempt to override the tablelog for the next block. + TableLog uint8 } // Histogram allows to populate the histogram and skip that step in the compression, diff --git a/vendor/github.com/klauspost/compress/huff0/huff0.go b/vendor/github.com/klauspost/compress/huff0/huff0.go index 53249df05..177d6c4ea 100644 --- a/vendor/github.com/klauspost/compress/huff0/huff0.go +++ b/vendor/github.com/klauspost/compress/huff0/huff0.go @@ -79,6 +79,13 @@ type Scratch struct { // Slice of the returned data. OutData []byte + // MaxDecodedSize will set the maximum allowed output size. + // This value will automatically be set to BlockSizeMax if not set. + // Decoders will return ErrMaxDecodedSizeExceeded is this limit is exceeded. + MaxDecodedSize int + + br byteReader + // MaxSymbolValue will override the maximum symbol value of the next block. MaxSymbolValue uint8 @@ -95,12 +102,6 @@ type Scratch struct { // If WantLogLess == 0 any improvement will do. WantLogLess uint8 - // MaxDecodedSize will set the maximum allowed output size. - // This value will automatically be set to BlockSizeMax if not set. - // Decoders will return ErrMaxDecodedSizeExceeded is this limit is exceeded. - MaxDecodedSize int - - br byteReader symbolLen uint16 // Length of active part of the symbol table. maxCount int // count of the most probable symbol clearCount bool // clear count diff --git a/vendor/github.com/klauspost/compress/zstd/blockdec.go b/vendor/github.com/klauspost/compress/zstd/blockdec.go index ed670bcc7..19181caea 100644 --- a/vendor/github.com/klauspost/compress/zstd/blockdec.go +++ b/vendor/github.com/klauspost/compress/zstd/blockdec.go @@ -75,21 +75,25 @@ type blockDec struct { // Window size of the block. WindowSize uint64 - Type blockType - RLESize uint32 - // Is this the last block of a frame? - Last bool - - // Use less memory - lowMem bool history chan *history input chan struct{} result chan decodeOutput sequenceBuf []seq - tmp [4]byte err error decWG sync.WaitGroup + + // Block is RLE, this is the size. + RLESize uint32 + tmp [4]byte + + Type blockType + + // Is this the last block of a frame? + Last bool + + // Use less memory + lowMem bool } func (b *blockDec) String() string { @@ -127,25 +131,37 @@ func (b *blockDec) reset(br byteBuffer, windowSize uint64) error { b.Type = blockType((bh >> 1) & 3) // find size. cSize := int(bh >> 3) + maxSize := maxBlockSize switch b.Type { case blockTypeReserved: return ErrReservedBlockType case blockTypeRLE: b.RLESize = uint32(cSize) + if b.lowMem { + maxSize = cSize + } cSize = 1 case blockTypeCompressed: if debug { println("Data size on stream:", cSize) } b.RLESize = 0 + maxSize = maxCompressedBlockSize + if windowSize < maxCompressedBlockSize && b.lowMem { + maxSize = int(windowSize) + } if cSize > maxCompressedBlockSize || uint64(cSize) > b.WindowSize { if debug { printf("compressed block too big: csize:%d block: %+v\n", uint64(cSize), b) } return ErrCompressedSizeTooBig } - default: + case blockTypeRaw: b.RLESize = 0 + // We do not need a destination for raw blocks. + maxSize = -1 + default: + panic("Invalid block type") } // Read block data. @@ -156,8 +172,8 @@ func (b *blockDec) reset(br byteBuffer, windowSize uint64) error { b.dataStorage = make([]byte, 0, maxBlockSize) } } - if cap(b.dst) <= maxBlockSize { - b.dst = make([]byte, 0, maxBlockSize+1) + if cap(b.dst) <= maxSize { + b.dst = make([]byte, 0, maxSize+1) } var err error b.data, err = br.readBig(cSize, b.dataStorage) @@ -675,8 +691,11 @@ func (b *blockDec) decodeCompressed(hist *history) error { println("initializing sequences:", err) return err } - - err = seqs.decode(nSeqs, br, hist.b) + hbytes := hist.b + if len(hbytes) > hist.windowSize { + hbytes = hbytes[len(hbytes)-hist.windowSize:] + } + err = seqs.decode(nSeqs, br, hbytes) if err != nil { return err } diff --git a/vendor/github.com/klauspost/compress/zstd/decoder.go b/vendor/github.com/klauspost/compress/zstd/decoder.go index 86553c2c3..324347623 100644 --- a/vendor/github.com/klauspost/compress/zstd/decoder.go +++ b/vendor/github.com/klauspost/compress/zstd/decoder.go @@ -169,7 +169,12 @@ func (d *Decoder) Reset(r io.Reader) error { println("*bytes.Buffer detected, doing sync decode, len:", bb.Len()) } b := bb.Bytes() - dst, err := d.DecodeAll(b, nil) + var dst []byte + if cap(d.current.b) > 0 { + dst = d.current.b + } + + dst, err := d.DecodeAll(b, dst[:0]) if err == nil { err = io.EOF } @@ -456,6 +461,7 @@ func (d *Decoder) startStreamDecoder(inStream chan decodeStream) { br := readerWrapper{r: stream.r} decodeStream: for { + frame.history.reset() err := frame.reset(&br) if debug && err != nil { println("Frame decoder returned", err) diff --git a/vendor/github.com/klauspost/compress/zstd/enc_better.go b/vendor/github.com/klauspost/compress/zstd/enc_better.go index 4375e08b4..c120d9054 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_better.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_better.go @@ -104,10 +104,7 @@ func (e *betterFastEncoder) Encode(blk *blockEnc, src []byte) { sLimit := int32(len(src)) - inputMargin // stepSize is the number of bytes to skip on every main loop iteration. // It should be >= 1. - stepSize := int32(e.o.targetLength) - if stepSize == 0 { - stepSize++ - } + const stepSize = 1 const kSearchStrength = 9 diff --git a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go index d640e6a9f..5ebead9dc 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go @@ -80,10 +80,7 @@ func (e *doubleFastEncoder) Encode(blk *blockEnc, src []byte) { sLimit := int32(len(src)) - inputMargin // stepSize is the number of bytes to skip on every main loop iteration. // It should be >= 1. - stepSize := int32(e.o.targetLength) - if stepSize == 0 { - stepSize++ - } + const stepSize = 1 const kSearchStrength = 8 @@ -401,10 +398,7 @@ func (e *doubleFastEncoder) EncodeNoHist(blk *blockEnc, src []byte) { sLimit := int32(len(src)) - inputMargin // stepSize is the number of bytes to skip on every main loop iteration. // It should be >= 1. - stepSize := int32(e.o.targetLength) - if stepSize == 0 { - stepSize++ - } + const stepSize = 1 const kSearchStrength = 8 diff --git a/vendor/github.com/klauspost/compress/zstd/enc_fast.go b/vendor/github.com/klauspost/compress/zstd/enc_fast.go index 1387b8082..d1d3658e6 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_fast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_fast.go @@ -25,7 +25,6 @@ type tableEntry struct { } type fastBase struct { - o encParams // cur is the offset at the start of hist cur int32 // maximum offset. Should be at least 2x block size. @@ -117,11 +116,7 @@ func (e *fastEncoder) Encode(blk *blockEnc, src []byte) { sLimit := int32(len(src)) - inputMargin // stepSize is the number of bytes to skip on every main loop iteration. // It should be >= 2. - stepSize := int32(e.o.targetLength) - if stepSize == 0 { - stepSize++ - } - stepSize++ + const stepSize = 2 // TEMPLATE const hashLog = tableBits diff --git a/vendor/github.com/klauspost/compress/zstd/enc_params.go b/vendor/github.com/klauspost/compress/zstd/enc_params.go index b6779ecb6..d874116f7 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_params.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_params.go @@ -4,6 +4,8 @@ package zstd +/* +// encParams are not really used, just here for reference. type encParams struct { // largest match distance : larger == more compression, more memory needed during decompression windowLog uint8 @@ -152,3 +154,4 @@ var defEncParams = [4][]encParams{ {14, 15, 15, 10, 3, 999, strategyBtultra2}, // level 22. }, } +*/ diff --git a/vendor/github.com/klauspost/compress/zstd/encoder.go b/vendor/github.com/klauspost/compress/zstd/encoder.go index 67d45efb9..af4f00b73 100644 --- a/vendor/github.com/klauspost/compress/zstd/encoder.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder.go @@ -39,17 +39,18 @@ type encoder interface { } type encoderState struct { - w io.Writer - filling []byte - current []byte - previous []byte - encoder encoder - writing *blockEnc - err error - writeErr error - nWritten int64 - headerWritten bool - eofWritten bool + w io.Writer + filling []byte + current []byte + previous []byte + encoder encoder + writing *blockEnc + err error + writeErr error + nWritten int64 + headerWritten bool + eofWritten bool + fullFrameWritten bool // This waitgroup indicates an encode is running. wg sync.WaitGroup @@ -114,6 +115,7 @@ func (e *Encoder) Reset(w io.Writer) { s.encoder.Reset() s.headerWritten = false s.eofWritten = false + s.fullFrameWritten = false s.w = w s.err = nil s.nWritten = 0 @@ -172,6 +174,22 @@ func (e *Encoder) nextBlock(final bool) error { return fmt.Errorf("block > maxStoreBlockSize") } if !s.headerWritten { + // If we have a single block encode, do a sync compression. + if final && len(s.filling) > 0 { + s.current = e.EncodeAll(s.filling, s.current[:0]) + var n2 int + n2, s.err = s.w.Write(s.current) + if s.err != nil { + return s.err + } + s.nWritten += int64(n2) + s.current = s.current[:0] + s.filling = s.filling[:0] + s.headerWritten = true + s.fullFrameWritten = true + return nil + } + var tmp [maxHeaderSize]byte fh := frameHeader{ ContentSize: 0, @@ -294,7 +312,9 @@ func (e *Encoder) ReadFrom(r io.Reader) (n int64, err error) { src := e.state.filling for { n2, err := r.Read(src) - _, _ = e.state.encoder.CRC().Write(src[:n2]) + if e.o.crc { + _, _ = e.state.encoder.CRC().Write(src[:n2]) + } // src is now the unfilled part... src = src[n2:] n += int64(n2) @@ -359,6 +379,9 @@ func (e *Encoder) Close() error { if err != nil { return err } + if e.state.fullFrameWritten { + return s.err + } s.wg.Wait() s.wWg.Wait() diff --git a/vendor/github.com/klauspost/compress/zstd/encoder_options.go b/vendor/github.com/klauspost/compress/zstd/encoder_options.go index 0ff970dac..3fc03097a 100644 --- a/vendor/github.com/klauspost/compress/zstd/encoder_options.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder_options.go @@ -12,15 +12,16 @@ type EOption func(*encoderOptions) error // options retains accumulated state of multiple options. type encoderOptions struct { - concurrent int - crc bool - single *bool - pad int - blockSize int - windowSize int - level EncoderLevel - fullZero bool - noEntropy bool + concurrent int + level EncoderLevel + single *bool + pad int + blockSize int + windowSize int + crc bool + fullZero bool + noEntropy bool + customWindow bool } func (o *encoderOptions) setDefault() { @@ -30,7 +31,7 @@ func (o *encoderOptions) setDefault() { crc: true, single: nil, blockSize: 1 << 16, - windowSize: 1 << 22, + windowSize: 8 << 20, level: SpeedDefault, } } @@ -85,6 +86,7 @@ func WithWindowSize(n int) EOption { } o.windowSize = n + o.customWindow = true if o.blockSize > o.windowSize { o.blockSize = o.windowSize } @@ -195,6 +197,16 @@ func WithEncoderLevel(l EncoderLevel) EOption { return fmt.Errorf("unknown encoder level") } o.level = l + if !o.customWindow { + switch o.level { + case SpeedFastest: + o.windowSize = 4 << 20 + case SpeedDefault: + o.windowSize = 8 << 20 + case SpeedBetterCompression: + o.windowSize = 16 << 20 + } + } return nil } } diff --git a/vendor/github.com/klauspost/compress/zstd/framedec.go b/vendor/github.com/klauspost/compress/zstd/framedec.go index cda590b5f..780880ebe 100644 --- a/vendor/github.com/klauspost/compress/zstd/framedec.go +++ b/vendor/github.com/klauspost/compress/zstd/framedec.go @@ -16,16 +16,11 @@ import ( ) type frameDec struct { - o decoderOptions - crc hash.Hash64 - frameDone sync.WaitGroup - offset int64 + o decoderOptions + crc hash.Hash64 + offset int64 - WindowSize uint64 - DictionaryID uint32 - FrameContentSize uint64 - HasCheckSum bool - SingleSegment bool + WindowSize uint64 // maxWindowSize is the maximum windows size to support. // should never be bigger than max-int. @@ -42,9 +37,16 @@ type frameDec struct { // Byte buffer that can be reused for small input blocks. bBuf byteBuf + FrameContentSize uint64 + frameDone sync.WaitGroup + + DictionaryID uint32 + HasCheckSum bool + SingleSegment bool + // asyncRunning indicates whether the async routine processes input on 'decoding'. - asyncRunning bool asyncRunningMu sync.Mutex + asyncRunning bool } const ( @@ -231,7 +233,11 @@ func (d *frameDec) reset(br byteBuffer) error { return ErrWindowSizeTooSmall } d.history.windowSize = int(d.WindowSize) - d.history.maxSize = d.history.windowSize + maxBlockSize + if d.o.lowMem && d.history.windowSize < maxBlockSize { + d.history.maxSize = d.history.windowSize * 2 + } else { + d.history.maxSize = d.history.windowSize + maxBlockSize + } // history contains input - maybe we do something d.rawInput = br return nil @@ -318,8 +324,8 @@ func (d *frameDec) checkCRC() error { func (d *frameDec) initAsync() { if !d.o.lowMem && !d.SingleSegment { - // set max extra size history to 20MB. - d.history.maxSize = d.history.windowSize + maxBlockSize*10 + // set max extra size history to 10MB. + d.history.maxSize = d.history.windowSize + maxBlockSize*5 } // re-alloc if more than one extra block size. if d.o.lowMem && cap(d.history.b) > d.history.maxSize+maxBlockSize { diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec.go b/vendor/github.com/klauspost/compress/zstd/seqdec.go index 15a45f7b5..39238e16a 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec.go @@ -64,6 +64,7 @@ type sequenceDecs struct { hist []byte literals []byte out []byte + windowSize int maxBits uint8 } @@ -82,6 +83,7 @@ func (s *sequenceDecs) initialize(br *bitReader, hist *history, literals, out [] s.hist = hist.b s.prevOffset = hist.recentOffsets s.maxBits = s.litLengths.fse.maxBits + s.offsets.fse.maxBits + s.matchLengths.fse.maxBits + s.windowSize = hist.windowSize s.out = out return nil } @@ -131,6 +133,9 @@ func (s *sequenceDecs) decode(seqs int, br *bitReader, hist []byte) error { if matchOff > len(s.out)+len(hist)+litLen { return fmt.Errorf("match offset (%d) bigger than current history (%d)", matchOff, len(s.out)+len(hist)+litLen) } + if matchOff > s.windowSize { + return fmt.Errorf("match offset (%d) bigger than window size (%d)", matchOff, s.windowSize) + } if matchOff == 0 && matchLen > 0 { return fmt.Errorf("zero matchoff and matchlen > 0") } diff --git a/vendor/github.com/klauspost/pgzip/LICENSE b/vendor/github.com/klauspost/pgzip/LICENSE index 2bdc0d751..3909da410 100644 --- a/vendor/github.com/klauspost/pgzip/LICENSE +++ b/vendor/github.com/klauspost/pgzip/LICENSE @@ -1,4 +1,4 @@ -The MIT License (MIT) +MIT License Copyright (c) 2014 Klaus Post @@ -19,4 +19,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - diff --git a/vendor/github.com/klauspost/pgzip/gzip.go b/vendor/github.com/klauspost/pgzip/gzip.go index cb3dc0896..257c4d299 100644 --- a/vendor/github.com/klauspost/pgzip/gzip.go +++ b/vendor/github.com/klauspost/pgzip/gzip.go @@ -405,7 +405,7 @@ func (z *Writer) Write(p []byte) (int, error) { if len(z.currentBuffer) == z.blockSize { z.compressCurrent(false) if err := z.checkError(); err != nil { - return len(p) - len(q) - length, err + return len(p) - len(q), err } } z.size += length |