diff options
Diffstat (limited to 'vendor')
50 files changed, 1466 insertions, 671 deletions
diff --git a/vendor/github.com/BurntSushi/toml/COPYING b/vendor/github.com/BurntSushi/toml/COPYING index 5a8e33254..01b574320 100644 --- a/vendor/github.com/BurntSushi/toml/COPYING +++ b/vendor/github.com/BurntSushi/toml/COPYING @@ -1,14 +1,21 @@ - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - Version 2, December 2004 +The MIT License (MIT) - Copyright (C) 2004 Sam Hocevar <sam@hocevar.net> +Copyright (c) 2013 TOML authors - Everyone is permitted to copy and distribute verbatim or modified - copies of this license document, and changing it is allowed as long - as the name is changed. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. You just DO WHAT THE FUCK YOU WANT TO. +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/BurntSushi/toml/README.md b/vendor/github.com/BurntSushi/toml/README.md index 5a5df6370..7c1b37ecc 100644 --- a/vendor/github.com/BurntSushi/toml/README.md +++ b/vendor/github.com/BurntSushi/toml/README.md @@ -1,17 +1,17 @@ ## TOML parser and encoder for Go with reflection TOML stands for Tom's Obvious, Minimal Language. This Go package provides a -reflection interface similar to Go's standard library `json` and `xml` +reflection interface similar to Go's standard library `json` and `xml` packages. This package also supports the `encoding.TextUnmarshaler` and -`encoding.TextMarshaler` interfaces so that you can define custom data +`encoding.TextMarshaler` interfaces so that you can define custom data representations. (There is an example of this below.) -Spec: https://github.com/mojombo/toml +Spec: https://github.com/toml-lang/toml Compatible with TOML version -[v0.2.0](https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.2.0.md) +[v0.4.0](https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.4.0.md) -Documentation: http://godoc.org/github.com/BurntSushi/toml +Documentation: https://godoc.org/github.com/BurntSushi/toml Installation: @@ -26,8 +26,7 @@ go get github.com/BurntSushi/toml/cmd/tomlv tomlv some-toml-file.toml ``` -[](https://travis-ci.org/BurntSushi/toml) - +[](https://travis-ci.org/BurntSushi/toml) [](https://godoc.org/github.com/BurntSushi/toml) ### Testing @@ -87,7 +86,7 @@ type TOML struct { ### Using the `encoding.TextUnmarshaler` interface -Here's an example that automatically parses duration strings into +Here's an example that automatically parses duration strings into `time.Duration` values: ```toml @@ -120,7 +119,7 @@ for _, s := range favorites.Song { } ``` -And you'll also need a `duration` type that satisfies the +And you'll also need a `duration` type that satisfies the `encoding.TextUnmarshaler` interface: ```go @@ -217,4 +216,3 @@ Note that a case insensitive match will be tried if an exact match can't be found. A working example of the above can be found in `_examples/example.{go,toml}`. - diff --git a/vendor/github.com/BurntSushi/toml/decode.go b/vendor/github.com/BurntSushi/toml/decode.go index c26b00c01..b0fd51d5b 100644 --- a/vendor/github.com/BurntSushi/toml/decode.go +++ b/vendor/github.com/BurntSushi/toml/decode.go @@ -10,7 +10,9 @@ import ( "time" ) -var e = fmt.Errorf +func e(format string, args ...interface{}) error { + return fmt.Errorf("toml: "+format, args...) +} // Unmarshaler is the interface implemented by objects that can unmarshal a // TOML description of themselves. @@ -103,6 +105,13 @@ func (md *MetaData) PrimitiveDecode(primValue Primitive, v interface{}) error { // This decoder will not handle cyclic types. If a cyclic type is passed, // `Decode` will not terminate. func Decode(data string, v interface{}) (MetaData, error) { + rv := reflect.ValueOf(v) + if rv.Kind() != reflect.Ptr { + return MetaData{}, e("Decode of non-pointer %s", reflect.TypeOf(v)) + } + if rv.IsNil() { + return MetaData{}, e("Decode of nil %s", reflect.TypeOf(v)) + } p, err := parse(data) if err != nil { return MetaData{}, err @@ -111,7 +120,7 @@ func Decode(data string, v interface{}) (MetaData, error) { p.mapping, p.types, p.ordered, make(map[string]bool, len(p.ordered)), nil, } - return md, md.unify(p.mapping, rvalue(v)) + return md, md.unify(p.mapping, indirect(rv)) } // DecodeFile is just like Decode, except it will automatically read the @@ -211,7 +220,7 @@ func (md *MetaData) unify(data interface{}, rv reflect.Value) error { case reflect.Interface: // we only support empty interfaces. if rv.NumMethod() > 0 { - return e("Unsupported type '%s'.", rv.Kind()) + return e("unsupported type %s", rv.Type()) } return md.unifyAnything(data, rv) case reflect.Float32: @@ -219,7 +228,7 @@ func (md *MetaData) unify(data interface{}, rv reflect.Value) error { case reflect.Float64: return md.unifyFloat64(data, rv) } - return e("Unsupported type '%s'.", rv.Kind()) + return e("unsupported type %s", rv.Kind()) } func (md *MetaData) unifyStruct(mapping interface{}, rv reflect.Value) error { @@ -228,7 +237,8 @@ func (md *MetaData) unifyStruct(mapping interface{}, rv reflect.Value) error { if mapping == nil { return nil } - return mismatch(rv, "map", mapping) + return e("type mismatch for %s: expected table but found %T", + rv.Type().String(), mapping) } for key, datum := range tmap { @@ -253,14 +263,13 @@ func (md *MetaData) unifyStruct(mapping interface{}, rv reflect.Value) error { md.decoded[md.context.add(key).String()] = true md.context = append(md.context, key) if err := md.unify(datum, subv); err != nil { - return e("Type mismatch for '%s.%s': %s", - rv.Type().String(), f.name, err) + return err } md.context = md.context[0 : len(md.context)-1] } else if f.name != "" { // Bad user! No soup for you! - return e("Field '%s.%s' is unexported, and therefore cannot "+ - "be loaded with reflection.", rv.Type().String(), f.name) + return e("cannot write unexported field %s.%s", + rv.Type().String(), f.name) } } } @@ -378,15 +387,15 @@ func (md *MetaData) unifyInt(data interface{}, rv reflect.Value) error { // No bounds checking necessary. case reflect.Int8: if num < math.MinInt8 || num > math.MaxInt8 { - return e("Value '%d' is out of range for int8.", num) + return e("value %d is out of range for int8", num) } case reflect.Int16: if num < math.MinInt16 || num > math.MaxInt16 { - return e("Value '%d' is out of range for int16.", num) + return e("value %d is out of range for int16", num) } case reflect.Int32: if num < math.MinInt32 || num > math.MaxInt32 { - return e("Value '%d' is out of range for int32.", num) + return e("value %d is out of range for int32", num) } } rv.SetInt(num) @@ -397,15 +406,15 @@ func (md *MetaData) unifyInt(data interface{}, rv reflect.Value) error { // No bounds checking necessary. case reflect.Uint8: if num < 0 || unum > math.MaxUint8 { - return e("Value '%d' is out of range for uint8.", num) + return e("value %d is out of range for uint8", num) } case reflect.Uint16: if num < 0 || unum > math.MaxUint16 { - return e("Value '%d' is out of range for uint16.", num) + return e("value %d is out of range for uint16", num) } case reflect.Uint32: if num < 0 || unum > math.MaxUint32 { - return e("Value '%d' is out of range for uint32.", num) + return e("value %d is out of range for uint32", num) } } rv.SetUint(unum) @@ -471,7 +480,7 @@ func rvalue(v interface{}) reflect.Value { // interest to us (like encoding.TextUnmarshaler). func indirect(v reflect.Value) reflect.Value { if v.Kind() != reflect.Ptr { - if v.CanAddr() { + if v.CanSet() { pv := v.Addr() if _, ok := pv.Interface().(TextUnmarshaler); ok { return pv @@ -496,10 +505,5 @@ func isUnifiable(rv reflect.Value) bool { } func badtype(expected string, data interface{}) error { - return e("Expected %s but found '%T'.", expected, data) -} - -func mismatch(user reflect.Value, expected string, data interface{}) error { - return e("Type mismatch for %s. Expected %s but found '%T'.", - user.Type().String(), expected, data) + return e("cannot load TOML value of type %T into a Go %s", data, expected) } diff --git a/vendor/github.com/BurntSushi/toml/decode_meta.go b/vendor/github.com/BurntSushi/toml/decode_meta.go index ef6f545fa..b9914a679 100644 --- a/vendor/github.com/BurntSushi/toml/decode_meta.go +++ b/vendor/github.com/BurntSushi/toml/decode_meta.go @@ -77,9 +77,8 @@ func (k Key) maybeQuoted(i int) string { } if quote { return "\"" + strings.Replace(k[i], "\"", "\\\"", -1) + "\"" - } else { - return k[i] } + return k[i] } func (k Key) add(piece string) Key { diff --git a/vendor/github.com/BurntSushi/toml/doc.go b/vendor/github.com/BurntSushi/toml/doc.go index fe2680004..b371f396e 100644 --- a/vendor/github.com/BurntSushi/toml/doc.go +++ b/vendor/github.com/BurntSushi/toml/doc.go @@ -4,7 +4,7 @@ files via reflection. There is also support for delaying decoding with the Primitive type, and querying the set of keys in a TOML document with the MetaData type. -The specification implemented: https://github.com/mojombo/toml +The specification implemented: https://github.com/toml-lang/toml The sub-command github.com/BurntSushi/toml/cmd/tomlv can be used to verify whether a file is a valid TOML document. It can also be used to print the diff --git a/vendor/github.com/BurntSushi/toml/encode.go b/vendor/github.com/BurntSushi/toml/encode.go index 4e4c97aed..d905c21a2 100644 --- a/vendor/github.com/BurntSushi/toml/encode.go +++ b/vendor/github.com/BurntSushi/toml/encode.go @@ -16,17 +16,17 @@ type tomlEncodeError struct{ error } var ( errArrayMixedElementTypes = errors.New( - "can't encode array with mixed element types") + "toml: cannot encode array with mixed element types") errArrayNilElement = errors.New( - "can't encode array with nil element") + "toml: cannot encode array with nil element") errNonString = errors.New( - "can't encode a map with non-string key type") + "toml: cannot encode a map with non-string key type") errAnonNonStruct = errors.New( - "can't encode an anonymous field that is not a struct") + "toml: cannot encode an anonymous field that is not a struct") errArrayNoTable = errors.New( - "TOML array element can't contain a table") + "toml: TOML array element cannot contain a table") errNoKey = errors.New( - "top-level values must be a Go map or struct") + "toml: top-level values must be Go maps or structs") errAnything = errors.New("") // used in testing ) @@ -148,7 +148,7 @@ func (enc *Encoder) encode(key Key, rv reflect.Value) { case reflect.Struct: enc.eTable(key, rv) default: - panic(e("Unsupported type for key '%s': %s", key, k)) + panic(e("unsupported type for key '%s': %s", key, k)) } } @@ -160,7 +160,7 @@ func (enc *Encoder) eElement(rv reflect.Value) { // Special case time.Time as a primitive. Has to come before // TextMarshaler below because time.Time implements // encoding.TextMarshaler, but we need to always use UTC. - enc.wf(v.In(time.FixedZone("UTC", 0)).Format("2006-01-02T15:04:05Z")) + enc.wf(v.UTC().Format("2006-01-02T15:04:05Z")) return case TextMarshaler: // Special case. Use text marshaler if it's available for this value. @@ -191,7 +191,7 @@ func (enc *Encoder) eElement(rv reflect.Value) { case reflect.String: enc.writeQuoted(rv.String()) default: - panic(e("Unexpected primitive type: %s", rv.Kind())) + panic(e("unexpected primitive type: %s", rv.Kind())) } } @@ -241,7 +241,7 @@ func (enc *Encoder) eArrayOfTables(key Key, rv reflect.Value) { func (enc *Encoder) eTable(key Key, rv reflect.Value) { panicIfInvalidKey(key) if len(key) == 1 { - // Output an extra new line between top-level tables. + // Output an extra newline between top-level tables. // (The newline isn't written if nothing else has been written though.) enc.newline() } @@ -315,10 +315,16 @@ func (enc *Encoder) eStruct(key Key, rv reflect.Value) { t := f.Type switch t.Kind() { case reflect.Struct: - addFields(t, frv, f.Index) - continue + // Treat anonymous struct fields with + // tag names as though they are not + // anonymous, like encoding/json does. + if getOptions(f.Tag).name == "" { + addFields(t, frv, f.Index) + continue + } case reflect.Ptr: - if t.Elem().Kind() == reflect.Struct { + if t.Elem().Kind() == reflect.Struct && + getOptions(f.Tag).name == "" { if !frv.IsNil() { addFields(t.Elem(), frv.Elem(), f.Index) } @@ -347,17 +353,18 @@ func (enc *Encoder) eStruct(key Key, rv reflect.Value) { continue } - tag := sft.Tag.Get("toml") - if tag == "-" { + opts := getOptions(sft.Tag) + if opts.skip { continue } - keyName, opts := getOptions(tag) - if keyName == "" { - keyName = sft.Name + keyName := sft.Name + if opts.name != "" { + keyName = opts.name } - if _, ok := opts["omitempty"]; ok && isEmpty(sf) { + if opts.omitempty && isEmpty(sf) { continue - } else if _, ok := opts["omitzero"]; ok && isZero(sf) { + } + if opts.omitzero && isZero(sf) { continue } @@ -392,9 +399,8 @@ func tomlTypeOfGo(rv reflect.Value) tomlType { case reflect.Array, reflect.Slice: if typeEqual(tomlHash, tomlArrayType(rv)) { return tomlArrayHash - } else { - return tomlArray } + return tomlArray case reflect.Ptr, reflect.Interface: return tomlTypeOfGo(rv.Elem()) case reflect.String: @@ -451,17 +457,30 @@ func tomlArrayType(rv reflect.Value) tomlType { return firstType } -func getOptions(keyName string) (string, map[string]struct{}) { - opts := make(map[string]struct{}) - ss := strings.Split(keyName, ",") - name := ss[0] - if len(ss) > 1 { - for _, opt := range ss { - opts[opt] = struct{}{} +type tagOptions struct { + skip bool // "-" + name string + omitempty bool + omitzero bool +} + +func getOptions(tag reflect.StructTag) tagOptions { + t := tag.Get("toml") + if t == "-" { + return tagOptions{skip: true} + } + var opts tagOptions + parts := strings.Split(t, ",") + opts.name = parts[0] + for _, s := range parts[1:] { + switch s { + case "omitempty": + opts.omitempty = true + case "omitzero": + opts.omitzero = true } } - - return name, opts + return opts } func isZero(rv reflect.Value) bool { diff --git a/vendor/github.com/BurntSushi/toml/lex.go b/vendor/github.com/BurntSushi/toml/lex.go index 9b20b3a81..e0a742a88 100644 --- a/vendor/github.com/BurntSushi/toml/lex.go +++ b/vendor/github.com/BurntSushi/toml/lex.go @@ -3,6 +3,7 @@ package toml import ( "fmt" "strings" + "unicode" "unicode/utf8" ) @@ -29,24 +30,28 @@ const ( itemArrayTableEnd itemKeyStart itemCommentStart + itemInlineTableStart + itemInlineTableEnd ) const ( - eof = 0 - tableStart = '[' - tableEnd = ']' - arrayTableStart = '[' - arrayTableEnd = ']' - tableSep = '.' - keySep = '=' - arrayStart = '[' - arrayEnd = ']' - arrayValTerm = ',' - commentStart = '#' - stringStart = '"' - stringEnd = '"' - rawStringStart = '\'' - rawStringEnd = '\'' + eof = 0 + comma = ',' + tableStart = '[' + tableEnd = ']' + arrayTableStart = '[' + arrayTableEnd = ']' + tableSep = '.' + keySep = '=' + arrayStart = '[' + arrayEnd = ']' + commentStart = '#' + stringStart = '"' + stringEnd = '"' + rawStringStart = '\'' + rawStringEnd = '\'' + inlineTableStart = '{' + inlineTableEnd = '}' ) type stateFn func(lx *lexer) stateFn @@ -55,11 +60,18 @@ type lexer struct { input string start int pos int - width int line int state stateFn items chan item + // Allow for backing up up to three runes. + // This is necessary because TOML contains 3-rune tokens (""" and '''). + prevWidths [3]int + nprev int // how many of prevWidths are in use + // If we emit an eof, we can still back up, but it is not OK to call + // next again. + atEOF bool + // A stack of state functions used to maintain context. // The idea is to reuse parts of the state machine in various places. // For example, values can appear at the top level or within arbitrarily @@ -87,7 +99,7 @@ func (lx *lexer) nextItem() item { func lex(input string) *lexer { lx := &lexer{ - input: input + "\n", + input: input, state: lexTop, line: 1, items: make(chan item, 10), @@ -102,7 +114,7 @@ func (lx *lexer) push(state stateFn) { func (lx *lexer) pop() stateFn { if len(lx.stack) == 0 { - return lx.errorf("BUG in lexer: no states to pop.") + return lx.errorf("BUG in lexer: no states to pop") } last := lx.stack[len(lx.stack)-1] lx.stack = lx.stack[0 : len(lx.stack)-1] @@ -124,16 +136,25 @@ func (lx *lexer) emitTrim(typ itemType) { } func (lx *lexer) next() (r rune) { + if lx.atEOF { + panic("next called after EOF") + } if lx.pos >= len(lx.input) { - lx.width = 0 + lx.atEOF = true return eof } if lx.input[lx.pos] == '\n' { lx.line++ } - r, lx.width = utf8.DecodeRuneInString(lx.input[lx.pos:]) - lx.pos += lx.width + lx.prevWidths[2] = lx.prevWidths[1] + lx.prevWidths[1] = lx.prevWidths[0] + if lx.nprev < 3 { + lx.nprev++ + } + r, w := utf8.DecodeRuneInString(lx.input[lx.pos:]) + lx.prevWidths[0] = w + lx.pos += w return r } @@ -142,9 +163,20 @@ func (lx *lexer) ignore() { lx.start = lx.pos } -// backup steps back one rune. Can be called only once per call of next. +// backup steps back one rune. Can be called only twice between calls to next. func (lx *lexer) backup() { - lx.pos -= lx.width + if lx.atEOF { + lx.atEOF = false + return + } + if lx.nprev < 1 { + panic("backed up too far") + } + w := lx.prevWidths[0] + lx.prevWidths[0] = lx.prevWidths[1] + lx.prevWidths[1] = lx.prevWidths[2] + lx.nprev-- + lx.pos -= w if lx.pos < len(lx.input) && lx.input[lx.pos] == '\n' { lx.line-- } @@ -166,9 +198,22 @@ func (lx *lexer) peek() rune { return r } +// skip ignores all input that matches the given predicate. +func (lx *lexer) skip(pred func(rune) bool) { + for { + r := lx.next() + if pred(r) { + continue + } + lx.backup() + lx.ignore() + return + } +} + // errorf stops all lexing by emitting an error and returning `nil`. // Note that any value that is a character is escaped if it's a special -// character (new lines, tabs, etc.). +// character (newlines, tabs, etc.). func (lx *lexer) errorf(format string, values ...interface{}) stateFn { lx.items <- item{ itemError, @@ -184,7 +229,6 @@ func lexTop(lx *lexer) stateFn { if isWhitespace(r) || isNL(r) { return lexSkip(lx, lexTop) } - switch r { case commentStart: lx.push(lexTop) @@ -193,7 +237,7 @@ func lexTop(lx *lexer) stateFn { return lexTableStart case eof: if lx.pos > lx.start { - return lx.errorf("Unexpected EOF.") + return lx.errorf("unexpected EOF") } lx.emit(itemEOF) return nil @@ -208,12 +252,12 @@ func lexTop(lx *lexer) stateFn { // lexTopEnd is entered whenever a top-level item has been consumed. (A value // or a table.) It must see only whitespace, and will turn back to lexTop -// upon a new line. If it sees EOF, it will quit the lexer successfully. +// upon a newline. If it sees EOF, it will quit the lexer successfully. func lexTopEnd(lx *lexer) stateFn { r := lx.next() switch { case r == commentStart: - // a comment will read to a new line for us. + // a comment will read to a newline for us. lx.push(lexTop) return lexCommentStart case isWhitespace(r): @@ -222,11 +266,11 @@ func lexTopEnd(lx *lexer) stateFn { lx.ignore() return lexTop case r == eof: - lx.ignore() - return lexTop + lx.emit(itemEOF) + return nil } - return lx.errorf("Expected a top-level item to end with a new line, "+ - "comment or EOF, but got %q instead.", r) + return lx.errorf("expected a top-level item to end with a newline, "+ + "comment, or EOF, but got %q instead", r) } // lexTable lexes the beginning of a table. Namely, it makes sure that @@ -253,21 +297,22 @@ func lexTableEnd(lx *lexer) stateFn { func lexArrayTableEnd(lx *lexer) stateFn { if r := lx.next(); r != arrayTableEnd { - return lx.errorf("Expected end of table array name delimiter %q, "+ - "but got %q instead.", arrayTableEnd, r) + return lx.errorf("expected end of table array name delimiter %q, "+ + "but got %q instead", arrayTableEnd, r) } lx.emit(itemArrayTableEnd) return lexTopEnd } func lexTableNameStart(lx *lexer) stateFn { + lx.skip(isWhitespace) switch r := lx.peek(); { case r == tableEnd || r == eof: - return lx.errorf("Unexpected end of table name. (Table names cannot " + - "be empty.)") + return lx.errorf("unexpected end of table name " + + "(table names cannot be empty)") case r == tableSep: - return lx.errorf("Unexpected table separator. (Table names cannot " + - "be empty.)") + return lx.errorf("unexpected table separator " + + "(table names cannot be empty)") case r == stringStart || r == rawStringStart: lx.ignore() lx.push(lexTableNameEnd) @@ -277,24 +322,22 @@ func lexTableNameStart(lx *lexer) stateFn { } } -// lexTableName lexes the name of a table. It assumes that at least one +// lexBareTableName lexes the name of a table. It assumes that at least one // valid character for the table has already been read. func lexBareTableName(lx *lexer) stateFn { - switch r := lx.next(); { - case isBareKeyChar(r): + r := lx.next() + if isBareKeyChar(r) { return lexBareTableName - case r == tableSep || r == tableEnd: - lx.backup() - lx.emitTrim(itemText) - return lexTableNameEnd - default: - return lx.errorf("Bare keys cannot contain %q.", r) } + lx.backup() + lx.emit(itemText) + return lexTableNameEnd } // lexTableNameEnd reads the end of a piece of a table name, optionally // consuming whitespace. func lexTableNameEnd(lx *lexer) stateFn { + lx.skip(isWhitespace) switch r := lx.next(); { case isWhitespace(r): return lexTableNameEnd @@ -304,8 +347,8 @@ func lexTableNameEnd(lx *lexer) stateFn { case r == tableEnd: return lx.pop() default: - return lx.errorf("Expected '.' or ']' to end table name, but got %q "+ - "instead.", r) + return lx.errorf("expected '.' or ']' to end table name, "+ + "but got %q instead", r) } } @@ -315,7 +358,7 @@ func lexKeyStart(lx *lexer) stateFn { r := lx.peek() switch { case r == keySep: - return lx.errorf("Unexpected key separator %q.", keySep) + return lx.errorf("unexpected key separator %q", keySep) case isWhitespace(r) || isNL(r): lx.next() return lexSkip(lx, lexKeyStart) @@ -338,14 +381,15 @@ func lexBareKey(lx *lexer) stateFn { case isBareKeyChar(r): return lexBareKey case isWhitespace(r): - lx.emitTrim(itemText) + lx.backup() + lx.emit(itemText) return lexKeyEnd case r == keySep: lx.backup() - lx.emitTrim(itemText) + lx.emit(itemText) return lexKeyEnd default: - return lx.errorf("Bare keys cannot contain %q.", r) + return lx.errorf("bare keys cannot contain %q", r) } } @@ -358,7 +402,7 @@ func lexKeyEnd(lx *lexer) stateFn { case isWhitespace(r): return lexSkip(lx, lexKeyEnd) default: - return lx.errorf("Expected key separator %q, but got %q instead.", + return lx.errorf("expected key separator %q, but got %q instead", keySep, r) } } @@ -367,20 +411,26 @@ func lexKeyEnd(lx *lexer) stateFn { // lexValue will ignore whitespace. // After a value is lexed, the last state on the next is popped and returned. func lexValue(lx *lexer) stateFn { - // We allow whitespace to precede a value, but NOT new lines. - // In array syntax, the array states are responsible for ignoring new - // lines. + // We allow whitespace to precede a value, but NOT newlines. + // In array syntax, the array states are responsible for ignoring newlines. r := lx.next() - if isWhitespace(r) { + switch { + case isWhitespace(r): return lexSkip(lx, lexValue) + case isDigit(r): + lx.backup() // avoid an extra state and use the same as above + return lexNumberOrDateStart } - - switch { - case r == arrayStart: + switch r { + case arrayStart: lx.ignore() lx.emit(itemArray) return lexArrayValue - case r == stringStart: + case inlineTableStart: + lx.ignore() + lx.emit(itemInlineTableStart) + return lexInlineTableValue + case stringStart: if lx.accept(stringStart) { if lx.accept(stringStart) { lx.ignore() // Ignore """ @@ -390,7 +440,7 @@ func lexValue(lx *lexer) stateFn { } lx.ignore() // ignore the '"' return lexString - case r == rawStringStart: + case rawStringStart: if lx.accept(rawStringStart) { if lx.accept(rawStringStart) { lx.ignore() // Ignore """ @@ -400,23 +450,24 @@ func lexValue(lx *lexer) stateFn { } lx.ignore() // ignore the "'" return lexRawString - case r == 't': - return lexTrue - case r == 'f': - return lexFalse - case r == '-': + case '+', '-': return lexNumberStart - case isDigit(r): - lx.backup() // avoid an extra state and use the same as above - return lexNumberOrDateStart - case r == '.': // special error case, be kind to users - return lx.errorf("Floats must start with a digit, not '.'.") + case '.': // special error case, be kind to users + return lx.errorf("floats must start with a digit, not '.'") + } + if unicode.IsLetter(r) { + // Be permissive here; lexBool will give a nice error if the + // user wrote something like + // x = foo + // (i.e. not 'true' or 'false' but is something else word-like.) + lx.backup() + return lexBool } - return lx.errorf("Expected value but found %q instead.", r) + return lx.errorf("expected value but found %q instead", r) } // lexArrayValue consumes one value in an array. It assumes that '[' or ',' -// have already been consumed. All whitespace and new lines are ignored. +// have already been consumed. All whitespace and newlines are ignored. func lexArrayValue(lx *lexer) stateFn { r := lx.next() switch { @@ -425,10 +476,11 @@ func lexArrayValue(lx *lexer) stateFn { case r == commentStart: lx.push(lexArrayValue) return lexCommentStart - case r == arrayValTerm: - return lx.errorf("Unexpected array value terminator %q.", - arrayValTerm) + case r == comma: + return lx.errorf("unexpected comma") case r == arrayEnd: + // NOTE(caleb): The spec isn't clear about whether you can have + // a trailing comma or not, so we'll allow it. return lexArrayEnd } @@ -437,8 +489,9 @@ func lexArrayValue(lx *lexer) stateFn { return lexValue } -// lexArrayValueEnd consumes the cruft between values of an array. Namely, -// it ignores whitespace and expects either a ',' or a ']'. +// lexArrayValueEnd consumes everything between the end of an array value and +// the next value (or the end of the array): it ignores whitespace and newlines +// and expects either a ',' or a ']'. func lexArrayValueEnd(lx *lexer) stateFn { r := lx.next() switch { @@ -447,31 +500,88 @@ func lexArrayValueEnd(lx *lexer) stateFn { case r == commentStart: lx.push(lexArrayValueEnd) return lexCommentStart - case r == arrayValTerm: + case r == comma: lx.ignore() return lexArrayValue // move on to the next value case r == arrayEnd: return lexArrayEnd } - return lx.errorf("Expected an array value terminator %q or an array "+ - "terminator %q, but got %q instead.", arrayValTerm, arrayEnd, r) + return lx.errorf( + "expected a comma or array terminator %q, but got %q instead", + arrayEnd, r, + ) } -// lexArrayEnd finishes the lexing of an array. It assumes that a ']' has -// just been consumed. +// lexArrayEnd finishes the lexing of an array. +// It assumes that a ']' has just been consumed. func lexArrayEnd(lx *lexer) stateFn { lx.ignore() lx.emit(itemArrayEnd) return lx.pop() } +// lexInlineTableValue consumes one key/value pair in an inline table. +// It assumes that '{' or ',' have already been consumed. Whitespace is ignored. +func lexInlineTableValue(lx *lexer) stateFn { + r := lx.next() + switch { + case isWhitespace(r): + return lexSkip(lx, lexInlineTableValue) + case isNL(r): + return lx.errorf("newlines not allowed within inline tables") + case r == commentStart: + lx.push(lexInlineTableValue) + return lexCommentStart + case r == comma: + return lx.errorf("unexpected comma") + case r == inlineTableEnd: + return lexInlineTableEnd + } + lx.backup() + lx.push(lexInlineTableValueEnd) + return lexKeyStart +} + +// lexInlineTableValueEnd consumes everything between the end of an inline table +// key/value pair and the next pair (or the end of the table): +// it ignores whitespace and expects either a ',' or a '}'. +func lexInlineTableValueEnd(lx *lexer) stateFn { + r := lx.next() + switch { + case isWhitespace(r): + return lexSkip(lx, lexInlineTableValueEnd) + case isNL(r): + return lx.errorf("newlines not allowed within inline tables") + case r == commentStart: + lx.push(lexInlineTableValueEnd) + return lexCommentStart + case r == comma: + lx.ignore() + return lexInlineTableValue + case r == inlineTableEnd: + return lexInlineTableEnd + } + return lx.errorf("expected a comma or an inline table terminator %q, "+ + "but got %q instead", inlineTableEnd, r) +} + +// lexInlineTableEnd finishes the lexing of an inline table. +// It assumes that a '}' has just been consumed. +func lexInlineTableEnd(lx *lexer) stateFn { + lx.ignore() + lx.emit(itemInlineTableEnd) + return lx.pop() +} + // lexString consumes the inner contents of a string. It assumes that the // beginning '"' has already been consumed and ignored. func lexString(lx *lexer) stateFn { r := lx.next() switch { + case r == eof: + return lx.errorf("unexpected EOF") case isNL(r): - return lx.errorf("Strings cannot contain new lines.") + return lx.errorf("strings cannot contain newlines") case r == '\\': lx.push(lexString) return lexStringEscape @@ -488,11 +598,12 @@ func lexString(lx *lexer) stateFn { // lexMultilineString consumes the inner contents of a string. It assumes that // the beginning '"""' has already been consumed and ignored. func lexMultilineString(lx *lexer) stateFn { - r := lx.next() - switch { - case r == '\\': + switch lx.next() { + case eof: + return lx.errorf("unexpected EOF") + case '\\': return lexMultilineStringEscape - case r == stringEnd: + case stringEnd: if lx.accept(stringEnd) { if lx.accept(stringEnd) { lx.backup() @@ -516,8 +627,10 @@ func lexMultilineString(lx *lexer) stateFn { func lexRawString(lx *lexer) stateFn { r := lx.next() switch { + case r == eof: + return lx.errorf("unexpected EOF") case isNL(r): - return lx.errorf("Strings cannot contain new lines.") + return lx.errorf("strings cannot contain newlines") case r == rawStringEnd: lx.backup() lx.emit(itemRawString) @@ -529,12 +642,13 @@ func lexRawString(lx *lexer) stateFn { } // lexMultilineRawString consumes a raw string. Nothing can be escaped in such -// a string. It assumes that the beginning "'" has already been consumed and +// a string. It assumes that the beginning "'''" has already been consumed and // ignored. func lexMultilineRawString(lx *lexer) stateFn { - r := lx.next() - switch { - case r == rawStringEnd: + switch lx.next() { + case eof: + return lx.errorf("unexpected EOF") + case rawStringEnd: if lx.accept(rawStringEnd) { if lx.accept(rawStringEnd) { lx.backup() @@ -559,11 +673,10 @@ func lexMultilineStringEscape(lx *lexer) stateFn { // Handle the special case first: if isNL(lx.next()) { return lexMultilineString - } else { - lx.backup() - lx.push(lexMultilineString) - return lexStringEscape(lx) } + lx.backup() + lx.push(lexMultilineString) + return lexStringEscape(lx) } func lexStringEscape(lx *lexer) stateFn { @@ -588,10 +701,9 @@ func lexStringEscape(lx *lexer) stateFn { case 'U': return lexLongUnicodeEscape } - return lx.errorf("Invalid escape character %q. Only the following "+ + return lx.errorf("invalid escape character %q; only the following "+ "escape characters are allowed: "+ - "\\b, \\t, \\n, \\f, \\r, \\\", \\/, \\\\, "+ - "\\uXXXX and \\UXXXXXXXX.", r) + `\b, \t, \n, \f, \r, \", \\, \uXXXX, and \UXXXXXXXX`, r) } func lexShortUnicodeEscape(lx *lexer) stateFn { @@ -599,8 +711,8 @@ func lexShortUnicodeEscape(lx *lexer) stateFn { for i := 0; i < 4; i++ { r = lx.next() if !isHexadecimal(r) { - return lx.errorf("Expected four hexadecimal digits after '\\u', "+ - "but got '%s' instead.", lx.current()) + return lx.errorf(`expected four hexadecimal digits after '\u', `+ + "but got %q instead", lx.current()) } } return lx.pop() @@ -611,40 +723,43 @@ func lexLongUnicodeEscape(lx *lexer) stateFn { for i := 0; i < 8; i++ { r = lx.next() if !isHexadecimal(r) { - return lx.errorf("Expected eight hexadecimal digits after '\\U', "+ - "but got '%s' instead.", lx.current()) + return lx.errorf(`expected eight hexadecimal digits after '\U', `+ + "but got %q instead", lx.current()) } } return lx.pop() } -// lexNumberOrDateStart consumes either a (positive) integer, float or -// datetime. It assumes that NO negative sign has been consumed. +// lexNumberOrDateStart consumes either an integer, a float, or datetime. func lexNumberOrDateStart(lx *lexer) stateFn { r := lx.next() - if !isDigit(r) { - if r == '.' { - return lx.errorf("Floats must start with a digit, not '.'.") - } else { - return lx.errorf("Expected a digit but got %q.", r) - } + if isDigit(r) { + return lexNumberOrDate } - return lexNumberOrDate + switch r { + case '_': + return lexNumber + case 'e', 'E': + return lexFloat + case '.': + return lx.errorf("floats must start with a digit, not '.'") + } + return lx.errorf("expected a digit but got %q", r) } -// lexNumberOrDate consumes either a (positive) integer, float or datetime. +// lexNumberOrDate consumes either an integer, float or datetime. func lexNumberOrDate(lx *lexer) stateFn { r := lx.next() - switch { - case r == '-': - if lx.pos-lx.start != 5 { - return lx.errorf("All ISO8601 dates must be in full Zulu form.") - } - return lexDateAfterYear - case isDigit(r): + if isDigit(r) { return lexNumberOrDate - case r == '.': - return lexFloatStart + } + switch r { + case '-': + return lexDatetime + case '_': + return lexNumber + case '.', 'e', 'E': + return lexFloat } lx.backup() @@ -652,46 +767,34 @@ func lexNumberOrDate(lx *lexer) stateFn { return lx.pop() } -// lexDateAfterYear consumes a full Zulu Datetime in ISO8601 format. -// It assumes that "YYYY-" has already been consumed. -func lexDateAfterYear(lx *lexer) stateFn { - formats := []rune{ - // digits are '0'. - // everything else is direct equality. - '0', '0', '-', '0', '0', - 'T', - '0', '0', ':', '0', '0', ':', '0', '0', - 'Z', +// lexDatetime consumes a Datetime, to a first approximation. +// The parser validates that it matches one of the accepted formats. +func lexDatetime(lx *lexer) stateFn { + r := lx.next() + if isDigit(r) { + return lexDatetime } - for _, f := range formats { - r := lx.next() - if f == '0' { - if !isDigit(r) { - return lx.errorf("Expected digit in ISO8601 datetime, "+ - "but found %q instead.", r) - } - } else if f != r { - return lx.errorf("Expected %q in ISO8601 datetime, "+ - "but found %q instead.", f, r) - } + switch r { + case '-', 'T', ':', '.', 'Z', '+': + return lexDatetime } + + lx.backup() lx.emit(itemDatetime) return lx.pop() } -// lexNumberStart consumes either an integer or a float. It assumes that -// a negative sign has already been read, but that *no* digits have been -// consumed. lexNumberStart will move to the appropriate integer or float -// states. +// lexNumberStart consumes either an integer or a float. It assumes that a sign +// has already been read, but that *no* digits have been consumed. +// lexNumberStart will move to the appropriate integer or float states. func lexNumberStart(lx *lexer) stateFn { - // we MUST see a digit. Even floats have to start with a digit. + // We MUST see a digit. Even floats have to start with a digit. r := lx.next() if !isDigit(r) { if r == '.' { - return lx.errorf("Floats must start with a digit, not '.'.") - } else { - return lx.errorf("Expected a digit but got %q.", r) + return lx.errorf("floats must start with a digit, not '.'") } + return lx.errorf("expected a digit but got %q", r) } return lexNumber } @@ -699,11 +802,14 @@ func lexNumberStart(lx *lexer) stateFn { // lexNumber consumes an integer or a float after seeing the first digit. func lexNumber(lx *lexer) stateFn { r := lx.next() - switch { - case isDigit(r): + if isDigit(r) { return lexNumber - case r == '.': - return lexFloatStart + } + switch r { + case '_': + return lexNumber + case '.', 'e', 'E': + return lexFloat } lx.backup() @@ -711,60 +817,42 @@ func lexNumber(lx *lexer) stateFn { return lx.pop() } -// lexFloatStart starts the consumption of digits of a float after a '.'. -// Namely, at least one digit is required. -func lexFloatStart(lx *lexer) stateFn { - r := lx.next() - if !isDigit(r) { - return lx.errorf("Floats must have a digit after the '.', but got "+ - "%q instead.", r) - } - return lexFloat -} - -// lexFloat consumes the digits of a float after a '.'. -// Assumes that one digit has been consumed after a '.' already. +// lexFloat consumes the elements of a float. It allows any sequence of +// float-like characters, so floats emitted by the lexer are only a first +// approximation and must be validated by the parser. func lexFloat(lx *lexer) stateFn { r := lx.next() if isDigit(r) { return lexFloat } + switch r { + case '_', '.', '-', '+', 'e', 'E': + return lexFloat + } lx.backup() lx.emit(itemFloat) return lx.pop() } -// lexConst consumes the s[1:] in s. It assumes that s[0] has already been -// consumed. -func lexConst(lx *lexer, s string) stateFn { - for i := range s[1:] { - if r := lx.next(); r != rune(s[i+1]) { - return lx.errorf("Expected %q, but found %q instead.", s[:i+1], - s[:i]+string(r)) +// lexBool consumes a bool string: 'true' or 'false. +func lexBool(lx *lexer) stateFn { + var rs []rune + for { + r := lx.next() + if !unicode.IsLetter(r) { + lx.backup() + break } + rs = append(rs, r) } - return nil -} - -// lexTrue consumes the "rue" in "true". It assumes that 't' has already -// been consumed. -func lexTrue(lx *lexer) stateFn { - if fn := lexConst(lx, "true"); fn != nil { - return fn - } - lx.emit(itemBool) - return lx.pop() -} - -// lexFalse consumes the "alse" in "false". It assumes that 'f' has already -// been consumed. -func lexFalse(lx *lexer) stateFn { - if fn := lexConst(lx, "false"); fn != nil { - return fn + s := string(rs) + switch s { + case "true", "false": + lx.emit(itemBool) + return lx.pop() } - lx.emit(itemBool) - return lx.pop() + return lx.errorf("expected value but found %q instead", s) } // lexCommentStart begins the lexing of a comment. It will emit @@ -776,7 +864,7 @@ func lexCommentStart(lx *lexer) stateFn { } // lexComment lexes an entire comment. It assumes that '#' has been consumed. -// It will consume *up to* the first new line character, and pass control +// It will consume *up to* the first newline character, and pass control // back to the last state on the stack. func lexComment(lx *lexer) stateFn { r := lx.peek() @@ -834,13 +922,7 @@ func (itype itemType) String() string { return "EOF" case itemText: return "Text" - case itemString: - return "String" - case itemRawString: - return "String" - case itemMultilineString: - return "String" - case itemRawMultilineString: + case itemString, itemRawString, itemMultilineString, itemRawMultilineString: return "String" case itemBool: return "Bool" diff --git a/vendor/github.com/BurntSushi/toml/parse.go b/vendor/github.com/BurntSushi/toml/parse.go index 6a82e84f6..50869ef92 100644 --- a/vendor/github.com/BurntSushi/toml/parse.go +++ b/vendor/github.com/BurntSushi/toml/parse.go @@ -2,7 +2,6 @@ package toml import ( "fmt" - "log" "strconv" "strings" "time" @@ -81,7 +80,7 @@ func (p *parser) next() item { } func (p *parser) bug(format string, v ...interface{}) { - log.Panicf("BUG: %s\n\n", fmt.Sprintf(format, v...)) + panic(fmt.Sprintf("BUG: "+format+"\n\n", v...)) } func (p *parser) expect(typ itemType) item { @@ -179,10 +178,18 @@ func (p *parser) value(it item) (interface{}, tomlType) { } p.bug("Expected boolean value, but got '%s'.", it.val) case itemInteger: - num, err := strconv.ParseInt(it.val, 10, 64) + if !numUnderscoresOK(it.val) { + p.panicf("Invalid integer %q: underscores must be surrounded by digits", + it.val) + } + val := strings.Replace(it.val, "_", "", -1) + num, err := strconv.ParseInt(val, 10, 64) if err != nil { - // See comment below for floats describing why we make a - // distinction between a bug and a user error. + // Distinguish integer values. Normally, it'd be a bug if the lexer + // provides an invalid integer, but it's possible that the number is + // out of range of valid values (which the lexer cannot determine). + // So mark the former as a bug but the latter as a legitimate user + // error. if e, ok := err.(*strconv.NumError); ok && e.Err == strconv.ErrRange { @@ -194,29 +201,57 @@ func (p *parser) value(it item) (interface{}, tomlType) { } return num, p.typeOfPrimitive(it) case itemFloat: - num, err := strconv.ParseFloat(it.val, 64) + parts := strings.FieldsFunc(it.val, func(r rune) bool { + switch r { + case '.', 'e', 'E': + return true + } + return false + }) + for _, part := range parts { + if !numUnderscoresOK(part) { + p.panicf("Invalid float %q: underscores must be "+ + "surrounded by digits", it.val) + } + } + if !numPeriodsOK(it.val) { + // As a special case, numbers like '123.' or '1.e2', + // which are valid as far as Go/strconv are concerned, + // must be rejected because TOML says that a fractional + // part consists of '.' followed by 1+ digits. + p.panicf("Invalid float %q: '.' must be followed "+ + "by one or more digits", it.val) + } + val := strings.Replace(it.val, "_", "", -1) + num, err := strconv.ParseFloat(val, 64) if err != nil { - // Distinguish float values. Normally, it'd be a bug if the lexer - // provides an invalid float, but it's possible that the float is - // out of range of valid values (which the lexer cannot determine). - // So mark the former as a bug but the latter as a legitimate user - // error. - // - // This is also true for integers. if e, ok := err.(*strconv.NumError); ok && e.Err == strconv.ErrRange { p.panicf("Float '%s' is out of the range of 64-bit "+ "IEEE-754 floating-point numbers.", it.val) } else { - p.bug("Expected float value, but got '%s'.", it.val) + p.panicf("Invalid float value: %q", it.val) } } return num, p.typeOfPrimitive(it) case itemDatetime: - t, err := time.Parse("2006-01-02T15:04:05Z", it.val) - if err != nil { - p.panicf("Invalid RFC3339 Zulu DateTime: '%s'.", it.val) + var t time.Time + var ok bool + var err error + for _, format := range []string{ + "2006-01-02T15:04:05Z07:00", + "2006-01-02T15:04:05", + "2006-01-02", + } { + t, err = time.ParseInLocation(format, it.val, time.Local) + if err == nil { + ok = true + break + } + } + if !ok { + p.panicf("Invalid TOML Datetime: %q.", it.val) } return t, p.typeOfPrimitive(it) case itemArray: @@ -234,11 +269,75 @@ func (p *parser) value(it item) (interface{}, tomlType) { types = append(types, typ) } return array, p.typeOfArray(types) + case itemInlineTableStart: + var ( + hash = make(map[string]interface{}) + outerContext = p.context + outerKey = p.currentKey + ) + + p.context = append(p.context, p.currentKey) + p.currentKey = "" + for it := p.next(); it.typ != itemInlineTableEnd; it = p.next() { + if it.typ != itemKeyStart { + p.bug("Expected key start but instead found %q, around line %d", + it.val, p.approxLine) + } + if it.typ == itemCommentStart { + p.expect(itemText) + continue + } + + // retrieve key + k := p.next() + p.approxLine = k.line + kname := p.keyString(k) + + // retrieve value + p.currentKey = kname + val, typ := p.value(p.next()) + // make sure we keep metadata up to date + p.setType(kname, typ) + p.ordered = append(p.ordered, p.context.add(p.currentKey)) + hash[kname] = val + } + p.context = outerContext + p.currentKey = outerKey + return hash, tomlHash } p.bug("Unexpected value type: %s", it.typ) panic("unreachable") } +// numUnderscoresOK checks whether each underscore in s is surrounded by +// characters that are not underscores. +func numUnderscoresOK(s string) bool { + accept := false + for _, r := range s { + if r == '_' { + if !accept { + return false + } + accept = false + continue + } + accept = true + } + return accept +} + +// numPeriodsOK checks whether every period in s is followed by a digit. +func numPeriodsOK(s string) bool { + period := false + for _, r := range s { + if period && !isDigit(r) { + return false + } + period = r == '.' + } + return !period +} + // establishContext sets the current context of the parser, // where the context is either a hash or an array of hashes. Which one is // set depends on the value of the `array` parameter. diff --git a/vendor/github.com/BurntSushi/toml/type_fields.go b/vendor/github.com/BurntSushi/toml/type_fields.go index 6da608af4..608997c22 100644 --- a/vendor/github.com/BurntSushi/toml/type_fields.go +++ b/vendor/github.com/BurntSushi/toml/type_fields.go @@ -95,8 +95,8 @@ func typeFields(t reflect.Type) []field { if sf.PkgPath != "" && !sf.Anonymous { // unexported continue } - name, _ := getOptions(sf.Tag.Get("toml")) - if name == "-" { + opts := getOptions(sf.Tag) + if opts.skip { continue } index := make([]int, len(f.index)+1) @@ -110,8 +110,9 @@ func typeFields(t reflect.Type) []field { } // Record found field and index sequence. - if name != "" || !sf.Anonymous || ft.Kind() != reflect.Struct { - tagged := name != "" + if opts.name != "" || !sf.Anonymous || ft.Kind() != reflect.Struct { + tagged := opts.name != "" + name := opts.name if name == "" { name = sf.Name } diff --git a/vendor/github.com/containers/buildah/add.go b/vendor/github.com/containers/buildah/add.go index 589e090a8..11bfb6a12 100644 --- a/vendor/github.com/containers/buildah/add.go +++ b/vendor/github.com/containers/buildah/add.go @@ -14,6 +14,7 @@ import ( "github.com/containers/buildah/pkg/chrootuser" "github.com/containers/buildah/util" "github.com/containers/storage/pkg/archive" + "github.com/containers/storage/pkg/fileutils" "github.com/containers/storage/pkg/idtools" "github.com/containers/storage/pkg/system" "github.com/opencontainers/runtime-spec/specs-go" @@ -89,7 +90,10 @@ func addURL(destination, srcurl string, owner idtools.IDPair, hasher io.Writer) // filesystem, optionally extracting contents of local files that look like // non-empty archives. func (b *Builder) Add(destination string, extract bool, options AddAndCopyOptions, source ...string) error { - excludes := dockerIgnoreHelper(options.Excludes, options.ContextDir) + excludes, err := dockerIgnoreMatcher(options.Excludes, options.ContextDir) + if err != nil { + return err + } mountPoint, err := b.Mount(b.MountLabel) if err != nil { return err @@ -100,7 +104,7 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption } }() // Find out which user (and group) the destination should belong to. - user, err := b.user(mountPoint, options.Chown) + user, _, err := b.user(mountPoint, options.Chown) if err != nil { return err } @@ -153,12 +157,12 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption } // user returns the user (and group) information which the destination should belong to. -func (b *Builder) user(mountPoint string, userspec string) (specs.User, error) { +func (b *Builder) user(mountPoint string, userspec string) (specs.User, string, error) { if userspec == "" { userspec = b.User() } - uid, gid, err := chrootuser.GetUser(mountPoint, userspec) + uid, gid, homeDir, err := chrootuser.GetUser(mountPoint, userspec) u := specs.User{ UID: uid, GID: gid, @@ -175,45 +179,48 @@ func (b *Builder) user(mountPoint string, userspec string) (specs.User, error) { } } - return u, err + return u, homeDir, err } -// dockerIgnore struct keep info from .dockerignore -type dockerIgnore struct { - ExcludePath string - IsExcluded bool -} - -// dockerIgnoreHelper returns the lines from .dockerignore file without the comments -// and reverses the order -func dockerIgnoreHelper(lines []string, contextDir string) []dockerIgnore { - var excludes []dockerIgnore - // the last match of a file in the .dockerignmatches determines whether it is included or excluded - // reverse the order - for i := len(lines) - 1; i >= 0; i-- { - exclude := lines[i] - // ignore the comment in .dockerignore - if strings.HasPrefix(exclude, "#") || len(exclude) == 0 { +// dockerIgnoreMatcher returns a matcher based on the contents of the .dockerignore file under contextDir +func dockerIgnoreMatcher(lines []string, contextDir string) (*fileutils.PatternMatcher, error) { + // if there's no context dir, there's no .dockerignore file to consult + if contextDir == "" { + return nil, nil + } + patterns := []string{".dockerignore"} + for _, ignoreSpec := range lines { + ignoreSpec = strings.TrimSpace(ignoreSpec) + // ignore comments passed back from .dockerignore + if ignoreSpec == "" || ignoreSpec[0] == '#' { continue } - excludeFlag := true - if strings.HasPrefix(exclude, "!") { - exclude = strings.TrimPrefix(exclude, "!") - excludeFlag = false + // if the spec starts with '!' it means the pattern + // should be included. make a note so that we can move + // it to the front of the updated pattern + includeFlag := "" + if strings.HasPrefix(ignoreSpec, "!") { + includeFlag = "!" + ignoreSpec = ignoreSpec[1:] } - excludes = append(excludes, dockerIgnore{ExcludePath: filepath.Join(contextDir, exclude), IsExcluded: excludeFlag}) + if ignoreSpec == "" { + continue + } + patterns = append(patterns, includeFlag+filepath.Join(contextDir, ignoreSpec)) } - if len(excludes) != 0 { - excludes = append(excludes, dockerIgnore{ExcludePath: filepath.Join(contextDir, ".dockerignore"), IsExcluded: true}) + // if there are no patterns, save time by not constructing the object + if len(patterns) == 0 { + return nil, nil } - return excludes -} - -func addHelper(excludes []dockerIgnore, extract bool, dest string, destfi os.FileInfo, hostOwner idtools.IDPair, options AddAndCopyOptions, copyFileWithTar, copyWithTar, untarPath func(src, dest string) error, source ...string) error { - dirsInDockerignore, err := getDirsInDockerignore(options.ContextDir, excludes) + // return a matcher object + matcher, err := fileutils.NewPatternMatcher(patterns) if err != nil { - return errors.Wrapf(err, "error checking directories in .dockerignore") + return nil, errors.Wrapf(err, "error creating file matcher using patterns %v", patterns) } + return matcher, nil +} + +func addHelper(excludes *fileutils.PatternMatcher, extract bool, dest string, destfi os.FileInfo, hostOwner idtools.IDPair, options AddAndCopyOptions, copyFileWithTar, copyWithTar, untarPath func(src, dest string) error, source ...string) error { for _, src := range source { if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") { // We assume that source is a file, and we're copying @@ -242,7 +249,7 @@ func addHelper(excludes []dockerIgnore, extract bool, dest string, destfi os.Fil if len(glob) == 0 { return errors.Wrapf(syscall.ENOENT, "no files found matching %q", src) } - outer: + for _, gsrc := range glob { esrc, err := filepath.EvalSymlinks(gsrc) if err != nil { @@ -261,7 +268,7 @@ func addHelper(excludes []dockerIgnore, extract bool, dest string, destfi os.Fil return errors.Wrapf(err, "error creating directory %q", dest) } logrus.Debugf("copying %q to %q", esrc+string(os.PathSeparator)+"*", dest+string(os.PathSeparator)+"*") - if len(excludes) == 0 { + if excludes == nil { if err = copyWithTar(esrc, dest); err != nil { return errors.Wrapf(err, "error copying %q to %q", esrc, dest) } @@ -271,23 +278,12 @@ func addHelper(excludes []dockerIgnore, extract bool, dest string, destfi os.Fil if err != nil { return err } - for _, exclude := range excludes { - match, err := filepath.Match(filepath.Clean(exclude.ExcludePath), filepath.Clean(path)) - if err != nil { - return err - } - prefix, exist := dirsInDockerignore[exclude.ExcludePath] - hasPrefix := false - if exist { - hasPrefix = filepath.HasPrefix(path, prefix) - } - if !(match || hasPrefix) { - continue - } - if (hasPrefix && exclude.IsExcluded) || (match && exclude.IsExcluded) { - return nil - } - break + skip, err := excludes.Matches(path) + if err != nil { + return errors.Wrapf(err, "error checking if %s is an excluded path", path) + } + if skip { + return nil } // combine the filename with the dest directory fpath, err := filepath.Rel(esrc, path) @@ -297,8 +293,8 @@ func addHelper(excludes []dockerIgnore, extract bool, dest string, destfi os.Fil mtime := info.ModTime() atime := mtime times := []syscall.Timespec{ - {Sec: atime.Unix(), Nsec: atime.UnixNano() % 1000000000}, - {Sec: mtime.Unix(), Nsec: mtime.UnixNano() % 1000000000}, + syscall.NsecToTimespec(atime.Unix()), + syscall.NsecToTimespec(mtime.Unix()), } if info.IsDir() { return addHelperDirectory(esrc, path, filepath.Join(dest, fpath), info, hostOwner, times) @@ -320,20 +316,6 @@ func addHelper(excludes []dockerIgnore, extract bool, dest string, destfi os.Fil continue } - for _, exclude := range excludes { - match, err := filepath.Match(filepath.Clean(exclude.ExcludePath), esrc) - if err != nil { - return err - } - if !match { - continue - } - if exclude.IsExcluded { - continue outer - } - break - } - if !extract || !archive.IsArchivePath(esrc) { // This source is a file, and either it's not an // archive, or we don't care whether or not it's an @@ -349,6 +331,7 @@ func addHelper(excludes []dockerIgnore, extract bool, dest string, destfi os.Fil } continue } + // We're extracting an archive into the destination directory. logrus.Debugf("extracting contents of %q into %q", esrc, dest) if err = untarPath(esrc, dest); err != nil { @@ -381,7 +364,15 @@ func addHelperSymlink(src, dest string, info os.FileInfo, hostOwner idtools.IDPa return errors.Wrapf(err, "error reading contents of symbolic link at %q", src) } if err = os.Symlink(linkContents, dest); err != nil { - return errors.Wrapf(err, "error creating symbolic link to %q at %q", linkContents, dest) + if !os.IsExist(err) { + return errors.Wrapf(err, "error creating symbolic link to %q at %q", linkContents, dest) + } + if err = os.RemoveAll(dest); err != nil { + return errors.Wrapf(err, "error clearing symbolic link target %q", dest) + } + if err = os.Symlink(linkContents, dest); err != nil { + return errors.Wrapf(err, "error creating symbolic link to %q at %q (second try)", linkContents, dest) + } } if err = idtools.SafeLchown(dest, hostOwner.UID, hostOwner.GID); err != nil { return errors.Wrapf(err, "error setting owner of symbolic link %q to %d:%d", dest, hostOwner.UID, hostOwner.GID) @@ -392,35 +383,3 @@ func addHelperSymlink(src, dest string, info os.FileInfo, hostOwner idtools.IDPa logrus.Debugf("Symlink(%s, %s)", linkContents, dest) return nil } - -func getDirsInDockerignore(srcAbsPath string, excludes []dockerIgnore) (map[string]string, error) { - visitedDir := make(map[string]string) - if len(excludes) == 0 { - return visitedDir, nil - } - err := filepath.Walk(srcAbsPath, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - for _, exclude := range excludes { - match, err := filepath.Match(filepath.Clean(exclude.ExcludePath), filepath.Clean(path)) - if err != nil { - return err - } - if !match { - continue - } - if _, exist := visitedDir[exclude.ExcludePath]; exist { - continue - } - visitedDir[exclude.ExcludePath] = path - } - } - return nil - }) - if err != nil { - return visitedDir, err - } - return visitedDir, nil -} diff --git a/vendor/github.com/containers/buildah/buildah.go b/vendor/github.com/containers/buildah/buildah.go index 33b7afccd..56c8f088f 100644 --- a/vendor/github.com/containers/buildah/buildah.go +++ b/vendor/github.com/containers/buildah/buildah.go @@ -26,7 +26,7 @@ const ( Package = "buildah" // Version for the Package. Bump version in contrib/rpm/buildah.spec // too. - Version = "1.9.0-dev" + Version = "1.8.3" // The value we use to identify what type of information, currently a // serialized Builder structure, we are using as per-container state. // This should only be changed when we make incompatible changes to diff --git a/vendor/github.com/containers/buildah/chroot/run.go b/vendor/github.com/containers/buildah/chroot/run.go index c65926c8e..d6e5a61ea 100644 --- a/vendor/github.com/containers/buildah/chroot/run.go +++ b/vendor/github.com/containers/buildah/chroot/run.go @@ -84,9 +84,18 @@ type runUsingChrootExecSubprocOptions struct { // RunUsingChroot runs a chrooted process, using some of the settings from the // passed-in spec, and using the specified bundlePath to hold temporary files, // directories, and mountpoints. -func RunUsingChroot(spec *specs.Spec, bundlePath string, stdin io.Reader, stdout, stderr io.Writer) (err error) { +func RunUsingChroot(spec *specs.Spec, bundlePath, homeDir string, stdin io.Reader, stdout, stderr io.Writer) (err error) { var confwg sync.WaitGroup - + var homeFound bool + for _, env := range spec.Process.Env { + if strings.HasPrefix(env, "HOME=") { + homeFound = true + break + } + } + if !homeFound { + spec.Process.Env = append(spec.Process.Env, fmt.Sprintf("HOME=%s", homeDir)) + } runtime.LockOSThread() defer runtime.UnlockOSThread() diff --git a/vendor/github.com/containers/buildah/config.go b/vendor/github.com/containers/buildah/config.go index 05b0abb23..234f93259 100644 --- a/vendor/github.com/containers/buildah/config.go +++ b/vendor/github.com/containers/buildah/config.go @@ -3,7 +3,6 @@ package buildah import ( "context" "encoding/json" - "os" "runtime" "strings" "time" @@ -269,21 +268,11 @@ func (b *Builder) Env() []string { // built using an image built from this container. func (b *Builder) SetEnv(k string, v string) { reset := func(s *[]string) { - getenv := func(name string) string { - for i := range *s { - val := strings.SplitN((*s)[i], "=", 2) - if len(val) == 2 && val[0] == name { - return val[1] - } - } - return name - } n := []string{} for i := range *s { if !strings.HasPrefix((*s)[i], k+"=") { n = append(n, (*s)[i]) } - v = os.Expand(v, getenv) } n = append(n, k+"="+v) *s = n diff --git a/vendor/github.com/containers/buildah/imagebuildah/build.go b/vendor/github.com/containers/buildah/imagebuildah/build.go index b8b9db0f3..3665251cd 100644 --- a/vendor/github.com/containers/buildah/imagebuildah/build.go +++ b/vendor/github.com/containers/buildah/imagebuildah/build.go @@ -487,6 +487,7 @@ func (s *StageExecutor) Copy(excludes []string, copies ...imagebuilder.Copy) err // Check the file and see if part of it is a symlink. // Convert it to the target if so. To be ultrasafe // do the same for the mountpoint. + hadFinalPathSeparator := len(copy.Dest) > 0 && copy.Dest[len(copy.Dest)-1] == os.PathSeparator secureMountPoint, err := securejoin.SecureJoin("", s.mountPoint) finalPath, err := securejoin.SecureJoin(secureMountPoint, copy.Dest) if err != nil { @@ -496,6 +497,11 @@ func (s *StageExecutor) Copy(excludes []string, copies ...imagebuilder.Copy) err return errors.Wrapf(err, "error resolving copy destination %s", copy.Dest) } copy.Dest = strings.TrimPrefix(finalPath, secureMountPoint) + if len(copy.Dest) == 0 || copy.Dest[len(copy.Dest)-1] != os.PathSeparator { + if hadFinalPathSeparator { + copy.Dest += string(os.PathSeparator) + } + } if copy.Download { logrus.Debugf("ADD %#v, %#v", excludes, copy) @@ -507,29 +513,32 @@ func (s *StageExecutor) Copy(excludes []string, copies ...imagebuilder.Copy) err } sources := []string{} for _, src := range copy.Src { + contextDir := s.executor.contextDir + copyExcludes := excludes if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") { sources = append(sources, src) } else if len(copy.From) > 0 { if other, ok := s.executor.stages[copy.From]; ok && other.index < s.index { sources = append(sources, filepath.Join(other.mountPoint, src)) + contextDir = other.mountPoint } else if builder, ok := s.executor.containerMap[copy.From]; ok { sources = append(sources, filepath.Join(builder.MountPoint, src)) + contextDir = builder.MountPoint } else { return errors.Errorf("the stage %q has not been built", copy.From) } } else { sources = append(sources, filepath.Join(s.executor.contextDir, src)) + copyExcludes = append(s.executor.excludes, excludes...) + } + options := buildah.AddAndCopyOptions{ + Chown: copy.Chown, + ContextDir: contextDir, + Excludes: copyExcludes, + } + if err := s.builder.Add(copy.Dest, copy.Download, options, sources...); err != nil { + return err } - } - - options := buildah.AddAndCopyOptions{ - Chown: copy.Chown, - ContextDir: s.executor.contextDir, - Excludes: s.executor.excludes, - } - - if err := s.builder.Add(copy.Dest, copy.Download, options, sources...); err != nil { - return err } } return nil @@ -590,7 +599,11 @@ func (s *StageExecutor) Run(run imagebuilder.Run, config docker.Config) error { args := run.Args if run.Shell { - args = append([]string{"/bin/sh", "-c"}, args...) + if len(config.Shell) > 0 && s.builder.Format == buildah.Dockerv2ImageManifest { + args = append(config.Shell, args...) + } else { + args = append([]string{"/bin/sh", "-c"}, args...) + } } if err := s.volumeCacheSave(); err != nil { return err diff --git a/vendor/github.com/containers/buildah/imagebuildah/util.go b/vendor/github.com/containers/buildah/imagebuildah/util.go index f982fcebf..3962d1a9d 100644 --- a/vendor/github.com/containers/buildah/imagebuildah/util.go +++ b/vendor/github.com/containers/buildah/imagebuildah/util.go @@ -17,7 +17,7 @@ import ( ) func cloneToDirectory(url, dir string) error { - if !strings.HasPrefix(url, "git://") { + if !strings.HasPrefix(url, "git://") && !strings.HasSuffix(url, ".git") { url = "git://" + url } logrus.Debugf("cloning %q to %q", url, dir) @@ -72,7 +72,7 @@ func TempDirForURL(dir, prefix, url string) (name string, subdir string, err err if err != nil { return "", "", errors.Wrapf(err, "error creating temporary directory for %q", url) } - if strings.HasPrefix(url, "git://") { + if strings.HasPrefix(url, "git://") || strings.HasSuffix(url, ".git") { err = cloneToDirectory(url, name) if err != nil { if err2 := os.Remove(name); err2 != nil { diff --git a/vendor/github.com/containers/buildah/pkg/chrootuser/user.go b/vendor/github.com/containers/buildah/pkg/chrootuser/user.go index c83dcc230..26a67c35a 100644 --- a/vendor/github.com/containers/buildah/pkg/chrootuser/user.go +++ b/vendor/github.com/containers/buildah/pkg/chrootuser/user.go @@ -18,7 +18,7 @@ var ( // it will use the /etc/passwd and /etc/group files inside of the rootdir // to return this information. // userspec format [user | user:group | uid | uid:gid | user:gid | uid:group ] -func GetUser(rootdir, userspec string) (uint32, uint32, error) { +func GetUser(rootdir, userspec string) (uint32, uint32, string, error) { var gid64 uint64 var gerr error = user.UnknownGroupError("error looking up group") @@ -26,7 +26,7 @@ func GetUser(rootdir, userspec string) (uint32, uint32, error) { userspec = spec[0] groupspec := "" if userspec == "" { - return 0, 0, nil + return 0, 0, "/", nil } if len(spec) > 1 { groupspec = spec[1] @@ -65,15 +65,21 @@ func GetUser(rootdir, userspec string) (uint32, uint32, error) { } } + homedir, err := lookupHomedirInContainer(rootdir, uid64) + if err != nil { + homedir = "/" + } + if uerr == nil && gerr == nil { - return uint32(uid64), uint32(gid64), nil + return uint32(uid64), uint32(gid64), homedir, nil } - err := errors.Wrapf(uerr, "error determining run uid") + err = errors.Wrapf(uerr, "error determining run uid") if uerr == nil { err = errors.Wrapf(gerr, "error determining run gid") } - return 0, 0, err + + return 0, 0, homedir, err } // GetGroup returns the gid by looking it up in the /etc/group file diff --git a/vendor/github.com/containers/buildah/pkg/chrootuser/user_basic.go b/vendor/github.com/containers/buildah/pkg/chrootuser/user_basic.go index 79b0b24b5..6c997c4c9 100644 --- a/vendor/github.com/containers/buildah/pkg/chrootuser/user_basic.go +++ b/vendor/github.com/containers/buildah/pkg/chrootuser/user_basic.go @@ -25,3 +25,7 @@ func lookupAdditionalGroupsForUIDInContainer(rootdir string, userid uint64) (gid func lookupUIDInContainer(rootdir string, uid uint64) (string, uint64, error) { return "", 0, errors.New("UID lookup not supported") } + +func lookupHomedirInContainer(rootdir string, uid uint64) (string, error) { + return "", errors.New("Home directory lookup not supported") +} diff --git a/vendor/github.com/containers/buildah/pkg/chrootuser/user_linux.go b/vendor/github.com/containers/buildah/pkg/chrootuser/user_linux.go index 583eca569..ea20fca80 100644 --- a/vendor/github.com/containers/buildah/pkg/chrootuser/user_linux.go +++ b/vendor/github.com/containers/buildah/pkg/chrootuser/user_linux.go @@ -84,6 +84,7 @@ type lookupPasswdEntry struct { name string uid uint64 gid uint64 + home string } type lookupGroupEntry struct { name string @@ -135,6 +136,7 @@ func parseNextPasswd(rc *bufio.Reader) *lookupPasswdEntry { name: fields[0], uid: uid, gid: gid, + home: fields[5], } } @@ -291,3 +293,29 @@ func lookupUIDInContainer(rootdir string, uid uint64) (string, uint64, error) { return "", 0, user.UnknownUserError(fmt.Sprintf("error looking up uid %q", uid)) } + +func lookupHomedirInContainer(rootdir string, uid uint64) (string, error) { + cmd, f, err := openChrootedFile(rootdir, "/etc/passwd") + if err != nil { + return "", err + } + defer func() { + _ = cmd.Wait() + }() + rc := bufio.NewReader(f) + defer f.Close() + + lookupUser.Lock() + defer lookupUser.Unlock() + + pwd := parseNextPasswd(rc) + for pwd != nil { + if pwd.uid != uid { + pwd = parseNextPasswd(rc) + continue + } + return pwd.home, nil + } + + return "", user.UnknownUserError(fmt.Sprintf("error looking up uid %q for homedir", uid)) +} diff --git a/vendor/github.com/containers/buildah/pkg/secrets/secrets.go b/vendor/github.com/containers/buildah/pkg/secrets/secrets.go index 97b681125..70bd6a4b7 100644 --- a/vendor/github.com/containers/buildah/pkg/secrets/secrets.go +++ b/vendor/github.com/containers/buildah/pkg/secrets/secrets.go @@ -117,7 +117,12 @@ func getMounts(filePath string) []string { } var mounts []string for scanner.Scan() { - mounts = append(mounts, scanner.Text()) + if strings.HasPrefix(strings.TrimSpace(scanner.Text()), "/") { + mounts = append(mounts, scanner.Text()) + } else { + logrus.Debugf("skipping unrecognized mount in %v: %q", + filePath, scanner.Text()) + } } return mounts } @@ -190,58 +195,79 @@ func addSecretsFromMountsFile(filePath, mountLabel, containerWorkingDir, mountPr var mounts []rspec.Mount defaultMountsPaths := getMounts(filePath) for _, path := range defaultMountsPaths { - hostDir, ctrDir, err := getMountsMap(path) + hostDirOrFile, ctrDirOrFile, err := getMountsMap(path) if err != nil { return nil, err } - // skip if the hostDir path doesn't exist - if _, err = os.Stat(hostDir); err != nil { + // skip if the hostDirOrFile path doesn't exist + fileInfo, err := os.Stat(hostDirOrFile) + if err != nil { if os.IsNotExist(err) { - logrus.Warnf("Path %q from %q doesn't exist, skipping", hostDir, filePath) + logrus.Warnf("Path %q from %q doesn't exist, skipping", hostDirOrFile, filePath) continue } - return nil, errors.Wrapf(err, "failed to stat %q", hostDir) + return nil, errors.Wrapf(err, "failed to stat %q", hostDirOrFile) } - ctrDirOnHost := filepath.Join(containerWorkingDir, ctrDir) + ctrDirOrFileOnHost := filepath.Join(containerWorkingDir, ctrDirOrFile) - // In the event of a restart, don't want to copy secrets over again as they already would exist in ctrDirOnHost - _, err = os.Stat(ctrDirOnHost) + // In the event of a restart, don't want to copy secrets over again as they already would exist in ctrDirOrFileOnHost + _, err = os.Stat(ctrDirOrFileOnHost) if os.IsNotExist(err) { - if err = os.MkdirAll(ctrDirOnHost, 0755); err != nil { - return nil, errors.Wrapf(err, "making container directory %q failed", ctrDirOnHost) - } - hostDir, err = resolveSymbolicLink(hostDir) + + hostDirOrFile, err = resolveSymbolicLink(hostDirOrFile) if err != nil { return nil, err } - data, err := getHostSecretData(hostDir) - if err != nil { - return nil, errors.Wrapf(err, "getting host secret data failed") - } - for _, s := range data { - if err := s.saveTo(ctrDirOnHost); err != nil { - return nil, errors.Wrapf(err, "error saving data to container filesystem on host %q", ctrDirOnHost) + switch mode := fileInfo.Mode(); { + case mode.IsDir(): + if err = os.MkdirAll(ctrDirOrFileOnHost, 0755); err != nil { + return nil, errors.Wrapf(err, "making container directory %q failed", ctrDirOrFileOnHost) + } + data, err := getHostSecretData(hostDirOrFile) + if err != nil { + return nil, errors.Wrapf(err, "getting host secret data failed") + } + for _, s := range data { + if err := s.saveTo(ctrDirOrFileOnHost); err != nil { + return nil, errors.Wrapf(err, "error saving data to container filesystem on host %q", ctrDirOrFileOnHost) + } + } + case mode.IsRegular(): + data, err := readFile("", hostDirOrFile) + if err != nil { + return nil, errors.Wrapf(err, "error reading file %q", hostDirOrFile) + + } + for _, s := range data { + if err := os.MkdirAll(filepath.Dir(ctrDirOrFileOnHost), 0700); err != nil { + return nil, err + } + if err := ioutil.WriteFile(ctrDirOrFileOnHost, s.data, 0700); err != nil { + return nil, errors.Wrapf(err, "error saving data to container filesystem on host %q", ctrDirOrFileOnHost) + } } + default: + return nil, errors.Errorf("unsupported file type for: %q", hostDirOrFile) } - err = label.Relabel(ctrDirOnHost, mountLabel, false) + err = label.Relabel(ctrDirOrFileOnHost, mountLabel, false) if err != nil { return nil, errors.Wrap(err, "error applying correct labels") } if uid != 0 || gid != 0 { - if err := rchown(ctrDirOnHost, uid, gid); err != nil { + if err := rchown(ctrDirOrFileOnHost, uid, gid); err != nil { return nil, err } } } else if err != nil { - return nil, errors.Wrapf(err, "error getting status of %q", ctrDirOnHost) + return nil, errors.Wrapf(err, "error getting status of %q", ctrDirOrFileOnHost) } m := rspec.Mount{ - Source: filepath.Join(mountPrefix, ctrDir), - Destination: ctrDir, + Source: filepath.Join(mountPrefix, ctrDirOrFile), + Destination: ctrDirOrFile, Type: "bind", Options: []string{"bind", "rprivate"}, } diff --git a/vendor/github.com/containers/buildah/pkg/unshare/unshare.go b/vendor/github.com/containers/buildah/pkg/unshare/unshare.go index 33232740e..5a68d6b8d 100644 --- a/vendor/github.com/containers/buildah/pkg/unshare/unshare.go +++ b/vendor/github.com/containers/buildah/pkg/unshare/unshare.go @@ -64,6 +64,7 @@ func (c *Cmd) Start() error { if os.Geteuid() != 0 { c.Env = append(c.Env, "_CONTAINERS_USERNS_CONFIGURED=done") c.Env = append(c.Env, fmt.Sprintf("_CONTAINERS_ROOTLESS_UID=%d", os.Geteuid())) + c.Env = append(c.Env, fmt.Sprintf("_CONTAINERS_ROOTLESS_GID=%d", os.Getegid())) } // Create the pipe for reading the child's PID. @@ -183,6 +184,7 @@ func (c *Cmd) Start() error { for _, m := range c.GidMappings { fmt.Fprintf(g, "%d %d %d\n", m.ContainerID, m.HostID, m.Size) } + gidmapSet := false // Set the GID map. if c.UseNewgidmap { cmd := exec.Command("newgidmap", append([]string{pidString}, strings.Fields(strings.Replace(g.String(), "\n", " ", -1))...)...) @@ -190,11 +192,16 @@ func (c *Cmd) Start() error { cmd.Stdout = g cmd.Stderr = g err := cmd.Run() - if err != nil { + if err == nil { + gidmapSet = true + } else { fmt.Fprintf(continueWrite, "error running newgidmap: %v: %s", err, g.String()) - return errors.Wrapf(err, "error running newgidmap: %s", g.String()) + fmt.Fprintf(continueWrite, "falling back to single mapping\n") + g.Reset() + g.Write([]byte(fmt.Sprintf("0 %d 1\n", os.Getegid()))) } - } else { + } + if !gidmapSet { gidmap, err := os.OpenFile(fmt.Sprintf("/proc/%s/gid_map", pidString), os.O_TRUNC|os.O_WRONLY, 0) if err != nil { fmt.Fprintf(continueWrite, "error opening /proc/%s/gid_map: %v", pidString, err) @@ -214,6 +221,7 @@ func (c *Cmd) Start() error { for _, m := range c.UidMappings { fmt.Fprintf(u, "%d %d %d\n", m.ContainerID, m.HostID, m.Size) } + uidmapSet := false // Set the GID map. if c.UseNewuidmap { cmd := exec.Command("newuidmap", append([]string{pidString}, strings.Fields(strings.Replace(u.String(), "\n", " ", -1))...)...) @@ -221,11 +229,16 @@ func (c *Cmd) Start() error { cmd.Stdout = u cmd.Stderr = u err := cmd.Run() - if err != nil { + if err == nil { + uidmapSet = true + } else { fmt.Fprintf(continueWrite, "error running newuidmap: %v: %s", err, u.String()) - return errors.Wrapf(err, "error running newuidmap: %s", u.String()) + fmt.Fprintf(continueWrite, "falling back to single mapping\n") + u.Reset() + u.Write([]byte(fmt.Sprintf("0 %d 1\n", os.Geteuid()))) } - } else { + } + if !uidmapSet { uidmap, err := os.OpenFile(fmt.Sprintf("/proc/%s/uid_map", pidString), os.O_TRUNC|os.O_WRONLY, 0) if err != nil { fmt.Fprintf(continueWrite, "error opening /proc/%s/uid_map: %v", pidString, err) @@ -354,7 +367,9 @@ func MaybeReexecUsingUserNamespace(evenForRoot bool) { // range in /etc/subuid and /etc/subgid file is a starting host // ID and a range size. uidmap, gidmap, err = GetSubIDMappings(me.Username, me.Username) - bailOnError(err, "error reading allowed ID mappings") + if err != nil { + logrus.Warnf("error reading allowed ID mappings: %v", err) + } if len(uidmap) == 0 { logrus.Warnf("Found no UID ranges set aside for user %q in /etc/subuid.", me.Username) } diff --git a/vendor/github.com/containers/buildah/run_linux.go b/vendor/github.com/containers/buildah/run_linux.go index 81ce2b944..16c0550aa 100644 --- a/vendor/github.com/containers/buildah/run_linux.go +++ b/vendor/github.com/containers/buildah/run_linux.go @@ -131,7 +131,8 @@ func (b *Builder) Run(command []string, options RunOptions) error { return err } - if err := b.configureUIDGID(g, mountPoint, options); err != nil { + homeDir, err := b.configureUIDGID(g, mountPoint, options) + if err != nil { return err } @@ -210,7 +211,7 @@ func (b *Builder) Run(command []string, options RunOptions) error { } err = b.runUsingRuntimeSubproc(isolation, options, configureNetwork, configureNetworks, moreCreateArgs, spec, mountPoint, path, Package+"-"+filepath.Base(path)) case IsolationChroot: - err = chroot.RunUsingChroot(spec, path, options.Stdin, options.Stdout, options.Stderr) + err = chroot.RunUsingChroot(spec, path, homeDir, options.Stdin, options.Stdout, options.Stderr) case IsolationOCIRootless: moreCreateArgs := []string{"--no-new-keyring"} if options.NoPivot { @@ -1454,7 +1455,18 @@ func setupNamespaces(g *generate.Generator, namespaceOptions NamespaceOptions, i } if configureNetwork { for name, val := range util.DefaultNetworkSysctl { - g.AddLinuxSysctl(name, val) + // Check that the sysctl we are adding is actually supported + // by the kernel + p := filepath.Join("/proc/sys", strings.Replace(name, ".", "/", -1)) + _, err := os.Stat(p) + if err != nil && !os.IsNotExist(err) { + return false, nil, false, errors.Wrapf(err, "cannot stat %s", p) + } + if err == nil { + g.AddLinuxSysctl(name, val) + } else { + logrus.Warnf("ignoring sysctl %s since %s doesn't exist", name, p) + } } } return configureNetwork, configureNetworks, configureUTS, nil @@ -1775,14 +1787,14 @@ func getDNSIP(dnsServers []string) (dns []net.IP, err error) { return dns, nil } -func (b *Builder) configureUIDGID(g *generate.Generator, mountPoint string, options RunOptions) error { +func (b *Builder) configureUIDGID(g *generate.Generator, mountPoint string, options RunOptions) (string, error) { // Set the user UID/GID/supplemental group list/capabilities lists. - user, err := b.user(mountPoint, options.User) + user, homeDir, err := b.user(mountPoint, options.User) if err != nil { - return err + return "", err } if err := setupCapabilities(g, b.AddCapabilities, b.DropCapabilities, options.AddCapabilities, options.DropCapabilities); err != nil { - return err + return "", err } g.SetProcessUID(user.UID) g.SetProcessGID(user.GID) @@ -1797,7 +1809,7 @@ func (b *Builder) configureUIDGID(g *generate.Generator, mountPoint string, opti g.Config.Process.Capabilities.Bounding = bounding } - return nil + return homeDir, nil } func (b *Builder) configureEnvironment(g *generate.Generator, options RunOptions) { diff --git a/vendor/github.com/containers/storage/containers.go b/vendor/github.com/containers/storage/containers.go index bbac78b60..e69552361 100644 --- a/vendor/github.com/containers/storage/containers.go +++ b/vendor/github.com/containers/storage/containers.go @@ -572,6 +572,10 @@ func (r *containerStore) Lock() { r.lockfile.Lock() } +func (r *containerStore) RecursiveLock() { + r.lockfile.RecursiveLock() +} + func (r *containerStore) RLock() { r.lockfile.RLock() } diff --git a/vendor/github.com/containers/storage/drivers/aufs/aufs.go b/vendor/github.com/containers/storage/drivers/aufs/aufs.go index e821bc0c5..353d1707a 100644 --- a/vendor/github.com/containers/storage/drivers/aufs/aufs.go +++ b/vendor/github.com/containers/storage/drivers/aufs/aufs.go @@ -255,6 +255,9 @@ func (a *Driver) AdditionalImageStores() []string { // CreateFromTemplate creates a layer with the same contents and parent as another layer. func (a *Driver) CreateFromTemplate(id, template string, templateIDMappings *idtools.IDMappings, parent string, parentIDMappings *idtools.IDMappings, opts *graphdriver.CreateOpts, readWrite bool) error { + if opts == nil { + opts = &graphdriver.CreateOpts{} + } return graphdriver.NaiveCreateFromTemplate(a, id, template, templateIDMappings, parent, parentIDMappings, opts, readWrite) } diff --git a/vendor/github.com/containers/storage/drivers/chown.go b/vendor/github.com/containers/storage/drivers/chown.go index 4d4011ee0..f2f1ec386 100644 --- a/vendor/github.com/containers/storage/drivers/chown.go +++ b/vendor/github.com/containers/storage/drivers/chown.go @@ -55,6 +55,9 @@ func chownByMapsMain() { if err != nil { return fmt.Errorf("error walking to %q: %v", path, err) } + if path == "." { + return nil + } return platformLChown(path, info, toHost, toContainer) } if err := filepath.Walk(".", chown); err != nil { diff --git a/vendor/github.com/containers/storage/images.go b/vendor/github.com/containers/storage/images.go index 38b5a3ef3..6f487504a 100644 --- a/vendor/github.com/containers/storage/images.go +++ b/vendor/github.com/containers/storage/images.go @@ -82,6 +82,9 @@ type Image struct { // is set before using it. Created time.Time `json:"created,omitempty"` + // ReadOnly is true if this image resides in a read-only layer store. + ReadOnly bool `json:"-"` + Flags map[string]interface{} `json:"flags,omitempty"` } @@ -159,6 +162,7 @@ func copyImage(i *Image) *Image { BigDataSizes: copyStringInt64Map(i.BigDataSizes), BigDataDigests: copyStringDigestMap(i.BigDataDigests), Created: i.Created, + ReadOnly: i.ReadOnly, Flags: copyStringInterfaceMap(i.Flags), } } @@ -269,6 +273,7 @@ func (r *imageStore) Load() error { list := digests[digest] digests[digest] = append(list, image) } + image.ReadOnly = !r.IsReadWrite() } } if shouldSave && (!r.IsReadWrite() || !r.Locked()) { @@ -739,6 +744,10 @@ func (r *imageStore) Lock() { r.lockfile.Lock() } +func (r *imageStore) RecursiveLock() { + r.lockfile.RecursiveLock() +} + func (r *imageStore) RLock() { r.lockfile.RLock() } diff --git a/vendor/github.com/containers/storage/layers.go b/vendor/github.com/containers/storage/layers.go index a35dd476b..fb79238cd 100644 --- a/vendor/github.com/containers/storage/layers.go +++ b/vendor/github.com/containers/storage/layers.go @@ -103,6 +103,9 @@ type Layer struct { // for use inside of a user namespace where UID mapping is being used. UIDMap []idtools.IDMap `json:"uidmap,omitempty"` GIDMap []idtools.IDMap `json:"gidmap,omitempty"` + + // ReadOnly is true if this layer resides in a read-only layer store. + ReadOnly bool `json:"-"` } type layerMountPoint struct { @@ -259,6 +262,7 @@ func copyLayer(l *Layer) *Layer { UncompressedDigest: l.UncompressedDigest, UncompressedSize: l.UncompressedSize, CompressionType: l.CompressionType, + ReadOnly: l.ReadOnly, Flags: copyStringInterfaceMap(l.Flags), UIDMap: copyIDMap(l.UIDMap), GIDMap: copyIDMap(l.GIDMap), @@ -318,6 +322,7 @@ func (r *layerStore) Load() error { if layer.MountLabel != "" { label.ReserveLabel(layer.MountLabel) } + layer.ReadOnly = !r.IsReadWrite() } err = nil } @@ -1304,6 +1309,10 @@ func (r *layerStore) Lock() { r.lockfile.Lock() } +func (r *layerStore) RecursiveLock() { + r.lockfile.RecursiveLock() +} + func (r *layerStore) RLock() { r.lockfile.RLock() } diff --git a/vendor/github.com/containers/storage/layers_ffjson.go b/vendor/github.com/containers/storage/layers_ffjson.go index 09b5d0f33..125b5d8c9 100644 --- a/vendor/github.com/containers/storage/layers_ffjson.go +++ b/vendor/github.com/containers/storage/layers_ffjson.go @@ -1,5 +1,5 @@ // Code generated by ffjson <https://github.com/pquerna/ffjson>. DO NOT EDIT. -// source: ./layers.go +// source: layers.go package storage diff --git a/vendor/github.com/containers/storage/lockfile.go b/vendor/github.com/containers/storage/lockfile.go index ed8753337..c4f1b5549 100644 --- a/vendor/github.com/containers/storage/lockfile.go +++ b/vendor/github.com/containers/storage/lockfile.go @@ -15,6 +15,10 @@ type Locker interface { // Acquire a writer lock. Lock() + // Acquire a writer lock recursively, allowing for recursive acquisitions + // within the same process space. + RecursiveLock() + // Unlock the lock. Unlock() diff --git a/vendor/github.com/containers/storage/lockfile_linux.go b/vendor/github.com/containers/storage/lockfile_linux.go deleted file mode 100644 index 903387c66..000000000 --- a/vendor/github.com/containers/storage/lockfile_linux.go +++ /dev/null @@ -1,20 +0,0 @@ -// +build linux solaris - -package storage - -import ( - "time" - - "golang.org/x/sys/unix" -) - -// TouchedSince indicates if the lock file has been touched since the specified time -func (l *lockfile) TouchedSince(when time.Time) bool { - st := unix.Stat_t{} - err := unix.Fstat(int(l.fd), &st) - if err != nil { - return true - } - touched := time.Unix(st.Mtim.Unix()) - return when.Before(touched) -} diff --git a/vendor/github.com/containers/storage/lockfile_otherunix.go b/vendor/github.com/containers/storage/lockfile_otherunix.go deleted file mode 100644 index 041d54c05..000000000 --- a/vendor/github.com/containers/storage/lockfile_otherunix.go +++ /dev/null @@ -1,19 +0,0 @@ -// +build darwin freebsd - -package storage - -import ( - "time" - - "golang.org/x/sys/unix" -) - -func (l *lockfile) TouchedSince(when time.Time) bool { - st := unix.Stat_t{} - err := unix.Fstat(int(l.fd), &st) - if err != nil { - return true - } - touched := time.Unix(st.Mtimespec.Unix()) - return when.Before(touched) -} diff --git a/vendor/github.com/containers/storage/lockfile_unix.go b/vendor/github.com/containers/storage/lockfile_unix.go index 8e0f22cb5..00215e928 100644 --- a/vendor/github.com/containers/storage/lockfile_unix.go +++ b/vendor/github.com/containers/storage/lockfile_unix.go @@ -9,6 +9,7 @@ import ( "time" "github.com/containers/storage/pkg/stringid" + "github.com/containers/storage/pkg/system" "github.com/pkg/errors" "golang.org/x/sys/unix" ) @@ -25,6 +26,7 @@ type lockfile struct { locktype int16 locked bool ro bool + recursive bool } // openLock opens the file at path and returns the corresponding file @@ -75,7 +77,7 @@ func createLockerForPath(path string, ro bool) (Locker, error) { // lock locks the lockfile via FCTNL(2) based on the specified type and // command. -func (l *lockfile) lock(l_type int16) { +func (l *lockfile) lock(l_type int16, recursive bool) { lk := unix.Flock_t{ Type: l_type, Whence: int16(os.SEEK_SET), @@ -86,7 +88,13 @@ func (l *lockfile) lock(l_type int16) { case unix.F_RDLCK: l.rwMutex.RLock() case unix.F_WRLCK: - l.rwMutex.Lock() + if recursive { + // NOTE: that's okay as recursive is only set in RecursiveLock(), so + // there's no need to protect against hypothetical RDLCK cases. + l.rwMutex.RLock() + } else { + l.rwMutex.Lock() + } default: panic(fmt.Sprintf("attempted to acquire a file lock of unrecognized type %d", l_type)) } @@ -110,6 +118,7 @@ func (l *lockfile) lock(l_type int16) { } l.locktype = l_type l.locked = true + l.recursive = recursive l.counter++ } @@ -119,13 +128,24 @@ func (l *lockfile) Lock() { if l.ro { l.RLock() } else { - l.lock(unix.F_WRLCK) + l.lock(unix.F_WRLCK, false) + } +} + +// RecursiveLock locks the lockfile as a writer but allows for recursive +// acquisitions within the same process space. Note that RLock() will be called +// if it's a lockTypReader lock. +func (l *lockfile) RecursiveLock() { + if l.ro { + l.RLock() + } else { + l.lock(unix.F_WRLCK, true) } } // LockRead locks the lockfile as a reader. func (l *lockfile) RLock() { - l.lock(unix.F_RDLCK) + l.lock(unix.F_RDLCK, false) } // Unlock unlocks the lockfile. @@ -161,7 +181,7 @@ func (l *lockfile) Unlock() { // Close the file descriptor on the last unlock. unix.Close(int(l.fd)) } - if l.locktype == unix.F_RDLCK { + if l.locktype == unix.F_RDLCK || l.recursive { l.rwMutex.RUnlock() } else { l.rwMutex.Unlock() @@ -232,3 +252,14 @@ func (l *lockfile) Modified() (bool, error) { func (l *lockfile) IsReadWrite() bool { return !l.ro } + +// TouchedSince indicates if the lock file has been touched since the specified time +func (l *lockfile) TouchedSince(when time.Time) bool { + st, err := system.Fstat(int(l.fd)) + if err != nil { + return true + } + mtim := st.Mtim() + touched := time.Unix(mtim.Unix()) + return when.Before(touched) +} diff --git a/vendor/github.com/containers/storage/lockfile_windows.go b/vendor/github.com/containers/storage/lockfile_windows.go index c02069495..caf7c184a 100644 --- a/vendor/github.com/containers/storage/lockfile_windows.go +++ b/vendor/github.com/containers/storage/lockfile_windows.go @@ -36,6 +36,12 @@ func (l *lockfile) Lock() { l.locked = true } +func (l *lockfile) RecursiveLock() { + // We don't support Windows but a recursive writer-lock in one process-space + // is really a writer lock, so just panic. + panic("not supported") +} + func (l *lockfile) RLock() { l.mu.Lock() l.locked = true diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/archive.go b/vendor/github.com/containers/storage/pkg/chrootarchive/archive.go index a36ff1cb1..33ba6a128 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/archive.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/archive.go @@ -1,7 +1,7 @@ package chrootarchive import ( - "archive/tar" + stdtar "archive/tar" "fmt" "io" "io/ioutil" @@ -34,18 +34,34 @@ func NewArchiverWithChown(tarIDMappings *idtools.IDMappings, chownOpts *idtools. // The archive may be compressed with one of the following algorithms: // identity (uncompressed), gzip, bzip2, xz. func Untar(tarArchive io.Reader, dest string, options *archive.TarOptions) error { - return untarHandler(tarArchive, dest, options, true) + return untarHandler(tarArchive, dest, options, true, dest) +} + +// UntarWithRoot is the same as `Untar`, but allows you to pass in a root directory +// The root directory is the directory that will be chrooted to. +// `dest` must be a path within `root`, if it is not an error will be returned. +// +// `root` should set to a directory which is not controlled by any potentially +// malicious process. +// +// This should be used to prevent a potential attacker from manipulating `dest` +// such that it would provide access to files outside of `dest` through things +// like symlinks. Normally `ResolveSymlinksInScope` would handle this, however +// sanitizing symlinks in this manner is inherrently racey: +// ref: CVE-2018-15664 +func UntarWithRoot(tarArchive io.Reader, dest string, options *archive.TarOptions, root string) error { + return untarHandler(tarArchive, dest, options, true, root) } // UntarUncompressed reads a stream of bytes from `archive`, parses it as a tar archive, // and unpacks it into the directory at `dest`. // The archive must be an uncompressed stream. func UntarUncompressed(tarArchive io.Reader, dest string, options *archive.TarOptions) error { - return untarHandler(tarArchive, dest, options, false) + return untarHandler(tarArchive, dest, options, false, dest) } // Handler for teasing out the automatic decompression -func untarHandler(tarArchive io.Reader, dest string, options *archive.TarOptions, decompress bool) error { +func untarHandler(tarArchive io.Reader, dest string, options *archive.TarOptions, decompress bool, root string) error { if tarArchive == nil { return fmt.Errorf("Empty archive") } @@ -77,7 +93,15 @@ func untarHandler(tarArchive io.Reader, dest string, options *archive.TarOptions r = decompressedArchive } - return invokeUnpack(r, dest, options) + return invokeUnpack(r, dest, options, root) +} + +// Tar tars the requested path while chrooted to the specified root. +func Tar(srcPath string, options *archive.TarOptions, root string) (io.ReadCloser, error) { + if options == nil { + options = &archive.TarOptions{} + } + return invokePack(srcPath, options, root) } // CopyFileWithTarAndChown returns a function which copies a single file from outside @@ -99,7 +123,7 @@ func CopyFileWithTarAndChown(chownOpts *idtools.IDPair, hasher io.Writer, uidmap var hashWorker sync.WaitGroup hashWorker.Add(1) go func() { - t := tar.NewReader(contentReader) + t := stdtar.NewReader(contentReader) _, err := t.Next() if err != nil { hashError = err diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/archive_unix.go b/vendor/github.com/containers/storage/pkg/chrootarchive/archive_unix.go index e04ed787c..ca9fb10d7 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/archive_unix.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/archive_unix.go @@ -10,10 +10,13 @@ import ( "io" "io/ioutil" "os" + "path/filepath" "runtime" + "strings" "github.com/containers/storage/pkg/archive" "github.com/containers/storage/pkg/reexec" + "github.com/pkg/errors" ) // untar is the entry-point for storage-untar on re-exec. This is not used on @@ -23,18 +26,28 @@ func untar() { runtime.LockOSThread() flag.Parse() - var options *archive.TarOptions + var options archive.TarOptions //read the options from the pipe "ExtraFiles" if err := json.NewDecoder(os.NewFile(3, "options")).Decode(&options); err != nil { fatal(err) } - if err := chroot(flag.Arg(0)); err != nil { + dst := flag.Arg(0) + var root string + if len(flag.Args()) > 1 { + root = flag.Arg(1) + } + + if root == "" { + root = dst + } + + if err := chroot(root); err != nil { fatal(err) } - if err := archive.Unpack(os.Stdin, "/", options); err != nil { + if err := archive.Unpack(os.Stdin, dst, &options); err != nil { fatal(err) } // fully consume stdin in case it is zero padded @@ -45,7 +58,10 @@ func untar() { os.Exit(0) } -func invokeUnpack(decompressedArchive io.Reader, dest string, options *archive.TarOptions) error { +func invokeUnpack(decompressedArchive io.Reader, dest string, options *archive.TarOptions, root string) error { + if root == "" { + return errors.New("must specify a root to chroot to") + } // We can't pass a potentially large exclude list directly via cmd line // because we easily overrun the kernel's max argument/environment size @@ -57,7 +73,21 @@ func invokeUnpack(decompressedArchive io.Reader, dest string, options *archive.T return fmt.Errorf("Untar pipe failure: %v", err) } - cmd := reexec.Command("storage-untar", dest) + if root != "" { + relDest, err := filepath.Rel(root, dest) + if err != nil { + return err + } + if relDest == "." { + relDest = "/" + } + if relDest[0] != '/' { + relDest = "/" + relDest + } + dest = relDest + } + + cmd := reexec.Command("storage-untar", dest, root) cmd.Stdin = decompressedArchive cmd.ExtraFiles = append(cmd.ExtraFiles, r) @@ -68,6 +98,7 @@ func invokeUnpack(decompressedArchive io.Reader, dest string, options *archive.T if err := cmd.Start(); err != nil { return fmt.Errorf("Untar error on re-exec cmd: %v", err) } + //write the options to the pipe for the untar exec to read if err := json.NewEncoder(w).Encode(options); err != nil { return fmt.Errorf("Untar json encode to pipe failed: %v", err) @@ -84,3 +115,92 @@ func invokeUnpack(decompressedArchive io.Reader, dest string, options *archive.T } return nil } + +func tar() { + runtime.LockOSThread() + flag.Parse() + + src := flag.Arg(0) + var root string + if len(flag.Args()) > 1 { + root = flag.Arg(1) + } + + if root == "" { + root = src + } + + if err := realChroot(root); err != nil { + fatal(err) + } + + var options archive.TarOptions + if err := json.NewDecoder(os.Stdin).Decode(&options); err != nil { + fatal(err) + } + + rdr, err := archive.TarWithOptions(src, &options) + if err != nil { + fatal(err) + } + defer rdr.Close() + + if _, err := io.Copy(os.Stdout, rdr); err != nil { + fatal(err) + } + + os.Exit(0) +} + +func invokePack(srcPath string, options *archive.TarOptions, root string) (io.ReadCloser, error) { + if root == "" { + return nil, errors.New("root path must not be empty") + } + + relSrc, err := filepath.Rel(root, srcPath) + if err != nil { + return nil, err + } + if relSrc == "." { + relSrc = "/" + } + if relSrc[0] != '/' { + relSrc = "/" + relSrc + } + + // make sure we didn't trim a trailing slash with the call to `Rel` + if strings.HasSuffix(srcPath, "/") && !strings.HasSuffix(relSrc, "/") { + relSrc += "/" + } + + cmd := reexec.Command("storage-tar", relSrc, root) + + errBuff := bytes.NewBuffer(nil) + cmd.Stderr = errBuff + + tarR, tarW := io.Pipe() + cmd.Stdout = tarW + + stdin, err := cmd.StdinPipe() + if err != nil { + return nil, errors.Wrap(err, "error getting options pipe for tar process") + } + + if err := cmd.Start(); err != nil { + return nil, errors.Wrap(err, "tar error on re-exec cmd") + } + + go func() { + err := cmd.Wait() + err = errors.Wrapf(err, "error processing tar file: %s", errBuff) + tarW.CloseWithError(err) + }() + + if err := json.NewEncoder(stdin).Encode(options); err != nil { + stdin.Close() + return nil, errors.Wrap(err, "tar json encode to pipe failed") + } + stdin.Close() + + return tarR, nil +} diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/archive_windows.go b/vendor/github.com/containers/storage/pkg/chrootarchive/archive_windows.go index 93fde4220..8a5c680b1 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/archive_windows.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/archive_windows.go @@ -14,9 +14,16 @@ func chroot(path string) error { func invokeUnpack(decompressedArchive io.ReadCloser, dest string, - options *archive.TarOptions) error { + options *archive.TarOptions, root string) error { // Windows is different to Linux here because Windows does not support // chroot. Hence there is no point sandboxing a chrooted process to // do the unpack. We call inline instead within the daemon process. return archive.Unpack(decompressedArchive, longpath.AddPrefix(dest), options) } + +func invokePack(srcPath string, options *archive.TarOptions, root string) (io.ReadCloser, error) { + // Windows is different to Linux here because Windows does not support + // chroot. Hence there is no point sandboxing a chrooted process to + // do the pack. We call inline instead within the daemon process. + return archive.TarWithOptions(srcPath, options) +} diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/chroot_unix.go b/vendor/github.com/containers/storage/pkg/chrootarchive/chroot_unix.go index f9b5dece8..83278ee50 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/chroot_unix.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/chroot_unix.go @@ -4,9 +4,13 @@ package chrootarchive import "golang.org/x/sys/unix" -func chroot(path string) error { +func realChroot(path string) error { if err := unix.Chroot(path); err != nil { return err } return unix.Chdir("/") } + +func chroot(path string) error { + return realChroot(path) +} diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/init_unix.go b/vendor/github.com/containers/storage/pkg/chrootarchive/init_unix.go index 21cd87992..ea08135e4 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/init_unix.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/init_unix.go @@ -14,6 +14,7 @@ import ( func init() { reexec.Register("storage-applyLayer", applyLayer) reexec.Register("storage-untar", untar) + reexec.Register("storage-tar", tar) } func fatal(err error) { diff --git a/vendor/github.com/containers/storage/pkg/system/stat_unix.go b/vendor/github.com/containers/storage/pkg/system/stat_unix.go index 91c7d121c..f9a1b4877 100644 --- a/vendor/github.com/containers/storage/pkg/system/stat_unix.go +++ b/vendor/github.com/containers/storage/pkg/system/stat_unix.go @@ -58,3 +58,15 @@ func Stat(path string) (*StatT, error) { } return fromStatT(s) } + +// Fstat takes an open file descriptor and returns +// a system.StatT type pertaining to that file. +// +// Throws an error if the file descriptor is invalid +func Fstat(fd int) (*StatT, error) { + s := &syscall.Stat_t{} + if err := syscall.Fstat(fd, s); err != nil { + return nil, err + } + return fromStatT(s) +} diff --git a/vendor/github.com/coreos/go-systemd/NOTICE b/vendor/github.com/coreos/go-systemd/NOTICE new file mode 100644 index 000000000..23a0ada2f --- /dev/null +++ b/vendor/github.com/coreos/go-systemd/NOTICE @@ -0,0 +1,5 @@ +CoreOS Project +Copyright 2018 CoreOS, Inc + +This product includes software developed at CoreOS, Inc. +(http://www.coreos.com/). diff --git a/vendor/github.com/coreos/go-systemd/README.md b/vendor/github.com/coreos/go-systemd/README.md index cb87a1124..cad04a803 100644 --- a/vendor/github.com/coreos/go-systemd/README.md +++ b/vendor/github.com/coreos/go-systemd/README.md @@ -6,9 +6,11 @@ Go bindings to systemd. The project has several packages: - `activation` - for writing and using socket activation from Go +- `daemon` - for notifying systemd of service status changes - `dbus` - for starting/stopping/inspecting running services and units - `journal` - for writing to systemd's logging service, journald - `sdjournal` - for reading from journald by wrapping its C API +- `login1` - for integration with the systemd logind API - `machine1` - for registering machines/containers with systemd - `unit` - for (de)serialization and comparison of unit files @@ -18,10 +20,9 @@ An example HTTP server using socket activation can be quickly set up by followin https://github.com/coreos/go-systemd/tree/master/examples/activation/httpserver -## Journal +## systemd Service Notification -Using the pure-Go `journal` package you can submit journal entries directly to systemd's journal, taking advantage of features like indexed key/value pairs for each log entry. -The `sdjournal` package provides read access to the journal by wrapping around journald's native C API; consequently it requires cgo and the journal headers to be available. +The `daemon` package is an implementation of the [sd_notify protocol](https://www.freedesktop.org/software/systemd/man/sd_notify.html#Description). It can be used to inform systemd of service start-up completion, watchdog events, and other status changes. ## D-Bus @@ -45,6 +46,20 @@ Create `/etc/dbus-1/system-local.conf` that looks like this: </busconfig> ``` +## Journal + +### Writing to the Journal + +Using the pure-Go `journal` package you can submit journal entries directly to systemd's journal, taking advantage of features like indexed key/value pairs for each log entry. + +### Reading from the Journal + +The `sdjournal` package provides read access to the journal by wrapping around journald's native C API; consequently it requires cgo and the journal headers to be available. + +## logind + +The `login1` package provides functions to integrate with the [systemd logind API](http://www.freedesktop.org/wiki/Software/systemd/logind/). + ## machined The `machine1` package allows interaction with the [systemd machined D-Bus API](http://www.freedesktop.org/wiki/Software/systemd/machined/). diff --git a/vendor/github.com/coreos/go-systemd/activation/files.go b/vendor/github.com/coreos/go-systemd/activation/files.go index c8e85fcd5..29dd18def 100644 --- a/vendor/github.com/coreos/go-systemd/activation/files.go +++ b/vendor/github.com/coreos/go-systemd/activation/files.go @@ -18,18 +18,26 @@ package activation import ( "os" "strconv" + "strings" "syscall" ) -// based on: https://gist.github.com/alberts/4640792 const ( + // listenFdsStart corresponds to `SD_LISTEN_FDS_START`. listenFdsStart = 3 ) +// Files returns a slice containing a `os.File` object for each +// file descriptor passed to this process via systemd fd-passing protocol. +// +// The order of the file descriptors is preserved in the returned slice. +// `unsetEnv` is typically set to `true` in order to avoid clashes in +// fd usage and to avoid leaking environment flags to child processes. func Files(unsetEnv bool) []*os.File { if unsetEnv { defer os.Unsetenv("LISTEN_PID") defer os.Unsetenv("LISTEN_FDS") + defer os.Unsetenv("LISTEN_FDNAMES") } pid, err := strconv.Atoi(os.Getenv("LISTEN_PID")) @@ -42,10 +50,17 @@ func Files(unsetEnv bool) []*os.File { return nil } + names := strings.Split(os.Getenv("LISTEN_FDNAMES"), ":") + files := make([]*os.File, 0, nfds) for fd := listenFdsStart; fd < listenFdsStart+nfds; fd++ { syscall.CloseOnExec(fd) - files = append(files, os.NewFile(uintptr(fd), "LISTEN_FD_"+strconv.Itoa(fd))) + name := "LISTEN_FD_" + strconv.Itoa(fd) + offset := fd - listenFdsStart + if offset < len(names) && len(names[offset]) > 0 { + name = names[offset] + } + files = append(files, os.NewFile(uintptr(fd), name)) } return files diff --git a/vendor/github.com/coreos/go-systemd/activation/listeners.go b/vendor/github.com/coreos/go-systemd/activation/listeners.go index fd5dfc709..bb5cc2311 100644 --- a/vendor/github.com/coreos/go-systemd/activation/listeners.go +++ b/vendor/github.com/coreos/go-systemd/activation/listeners.go @@ -25,13 +25,33 @@ import ( // The order of the file descriptors is preserved in the returned slice. // Nil values are used to fill any gaps. For example if systemd were to return file descriptors // corresponding with "udp, tcp, tcp", then the slice would contain {nil, net.Listener, net.Listener} -func Listeners(unsetEnv bool) ([]net.Listener, error) { - files := Files(unsetEnv) +func Listeners() ([]net.Listener, error) { + files := Files(true) listeners := make([]net.Listener, len(files)) for i, f := range files { if pc, err := net.FileListener(f); err == nil { listeners[i] = pc + f.Close() + } + } + return listeners, nil +} + +// ListenersWithNames maps a listener name to a set of net.Listener instances. +func ListenersWithNames() (map[string][]net.Listener, error) { + files := Files(true) + listeners := map[string][]net.Listener{} + + for _, f := range files { + if pc, err := net.FileListener(f); err == nil { + current, ok := listeners[f.Name()] + if !ok { + listeners[f.Name()] = []net.Listener{pc} + } else { + listeners[f.Name()] = append(current, pc) + } + f.Close() } } return listeners, nil @@ -40,8 +60,8 @@ func Listeners(unsetEnv bool) ([]net.Listener, error) { // TLSListeners returns a slice containing a net.listener for each matching TCP socket type // passed to this process. // It uses default Listeners func and forces TCP sockets handlers to use TLS based on tlsConfig. -func TLSListeners(unsetEnv bool, tlsConfig *tls.Config) ([]net.Listener, error) { - listeners, err := Listeners(unsetEnv) +func TLSListeners(tlsConfig *tls.Config) ([]net.Listener, error) { + listeners, err := Listeners() if listeners == nil || err != nil { return nil, err @@ -58,3 +78,26 @@ func TLSListeners(unsetEnv bool, tlsConfig *tls.Config) ([]net.Listener, error) return listeners, err } + +// TLSListenersWithNames maps a listener name to a net.Listener with +// the associated TLS configuration. +func TLSListenersWithNames(tlsConfig *tls.Config) (map[string][]net.Listener, error) { + listeners, err := ListenersWithNames() + + if listeners == nil || err != nil { + return nil, err + } + + if tlsConfig != nil && err == nil { + for _, ll := range listeners { + // Activate TLS only for TCP sockets + for i, l := range ll { + if l.Addr().Network() == "tcp" { + ll[i] = tls.NewListener(l, tlsConfig) + } + } + } + } + + return listeners, err +} diff --git a/vendor/github.com/coreos/go-systemd/activation/packetconns.go b/vendor/github.com/coreos/go-systemd/activation/packetconns.go index 48b2ca029..a97206785 100644 --- a/vendor/github.com/coreos/go-systemd/activation/packetconns.go +++ b/vendor/github.com/coreos/go-systemd/activation/packetconns.go @@ -24,13 +24,14 @@ import ( // The order of the file descriptors is preserved in the returned slice. // Nil values are used to fill any gaps. For example if systemd were to return file descriptors // corresponding with "udp, tcp, udp", then the slice would contain {net.PacketConn, nil, net.PacketConn} -func PacketConns(unsetEnv bool) ([]net.PacketConn, error) { - files := Files(unsetEnv) +func PacketConns() ([]net.PacketConn, error) { + files := Files(true) conns := make([]net.PacketConn, len(files)) for i, f := range files { if pc, err := net.FilePacketConn(f); err == nil { conns[i] = pc + f.Close() } } return conns, nil diff --git a/vendor/github.com/coreos/go-systemd/dbus/dbus.go b/vendor/github.com/coreos/go-systemd/dbus/dbus.go index c1694fb52..1d54810af 100644 --- a/vendor/github.com/coreos/go-systemd/dbus/dbus.go +++ b/vendor/github.com/coreos/go-systemd/dbus/dbus.go @@ -16,6 +16,7 @@ package dbus import ( + "encoding/hex" "fmt" "os" "strconv" @@ -60,6 +61,27 @@ func PathBusEscape(path string) string { return string(n) } +// pathBusUnescape is the inverse of PathBusEscape. +func pathBusUnescape(path string) string { + if path == "_" { + return "" + } + n := []byte{} + for i := 0; i < len(path); i++ { + c := path[i] + if c == '_' && i+2 < len(path) { + res, err := hex.DecodeString(path[i+1 : i+3]) + if err == nil { + n = append(n, res...) + } + i += 2 + } else { + n = append(n, c) + } + } + return string(n) +} + // Conn is a connection to systemd's dbus endpoint. type Conn struct { // sysconn/sysobj are only used to call dbus methods @@ -74,13 +96,18 @@ type Conn struct { jobs map[dbus.ObjectPath]chan<- string sync.Mutex } - subscriber struct { + subStateSubscriber struct { updateCh chan<- *SubStateUpdate errCh chan<- error sync.Mutex ignore map[dbus.ObjectPath]int64 cleanIgnore int64 } + propertiesSubscriber struct { + updateCh chan<- *PropertiesUpdate + errCh chan<- error + sync.Mutex + } } // New establishes a connection to any available bus and authenticates. @@ -152,7 +179,7 @@ func NewConnection(dialBus func() (*dbus.Conn, error)) (*Conn, error) { sigobj: systemdObject(sigconn), } - c.subscriber.ignore = make(map[dbus.ObjectPath]int64) + c.subStateSubscriber.ignore = make(map[dbus.ObjectPath]int64) c.jobListener.jobs = make(map[dbus.ObjectPath]chan<- string) // Setup the listeners on jobs so that we can get completions diff --git a/vendor/github.com/coreos/go-systemd/dbus/methods.go b/vendor/github.com/coreos/go-systemd/dbus/methods.go index ab17f7cc7..0b4207229 100644 --- a/vendor/github.com/coreos/go-systemd/dbus/methods.go +++ b/vendor/github.com/coreos/go-systemd/dbus/methods.go @@ -1,4 +1,4 @@ -// Copyright 2015 CoreOS, Inc. +// Copyright 2015, 2018 CoreOS, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ package dbus import ( "errors" + "fmt" "path" "strconv" @@ -148,14 +149,27 @@ func (c *Conn) ResetFailedUnit(name string) error { return c.sysobj.Call("org.freedesktop.systemd1.Manager.ResetFailedUnit", 0, name).Store() } -// getProperties takes the unit name and returns all of its dbus object properties, for the given dbus interface -func (c *Conn) getProperties(unit string, dbusInterface string) (map[string]interface{}, error) { +// SystemState returns the systemd state. Equivalent to `systemctl is-system-running`. +func (c *Conn) SystemState() (*Property, error) { + var err error + var prop dbus.Variant + + obj := c.sysconn.Object("org.freedesktop.systemd1", "/org/freedesktop/systemd1") + err = obj.Call("org.freedesktop.DBus.Properties.Get", 0, "org.freedesktop.systemd1.Manager", "SystemState").Store(&prop) + if err != nil { + return nil, err + } + + return &Property{Name: "SystemState", Value: prop}, nil +} + +// getProperties takes the unit path and returns all of its dbus object properties, for the given dbus interface +func (c *Conn) getProperties(path dbus.ObjectPath, dbusInterface string) (map[string]interface{}, error) { var err error var props map[string]dbus.Variant - path := unitPath(unit) if !path.IsValid() { - return nil, errors.New("invalid unit name: " + unit) + return nil, fmt.Errorf("invalid unit name: %v", path) } obj := c.sysconn.Object("org.freedesktop.systemd1", path) @@ -172,9 +186,15 @@ func (c *Conn) getProperties(unit string, dbusInterface string) (map[string]inte return out, nil } -// GetUnitProperties takes the unit name and returns all of its dbus object properties. +// GetUnitProperties takes the (unescaped) unit name and returns all of its dbus object properties. func (c *Conn) GetUnitProperties(unit string) (map[string]interface{}, error) { - return c.getProperties(unit, "org.freedesktop.systemd1.Unit") + path := unitPath(unit) + return c.getProperties(path, "org.freedesktop.systemd1.Unit") +} + +// GetUnitProperties takes the (escaped) unit path and returns all of its dbus object properties. +func (c *Conn) GetUnitPathProperties(path dbus.ObjectPath) (map[string]interface{}, error) { + return c.getProperties(path, "org.freedesktop.systemd1.Unit") } func (c *Conn) getProperty(unit string, dbusInterface string, propertyName string) (*Property, error) { @@ -208,7 +228,8 @@ func (c *Conn) GetServiceProperty(service string, propertyName string) (*Propert // Valid values for unitType: Service, Socket, Target, Device, Mount, Automount, Snapshot, Timer, Swap, Path, Slice, Scope // return "dbus.Error: Unknown interface" if the unitType is not the correct type of the unit func (c *Conn) GetUnitTypeProperties(unit string, unitType string) (map[string]interface{}, error) { - return c.getProperties(unit, "org.freedesktop.systemd1."+unitType) + path := unitPath(unit) + return c.getProperties(path, "org.freedesktop.systemd1."+unitType) } // SetUnitProperties() may be used to modify certain unit properties at runtime. @@ -292,6 +313,7 @@ func (c *Conn) ListUnitsByPatterns(states []string, patterns []string) ([]UnitSt // names and returns an UnitStatus array. Comparing to ListUnitsByPatterns // method, this method returns statuses even for inactive or non-existing // units. Input array should contain exact unit names, but not patterns. +// Note: Requires systemd v230 or higher func (c *Conn) ListUnitsByNames(units []string) ([]UnitStatus, error) { return c.listUnitsInternal(c.sysobj.Call("org.freedesktop.systemd1.Manager.ListUnitsByNames", 0, units).Store) } @@ -563,3 +585,8 @@ func (c *Conn) Reload() error { func unitPath(name string) dbus.ObjectPath { return dbus.ObjectPath("/org/freedesktop/systemd1/unit/" + PathBusEscape(name)) } + +// unitName returns the unescaped base element of the supplied escaped path +func unitName(dpath dbus.ObjectPath) string { + return pathBusUnescape(path.Base(string(dpath))) +} diff --git a/vendor/github.com/coreos/go-systemd/dbus/set.go b/vendor/github.com/coreos/go-systemd/dbus/set.go index f92e6fbed..17c5d4856 100644 --- a/vendor/github.com/coreos/go-systemd/dbus/set.go +++ b/vendor/github.com/coreos/go-systemd/dbus/set.go @@ -36,7 +36,7 @@ func (s *set) Length() int { } func (s *set) Values() (values []string) { - for val, _ := range s.data { + for val := range s.data { values = append(values, val) } return diff --git a/vendor/github.com/coreos/go-systemd/dbus/subscription.go b/vendor/github.com/coreos/go-systemd/dbus/subscription.go index 996451445..70e63a6f1 100644 --- a/vendor/github.com/coreos/go-systemd/dbus/subscription.go +++ b/vendor/github.com/coreos/go-systemd/dbus/subscription.go @@ -16,6 +16,7 @@ package dbus import ( "errors" + "log" "time" "github.com/godbus/dbus" @@ -36,22 +37,12 @@ func (c *Conn) Subscribe() error { c.sigconn.BusObject().Call("org.freedesktop.DBus.AddMatch", 0, "type='signal',interface='org.freedesktop.DBus.Properties',member='PropertiesChanged'") - err := c.sigobj.Call("org.freedesktop.systemd1.Manager.Subscribe", 0).Store() - if err != nil { - return err - } - - return nil + return c.sigobj.Call("org.freedesktop.systemd1.Manager.Subscribe", 0).Store() } // Unsubscribe this connection from systemd dbus events. func (c *Conn) Unsubscribe() error { - err := c.sigobj.Call("org.freedesktop.systemd1.Manager.Unsubscribe", 0).Store() - if err != nil { - return err - } - - return nil + return c.sigobj.Call("org.freedesktop.systemd1.Manager.Unsubscribe", 0).Store() } func (c *Conn) dispatch() { @@ -70,7 +61,8 @@ func (c *Conn) dispatch() { c.jobComplete(signal) } - if c.subscriber.updateCh == nil { + if c.subStateSubscriber.updateCh == nil && + c.propertiesSubscriber.updateCh == nil { continue } @@ -84,6 +76,12 @@ func (c *Conn) dispatch() { case "org.freedesktop.DBus.Properties.PropertiesChanged": if signal.Body[0].(string) == "org.freedesktop.systemd1.Unit" { unitPath = signal.Path + + if len(signal.Body) >= 2 { + if changed, ok := signal.Body[1].(map[string]dbus.Variant); ok { + c.sendPropertiesUpdate(unitPath, changed) + } + } } } @@ -169,42 +167,80 @@ type SubStateUpdate struct { // is full, it attempts to write an error to errCh; if errCh is full, the error // passes silently. func (c *Conn) SetSubStateSubscriber(updateCh chan<- *SubStateUpdate, errCh chan<- error) { - c.subscriber.Lock() - defer c.subscriber.Unlock() - c.subscriber.updateCh = updateCh - c.subscriber.errCh = errCh + if c == nil { + msg := "nil receiver" + select { + case errCh <- errors.New(msg): + default: + log.Printf("full error channel while reporting: %s\n", msg) + } + return + } + + c.subStateSubscriber.Lock() + defer c.subStateSubscriber.Unlock() + c.subStateSubscriber.updateCh = updateCh + c.subStateSubscriber.errCh = errCh } -func (c *Conn) sendSubStateUpdate(path dbus.ObjectPath) { - c.subscriber.Lock() - defer c.subscriber.Unlock() +func (c *Conn) sendSubStateUpdate(unitPath dbus.ObjectPath) { + c.subStateSubscriber.Lock() + defer c.subStateSubscriber.Unlock() + + if c.subStateSubscriber.updateCh == nil { + return + } - if c.shouldIgnore(path) { + isIgnored := c.shouldIgnore(unitPath) + defer c.cleanIgnore() + if isIgnored { return } - info, err := c.GetUnitProperties(string(path)) + info, err := c.GetUnitPathProperties(unitPath) if err != nil { select { - case c.subscriber.errCh <- err: + case c.subStateSubscriber.errCh <- err: default: + log.Printf("full error channel while reporting: %s\n", err) } + return } + defer c.updateIgnore(unitPath, info) - name := info["Id"].(string) - substate := info["SubState"].(string) + name, ok := info["Id"].(string) + if !ok { + msg := "failed to cast info.Id" + select { + case c.subStateSubscriber.errCh <- errors.New(msg): + default: + log.Printf("full error channel while reporting: %s\n", err) + } + return + } + substate, ok := info["SubState"].(string) + if !ok { + msg := "failed to cast info.SubState" + select { + case c.subStateSubscriber.errCh <- errors.New(msg): + default: + log.Printf("full error channel while reporting: %s\n", msg) + } + return + } update := &SubStateUpdate{name, substate} select { - case c.subscriber.updateCh <- update: + case c.subStateSubscriber.updateCh <- update: default: + msg := "update channel is full" select { - case c.subscriber.errCh <- errors.New("update channel full!"): + case c.subStateSubscriber.errCh <- errors.New(msg): default: + log.Printf("full error channel while reporting: %s\n", msg) } + return } - - c.updateIgnore(path, info) } // The ignore functions work around a wart in the systemd dbus interface. @@ -222,29 +258,76 @@ func (c *Conn) sendSubStateUpdate(path dbus.ObjectPath) { // the properties). func (c *Conn) shouldIgnore(path dbus.ObjectPath) bool { - t, ok := c.subscriber.ignore[path] + t, ok := c.subStateSubscriber.ignore[path] return ok && t >= time.Now().UnixNano() } func (c *Conn) updateIgnore(path dbus.ObjectPath, info map[string]interface{}) { - c.cleanIgnore() + loadState, ok := info["LoadState"].(string) + if !ok { + return + } // unit is unloaded - it will trigger bad systemd dbus behavior - if info["LoadState"].(string) == "not-found" { - c.subscriber.ignore[path] = time.Now().UnixNano() + ignoreInterval + if loadState == "not-found" { + c.subStateSubscriber.ignore[path] = time.Now().UnixNano() + ignoreInterval } } // without this, ignore would grow unboundedly over time func (c *Conn) cleanIgnore() { now := time.Now().UnixNano() - if c.subscriber.cleanIgnore < now { - c.subscriber.cleanIgnore = now + cleanIgnoreInterval + if c.subStateSubscriber.cleanIgnore < now { + c.subStateSubscriber.cleanIgnore = now + cleanIgnoreInterval - for p, t := range c.subscriber.ignore { + for p, t := range c.subStateSubscriber.ignore { if t < now { - delete(c.subscriber.ignore, p) + delete(c.subStateSubscriber.ignore, p) } } } } + +// PropertiesUpdate holds a map of a unit's changed properties +type PropertiesUpdate struct { + UnitName string + Changed map[string]dbus.Variant +} + +// SetPropertiesSubscriber writes to updateCh when any unit's properties +// change. Every property change reported by systemd will be sent; that is, no +// transitions will be "missed" (as they might be with SetSubStateSubscriber). +// However, state changes will only be written to the channel with non-blocking +// writes. If updateCh is full, it attempts to write an error to errCh; if +// errCh is full, the error passes silently. +func (c *Conn) SetPropertiesSubscriber(updateCh chan<- *PropertiesUpdate, errCh chan<- error) { + c.propertiesSubscriber.Lock() + defer c.propertiesSubscriber.Unlock() + c.propertiesSubscriber.updateCh = updateCh + c.propertiesSubscriber.errCh = errCh +} + +// we don't need to worry about shouldIgnore() here because +// sendPropertiesUpdate doesn't call GetProperties() +func (c *Conn) sendPropertiesUpdate(unitPath dbus.ObjectPath, changedProps map[string]dbus.Variant) { + c.propertiesSubscriber.Lock() + defer c.propertiesSubscriber.Unlock() + + if c.propertiesSubscriber.updateCh == nil { + return + } + + update := &PropertiesUpdate{unitName(unitPath), changedProps} + + select { + case c.propertiesSubscriber.updateCh <- update: + default: + msg := "update channel is full" + select { + case c.propertiesSubscriber.errCh <- errors.New(msg): + default: + log.Printf("full error channel while reporting: %s\n", msg) + } + return + } +} diff --git a/vendor/github.com/coreos/go-systemd/journal/journal.go b/vendor/github.com/coreos/go-systemd/journal/journal.go index 7f434990d..ef85a3ba2 100644 --- a/vendor/github.com/coreos/go-systemd/journal/journal.go +++ b/vendor/github.com/coreos/go-systemd/journal/journal.go @@ -103,7 +103,10 @@ func Send(message string, priority Priority, vars map[string]string) error { if !ok { return journalError("can't send file through non-Unix connection") } - unixConn.WriteMsgUnix([]byte{}, rights, nil) + _, _, err = unixConn.WriteMsgUnix([]byte{}, rights, nil) + if err != nil { + return journalError(err.Error()) + } } else if err != nil { return journalError(err.Error()) } @@ -165,7 +168,7 @@ func tempFd() (*os.File, error) { if err != nil { return nil, err } - syscall.Unlink(file.Name()) + err = syscall.Unlink(file.Name()) if err != nil { return nil, err } diff --git a/vendor/github.com/coreos/go-systemd/sdjournal/journal.go b/vendor/github.com/coreos/go-systemd/sdjournal/journal.go index b00d606c1..9f3d92342 100644 --- a/vendor/github.com/coreos/go-systemd/sdjournal/journal.go +++ b/vendor/github.com/coreos/go-systemd/sdjournal/journal.go @@ -47,6 +47,15 @@ package sdjournal // return sd_journal_open_directory(ret, path, flags); // } // +// int +// my_sd_journal_open_files(void *f, sd_journal **ret, const char **paths, int flags) +// { +// int (*sd_journal_open_files)(sd_journal **, const char **, int); +// +// sd_journal_open_files = f; +// return sd_journal_open_files(ret, paths, flags); +// } +// // void // my_sd_journal_close(void *f, sd_journal *j) // { @@ -282,9 +291,19 @@ package sdjournal // sd_journal_restart_unique(j); // } // +// int +// my_sd_journal_get_catalog(void *f, sd_journal *j, char **ret) +// { +// int(*sd_journal_get_catalog)(sd_journal *, char **); +// +// sd_journal_get_catalog = f; +// return sd_journal_get_catalog(j, ret); +// } +// import "C" import ( "bytes" + "errors" "fmt" "strings" "sync" @@ -352,6 +371,12 @@ const ( IndefiniteWait time.Duration = 1<<63 - 1 ) +var ( + // ErrNoTestCursor gets returned when using TestCursor function and cursor + // parameter is not the same as the current cursor position. + ErrNoTestCursor = errors.New("Cursor parameter is not the same as current position") +) + // Journal is a Go wrapper of an sd_journal structure. type Journal struct { cjournal *C.sd_journal @@ -396,8 +421,7 @@ func NewJournal() (j *Journal, err error) { } // NewJournalFromDir returns a new Journal instance pointing to a journal residing -// in a given directory. The supplied path may be relative or absolute; if -// relative, it will be converted to an absolute path before being opened. +// in a given directory. func NewJournalFromDir(path string) (j *Journal, err error) { j = &Journal{} @@ -417,6 +441,32 @@ func NewJournalFromDir(path string) (j *Journal, err error) { return j, nil } +// NewJournalFromFiles returns a new Journal instance pointing to a journals residing +// in a given files. +func NewJournalFromFiles(paths ...string) (j *Journal, err error) { + j = &Journal{} + + sd_journal_open_files, err := getFunction("sd_journal_open_files") + if err != nil { + return nil, err + } + + // by making the slice 1 elem too long, we guarantee it'll be null-terminated + cPaths := make([]*C.char, len(paths)+1) + for idx, path := range paths { + p := C.CString(path) + cPaths[idx] = p + defer C.free(unsafe.Pointer(p)) + } + + r := C.my_sd_journal_open_files(sd_journal_open_files, &j.cjournal, &cPaths[0], 0) + if r < 0 { + return nil, fmt.Errorf("failed to open journals in paths %q: %d", paths, syscall.Errno(-r)) + } + + return j, nil +} + // Close closes a journal opened with NewJournal. func (j *Journal) Close() error { sd_journal_close, err := getFunction("sd_journal_close") @@ -598,7 +648,8 @@ func (j *Journal) getData(field string) (unsafe.Pointer, C.int, error) { } // GetData gets the data object associated with a specific field from the -// current journal entry. +// the journal entry referenced by the last completed Next/Previous function +// call. To call GetData, you must have first called one of these functions. func (j *Journal) GetData(field string) (string, error) { d, l, err := j.getData(field) if err != nil { @@ -609,7 +660,9 @@ func (j *Journal) GetData(field string) (string, error) { } // GetDataValue gets the data object associated with a specific field from the -// current journal entry, returning only the value of the object. +// journal entry referenced by the last completed Next/Previous function call, +// returning only the value of the object. To call GetDataValue, you must first +// have called one of the Next/Previous functions. func (j *Journal) GetDataValue(field string) (string, error) { val, err := j.GetData(field) if err != nil { @@ -620,7 +673,8 @@ func (j *Journal) GetDataValue(field string) (string, error) { } // GetDataBytes gets the data object associated with a specific field from the -// current journal entry. +// journal entry referenced by the last completed Next/Previous function call. +// To call GetDataBytes, you must first have called one of these functions. func (j *Journal) GetDataBytes(field string) ([]byte, error) { d, l, err := j.getData(field) if err != nil { @@ -631,7 +685,9 @@ func (j *Journal) GetDataBytes(field string) ([]byte, error) { } // GetDataValueBytes gets the data object associated with a specific field from the -// current journal entry, returning only the value of the object. +// journal entry referenced by the last completed Next/Previous function call, +// returning only the value of the object. To call GetDataValueBytes, you must first +// have called one of the Next/Previous functions. func (j *Journal) GetDataValueBytes(field string) ([]byte, error) { val, err := j.GetDataBytes(field) if err != nil { @@ -641,9 +697,10 @@ func (j *Journal) GetDataValueBytes(field string) ([]byte, error) { return bytes.SplitN(val, []byte("="), 2)[1], nil } -// GetEntry returns a full representation of a journal entry with -// all key-value pairs of data as well as address fields (cursor, realtime -// timestamp and monotonic timestamp) +// GetEntry returns a full representation of the journal entry referenced by the +// last completed Next/Previous function call, with all key-value pairs of data +// as well as address fields (cursor, realtime timestamp and monotonic timestamp). +// To call GetEntry, you must first have called one of the Next/Previous functions. func (j *Journal) GetEntry() (*JournalEntry, error) { sd_journal_get_realtime_usec, err := getFunction("sd_journal_get_realtime_usec") if err != nil { @@ -731,7 +788,7 @@ func (j *Journal) GetEntry() (*JournalEntry, error) { return entry, nil } -// SetDataThresold sets the data field size threshold for data returned by +// SetDataThreshold sets the data field size threshold for data returned by // GetData. To retrieve the complete data fields this threshold should be // turned off by setting it to 0, so that the library always returns the // complete data objects. @@ -752,8 +809,10 @@ func (j *Journal) SetDataThreshold(threshold uint64) error { return nil } -// GetRealtimeUsec gets the realtime (wallclock) timestamp of the current -// journal entry. +// GetRealtimeUsec gets the realtime (wallclock) timestamp of the journal +// entry referenced by the last completed Next/Previous function call. To +// call GetRealtimeUsec, you must first have called one of the Next/Previous +// functions. func (j *Journal) GetRealtimeUsec() (uint64, error) { var usec C.uint64_t @@ -773,7 +832,10 @@ func (j *Journal) GetRealtimeUsec() (uint64, error) { return uint64(usec), nil } -// GetMonotonicUsec gets the monotonic timestamp of the current journal entry. +// GetMonotonicUsec gets the monotonic timestamp of the journal entry +// referenced by the last completed Next/Previous function call. To call +// GetMonotonicUsec, you must first have called one of the Next/Previous +// functions. func (j *Journal) GetMonotonicUsec() (uint64, error) { var usec C.uint64_t var boot_id C.sd_id128_t @@ -794,7 +856,9 @@ func (j *Journal) GetMonotonicUsec() (uint64, error) { return uint64(usec), nil } -// GetCursor gets the cursor of the current journal entry. +// GetCursor gets the cursor of the last journal entry reeferenced by the +// last completed Next/Previous function call. To call GetCursor, you must +// first have called one of the Next/Previous functions. func (j *Journal) GetCursor() (string, error) { sd_journal_get_cursor, err := getFunction("sd_journal_get_cursor") if err != nil { @@ -836,13 +900,16 @@ func (j *Journal) TestCursor(cursor string) error { if r < 0 { return fmt.Errorf("failed to test to cursor %q: %d", cursor, syscall.Errno(-r)) + } else if r == 0 { + return ErrNoTestCursor } return nil } // SeekHead seeks to the beginning of the journal, i.e. the oldest available -// entry. +// entry. This call must be followed by a call to Next before any call to +// Get* will return data about the first element. func (j *Journal) SeekHead() error { sd_journal_seek_head, err := getFunction("sd_journal_seek_head") if err != nil { @@ -861,7 +928,8 @@ func (j *Journal) SeekHead() error { } // SeekTail may be used to seek to the end of the journal, i.e. the most recent -// available entry. +// available entry. This call must be followed by a call to Next before any +// call to Get* will return data about the last element. func (j *Journal) SeekTail() error { sd_journal_seek_tail, err := getFunction("sd_journal_seek_tail") if err != nil { @@ -880,7 +948,8 @@ func (j *Journal) SeekTail() error { } // SeekRealtimeUsec seeks to the entry with the specified realtime (wallclock) -// timestamp, i.e. CLOCK_REALTIME. +// timestamp, i.e. CLOCK_REALTIME. This call must be followed by a call to +// Next/Previous before any call to Get* will return data about the sought entry. func (j *Journal) SeekRealtimeUsec(usec uint64) error { sd_journal_seek_realtime_usec, err := getFunction("sd_journal_seek_realtime_usec") if err != nil { @@ -898,7 +967,9 @@ func (j *Journal) SeekRealtimeUsec(usec uint64) error { return nil } -// SeekCursor seeks to a concrete journal cursor. +// SeekCursor seeks to a concrete journal cursor. This call must be +// followed by a call to Next/Previous before any call to Get* will return +// data about the sought entry. func (j *Journal) SeekCursor(cursor string) error { sd_journal_seek_cursor, err := getFunction("sd_journal_seek_cursor") if err != nil { @@ -937,7 +1008,7 @@ func (j *Journal) Wait(timeout time.Duration) int { // equivalent hex value. to = 0xffffffffffffffff } else { - to = uint64(time.Now().Add(timeout).Unix() / 1000) + to = uint64(timeout / time.Microsecond) } j.mu.Lock() r := C.my_sd_journal_wait(sd_journal_wait, j.cjournal, C.uint64_t(to)) @@ -1022,3 +1093,28 @@ func (j *Journal) GetUniqueValues(field string) ([]string, error) { return result, nil } + +// GetCatalog retrieves a message catalog entry for the journal entry referenced +// by the last completed Next/Previous function call. To call GetCatalog, you +// must first have called one of these functions. +func (j *Journal) GetCatalog() (string, error) { + sd_journal_get_catalog, err := getFunction("sd_journal_get_catalog") + if err != nil { + return "", err + } + + var c *C.char + + j.mu.Lock() + r := C.my_sd_journal_get_catalog(sd_journal_get_catalog, j.cjournal, &c) + j.mu.Unlock() + defer C.free(unsafe.Pointer(c)) + + if r < 0 { + return "", fmt.Errorf("failed to retrieve catalog entry for current journal entry: %d", syscall.Errno(-r)) + } + + catalog := C.GoString(c) + + return catalog, nil +} diff --git a/vendor/github.com/coreos/go-systemd/sdjournal/read.go b/vendor/github.com/coreos/go-systemd/sdjournal/read.go index b581f03b4..51a060fb5 100644 --- a/vendor/github.com/coreos/go-systemd/sdjournal/read.go +++ b/vendor/github.com/coreos/go-systemd/sdjournal/read.go @@ -21,10 +21,13 @@ import ( "io" "log" "strings" + "sync" "time" ) var ( + // ErrExpired gets returned when the Follow function runs into the + // specified timeout. ErrExpired = errors.New("Timeout expired") ) @@ -44,6 +47,11 @@ type JournalReaderConfig struct { // If not empty, the journal instance will point to a journal residing // in this directory. The supplied path may be relative or absolute. Path string + + // If not nil, Formatter will be used to translate the resulting entries + // into strings. If not set, the default format (timestamp and message field) + // will be used. If Formatter returns an error, Read will stop and return the error. + Formatter func(entry *JournalEntry) (string, error) } // JournalReader is an io.ReadCloser which provides a simple interface for iterating through the @@ -51,12 +59,20 @@ type JournalReaderConfig struct { type JournalReader struct { journal *Journal msgReader *strings.Reader + formatter func(entry *JournalEntry) (string, error) } // NewJournalReader creates a new JournalReader with configuration options that are similar to the // systemd journalctl tool's iteration and filtering features. func NewJournalReader(config JournalReaderConfig) (*JournalReader, error) { - r := &JournalReader{} + // use simpleMessageFormatter as default formatter. + if config.Formatter == nil { + config.Formatter = simpleMessageFormatter + } + + r := &JournalReader{ + formatter: config.Formatter, + } // Open the journal var err error @@ -71,7 +87,9 @@ func NewJournalReader(config JournalReaderConfig) (*JournalReader, error) { // Add any supplied matches for _, m := range config.Matches { - r.journal.AddMatch(m.String()) + if err = r.journal.AddMatch(m.String()); err != nil { + return nil, err + } } // Set the start position based on options @@ -118,14 +136,10 @@ func NewJournalReader(config JournalReaderConfig) (*JournalReader, error) { // don't fit in the read buffer. Callers should keep calling until 0 and/or an // error is returned. func (r *JournalReader) Read(b []byte) (int, error) { - var err error - if r.msgReader == nil { - var c uint64 - // Advance the journal cursor. It has to be called at least one time // before reading - c, err = r.journal.Next() + c, err := r.journal.Next() // An unexpected error if err != nil { @@ -137,10 +151,13 @@ func (r *JournalReader) Read(b []byte) (int, error) { return 0, io.EOF } - // Build a message - var msg string - msg, err = r.buildMessage() + entry, err := r.journal.GetEntry() + if err != nil { + return 0, err + } + // Build a message + msg, err := r.formatter(entry) if err != nil { return 0, err } @@ -148,8 +165,7 @@ func (r *JournalReader) Read(b []byte) (int, error) { } // Copy and return the message - var sz int - sz, err = r.msgReader.Read(b) + sz, err := r.msgReader.Read(b) if err == io.EOF { // The current entry has been fully read. Don't propagate this // EOF, so the next entry can be read at the next Read() @@ -180,80 +196,76 @@ func (r *JournalReader) Rewind() error { // Follow synchronously follows the JournalReader, writing each new journal entry to writer. The // follow will continue until a single time.Time is received on the until channel. -func (r *JournalReader) Follow(until <-chan time.Time, writer io.Writer) (err error) { +func (r *JournalReader) Follow(until <-chan time.Time, writer io.Writer) error { // Process journal entries and events. Entries are flushed until the tail or // timeout is reached, and then we wait for new events or the timeout. var msg = make([]byte, 64*1<<(10)) + var waitCh = make(chan int, 1) + var waitGroup sync.WaitGroup + defer waitGroup.Wait() + process: for { c, err := r.Read(msg) if err != nil && err != io.EOF { - break process + return err } select { case <-until: return ErrExpired default: - if c > 0 { - if _, err = writer.Write(msg[:c]); err != nil { - break process - } - continue process + } + if c > 0 { + if _, err = writer.Write(msg[:c]); err != nil { + return err } + continue process } // We're at the tail, so wait for new events or time out. // Holds journal events to process. Tightly bounded for now unless there's a // reason to unblock the journal watch routine more quickly. - events := make(chan int, 1) - pollDone := make(chan bool, 1) - go func() { - for { - select { - case <-pollDone: - return + for { + waitGroup.Add(1) + go func() { + status := r.journal.Wait(100 * time.Millisecond) + waitCh <- status + waitGroup.Done() + }() + + select { + case <-until: + return ErrExpired + case e := <-waitCh: + switch e { + case SD_JOURNAL_NOP: + // the journal did not change since the last invocation + case SD_JOURNAL_APPEND, SD_JOURNAL_INVALIDATE: + continue process default: - events <- r.journal.Wait(time.Duration(1) * time.Second) - } - } - }() + if e < 0 { + return fmt.Errorf("received error event: %d", e) + } - select { - case <-until: - pollDone <- true - return ErrExpired - case e := <-events: - pollDone <- true - switch e { - case SD_JOURNAL_NOP, SD_JOURNAL_APPEND, SD_JOURNAL_INVALIDATE: - // TODO: need to account for any of these? - default: - log.Printf("Received unknown event: %d\n", e) + log.Printf("received unknown event: %d\n", e) + } } - continue process } } - - return } -// buildMessage returns a string representing the current journal entry in a simple format which +// simpleMessageFormatter is the default formatter. +// It returns a string representing the current journal entry in a simple format which // includes the entry timestamp and MESSAGE field. -func (r *JournalReader) buildMessage() (string, error) { - var msg string - var usec uint64 - var err error - - if msg, err = r.journal.GetData("MESSAGE"); err != nil { - return "", err - } - - if usec, err = r.journal.GetRealtimeUsec(); err != nil { - return "", err +func simpleMessageFormatter(entry *JournalEntry) (string, error) { + msg, ok := entry.Fields["MESSAGE"] + if !ok { + return "", fmt.Errorf("no MESSAGE field present in journal entry") } + usec := entry.RealtimeTimestamp timestamp := time.Unix(0, int64(usec)*int64(time.Microsecond)) return fmt.Sprintf("%s %s\n", timestamp, msg), nil |