Fix the dependency issue (#231)
This commit is contained in:
37
vendor/github.com/dimchansky/utfbom/.gitignore
generated
vendored
37
vendor/github.com/dimchansky/utfbom/.gitignore
generated
vendored
@@ -1,37 +0,0 @@
|
||||
# Binaries for programs and plugins
|
||||
*.exe
|
||||
*.dll
|
||||
*.so
|
||||
*.dylib
|
||||
*.o
|
||||
*.a
|
||||
|
||||
# Folders
|
||||
_obj
|
||||
_test
|
||||
|
||||
# Architecture specific extensions/prefixes
|
||||
*.[568vq]
|
||||
[568vq].out
|
||||
|
||||
*.cgo1.go
|
||||
*.cgo2.c
|
||||
_cgo_defun.c
|
||||
_cgo_gotypes.go
|
||||
_cgo_export.*
|
||||
|
||||
_testmain.go
|
||||
|
||||
*.prof
|
||||
|
||||
# Test binary, build with `go test -c`
|
||||
*.test
|
||||
|
||||
# Output of the go coverage tool, specifically when used with LiteIDE
|
||||
*.out
|
||||
|
||||
# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736
|
||||
.glide/
|
||||
|
||||
# Gogland
|
||||
.idea/
|
||||
18
vendor/github.com/dimchansky/utfbom/.travis.yml
generated
vendored
18
vendor/github.com/dimchansky/utfbom/.travis.yml
generated
vendored
@@ -1,18 +0,0 @@
|
||||
language: go
|
||||
|
||||
go:
|
||||
- 1.7
|
||||
- tip
|
||||
|
||||
# sudo=false makes the build run using a container
|
||||
sudo: false
|
||||
|
||||
before_install:
|
||||
- go get github.com/mattn/goveralls
|
||||
- go get golang.org/x/tools/cmd/cover
|
||||
- go get golang.org/x/tools/cmd/goimports
|
||||
- go get github.com/golang/lint/golint
|
||||
script:
|
||||
- gofiles=$(find ./ -name '*.go') && [ -z "$gofiles" ] || unformatted=$(goimports -l $gofiles) && [ -z "$unformatted" ] || (echo >&2 "Go files must be formatted with gofmt. Following files has problem:\n $unformatted" && false)
|
||||
- golint ./... # This won't break the build, just show warnings
|
||||
- $HOME/gopath/bin/goveralls -service=travis-ci
|
||||
81
vendor/github.com/dimchansky/utfbom/README.md
generated
vendored
81
vendor/github.com/dimchansky/utfbom/README.md
generated
vendored
@@ -1,81 +0,0 @@
|
||||
# utfbom [](https://godoc.org/github.com/dimchansky/utfbom) [](https://opensource.org/licenses/Apache-2.0) [](https://travis-ci.org/dimchansky/utfbom) [](https://goreportcard.com/report/github.com/dimchansky/utfbom) [](https://coveralls.io/github/dimchansky/utfbom?branch=master)
|
||||
|
||||
The package utfbom implements the detection of the BOM (Unicode Byte Order Mark) and removing as necessary. It can also return the encoding detected by the BOM.
|
||||
|
||||
## Installation
|
||||
|
||||
go get -u github.com/dimchansky/utfbom
|
||||
|
||||
## Example
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
|
||||
"github.com/dimchansky/utfbom"
|
||||
)
|
||||
|
||||
func main() {
|
||||
trySkip([]byte("\xEF\xBB\xBFhello"))
|
||||
trySkip([]byte("hello"))
|
||||
}
|
||||
|
||||
func trySkip(byteData []byte) {
|
||||
fmt.Println("Input:", byteData)
|
||||
|
||||
// just skip BOM
|
||||
output, err := ioutil.ReadAll(utfbom.SkipOnly(bytes.NewReader(byteData)))
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return
|
||||
}
|
||||
fmt.Println("ReadAll with BOM skipping", output)
|
||||
|
||||
// skip BOM and detect encoding
|
||||
sr, enc := utfbom.Skip(bytes.NewReader(byteData))
|
||||
var encStr string
|
||||
switch enc {
|
||||
case utfbom.UTF8:
|
||||
encStr = "UTF8"
|
||||
case utfbom.UTF16BigEndian:
|
||||
encStr = "UTF16 big endian"
|
||||
case utfbom.UTF16LittleEndian:
|
||||
encStr = "UTF16 little endian"
|
||||
case utfbom.UTF32BigEndian:
|
||||
encStr = "UTF32 big endian"
|
||||
case utfbom.UTF32LittleEndian:
|
||||
encStr = "UTF32 little endian"
|
||||
default:
|
||||
encStr = "Unknown, no byte-order mark found"
|
||||
}
|
||||
fmt.Println("Detected encoding:", encStr)
|
||||
output, err = ioutil.ReadAll(sr)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return
|
||||
}
|
||||
fmt.Println("ReadAll with BOM detection and skipping", output)
|
||||
fmt.Println()
|
||||
}
|
||||
```
|
||||
|
||||
Output:
|
||||
|
||||
```
|
||||
$ go run main.go
|
||||
Input: [239 187 191 104 101 108 108 111]
|
||||
ReadAll with BOM skipping [104 101 108 108 111]
|
||||
Detected encoding: UTF8
|
||||
ReadAll with BOM detection and skipping [104 101 108 108 111]
|
||||
|
||||
Input: [104 101 108 108 111]
|
||||
ReadAll with BOM skipping [104 101 108 108 111]
|
||||
Detected encoding: Unknown, no byte-order mark found
|
||||
ReadAll with BOM detection and skipping [104 101 108 108 111]
|
||||
```
|
||||
|
||||
|
||||
233
vendor/github.com/dimchansky/utfbom/utfbom_test.go
generated
vendored
233
vendor/github.com/dimchansky/utfbom/utfbom_test.go
generated
vendored
@@ -1,233 +0,0 @@
|
||||
package utfbom
|
||||
|
||||
import (
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"reflect"
|
||||
"testing"
|
||||
"testing/iotest"
|
||||
"time"
|
||||
)
|
||||
|
||||
var testCases = []struct {
|
||||
name string
|
||||
input []byte
|
||||
inputError error
|
||||
encoding Encoding
|
||||
output []byte
|
||||
}{
|
||||
{"1", []byte{}, nil, Unknown, []byte{}},
|
||||
{"2", []byte("hello"), nil, Unknown, []byte("hello")},
|
||||
{"3", []byte("\xEF\xBB\xBF"), nil, UTF8, []byte{}},
|
||||
{"4", []byte("\xEF\xBB\xBFhello"), nil, UTF8, []byte("hello")},
|
||||
{"5", []byte("\xFE\xFF"), nil, UTF16BigEndian, []byte{}},
|
||||
{"6", []byte("\xFF\xFE"), nil, UTF16LittleEndian, []byte{}},
|
||||
{"7", []byte("\x00\x00\xFE\xFF"), nil, UTF32BigEndian, []byte{}},
|
||||
{"8", []byte("\xFF\xFE\x00\x00"), nil, UTF32LittleEndian, []byte{}},
|
||||
{"5", []byte("\xFE\xFF\x00\x68\x00\x65\x00\x6C\x00\x6C\x00\x6F"), nil,
|
||||
UTF16BigEndian, []byte{0x00, 0x68, 0x00, 0x65, 0x00, 0x6C, 0x00, 0x6C, 0x00, 0x6F}},
|
||||
{"6", []byte("\xFF\xFE\x68\x00\x65\x00\x6C\x00\x6C\x00\x6F\x00"), nil,
|
||||
UTF16LittleEndian, []byte{0x68, 0x00, 0x65, 0x00, 0x6C, 0x00, 0x6C, 0x00, 0x6F, 0x00}},
|
||||
{"7", []byte("\x00\x00\xFE\xFF\x00\x00\x00\x68\x00\x00\x00\x65\x00\x00\x00\x6C\x00\x00\x00\x6C\x00\x00\x00\x6F"), nil,
|
||||
UTF32BigEndian,
|
||||
[]byte{0x00, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00, 0x65, 0x00, 0x00, 0x00, 0x6C, 0x00, 0x00, 0x00, 0x6C, 0x00, 0x00, 0x00, 0x6F}},
|
||||
{"8", []byte("\xFF\xFE\x00\x00\x68\x00\x00\x00\x65\x00\x00\x00\x6C\x00\x00\x00\x6C\x00\x00\x00\x6F\x00\x00\x00"), nil,
|
||||
UTF32LittleEndian,
|
||||
[]byte{0x68, 0x00, 0x00, 0x00, 0x65, 0x00, 0x00, 0x00, 0x6C, 0x00, 0x00, 0x00, 0x6C, 0x00, 0x00, 0x00, 0x6F, 0x00, 0x00, 0x00}},
|
||||
{"9", []byte("\xEF"), nil, Unknown, []byte("\xEF")},
|
||||
{"10", []byte("\xEF\xBB"), nil, Unknown, []byte("\xEF\xBB")},
|
||||
{"11", []byte("\xEF\xBB\xBF"), io.ErrClosedPipe, UTF8, []byte{}},
|
||||
{"12", []byte("\xFE\xFF"), io.ErrClosedPipe, Unknown, []byte("\xFE\xFF")},
|
||||
{"13", []byte("\xFE"), io.ErrClosedPipe, Unknown, []byte("\xFE")},
|
||||
{"14", []byte("\xFF\xFE"), io.ErrClosedPipe, Unknown, []byte("\xFF\xFE")},
|
||||
{"15", []byte("\x00\x00\xFE\xFF"), io.ErrClosedPipe, UTF32BigEndian, []byte{}},
|
||||
{"16", []byte("\x00\x00\xFE"), io.ErrClosedPipe, Unknown, []byte{0x00, 0x00, 0xFE}},
|
||||
{"17", []byte("\x00\x00"), io.ErrClosedPipe, Unknown, []byte{0x00, 0x00}},
|
||||
{"18", []byte("\x00"), io.ErrClosedPipe, Unknown, []byte{0x00}},
|
||||
{"19", []byte("\xFF\xFE\x00\x00"), io.ErrClosedPipe, UTF32LittleEndian, []byte{}},
|
||||
{"20", []byte("\xFF\xFE\x00"), io.ErrClosedPipe, Unknown, []byte{0xFF, 0xFE, 0x00}},
|
||||
{"21", []byte("\xFF\xFE"), io.ErrClosedPipe, Unknown, []byte{0xFF, 0xFE}},
|
||||
{"22", []byte("\xFF"), io.ErrClosedPipe, Unknown, []byte{0xFF}},
|
||||
{"23", []byte("\x68\x65"), nil, Unknown, []byte{0x68, 0x65}},
|
||||
}
|
||||
|
||||
type sliceReader struct {
|
||||
input []byte
|
||||
inputError error
|
||||
}
|
||||
|
||||
func (r *sliceReader) Read(p []byte) (n int, err error) {
|
||||
if len(p) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
if err = r.getError(); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
n = copy(p, r.input)
|
||||
r.input = r.input[n:]
|
||||
err = r.getError()
|
||||
return
|
||||
}
|
||||
|
||||
func (r *sliceReader) getError() (err error) {
|
||||
if len(r.input) == 0 {
|
||||
if r.inputError == nil {
|
||||
err = io.EOF
|
||||
} else {
|
||||
err = r.inputError
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
var readMakers = []struct {
|
||||
name string
|
||||
fn func(io.Reader) io.Reader
|
||||
}{
|
||||
{"full", func(r io.Reader) io.Reader { return r }},
|
||||
{"byte", iotest.OneByteReader},
|
||||
}
|
||||
|
||||
func TestSkip(t *testing.T) {
|
||||
for _, tc := range testCases {
|
||||
for _, readMaker := range readMakers {
|
||||
r := readMaker.fn(&sliceReader{tc.input, tc.inputError})
|
||||
|
||||
sr, enc := Skip(r)
|
||||
if enc != tc.encoding {
|
||||
t.Fatalf("test %v reader=%s: expected encoding %v, but got %v", tc.name, readMaker.name, tc.encoding, enc)
|
||||
}
|
||||
|
||||
output, err := ioutil.ReadAll(sr)
|
||||
if !reflect.DeepEqual(output, tc.output) {
|
||||
t.Fatalf("test %v reader=%s: expected to read %+#v, but got %+#v", tc.name, readMaker.name, tc.output, output)
|
||||
}
|
||||
if err != tc.inputError {
|
||||
t.Fatalf("test %v reader=%s: expected to get %+#v error, but got %+#v", tc.name, readMaker.name, tc.inputError, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSkipSkip(t *testing.T) {
|
||||
for _, tc := range testCases {
|
||||
for _, readMaker := range readMakers {
|
||||
r := readMaker.fn(&sliceReader{tc.input, tc.inputError})
|
||||
|
||||
sr0, _ := Skip(r)
|
||||
sr, enc := Skip(sr0)
|
||||
if enc != Unknown {
|
||||
t.Fatalf("test %v reader=%s: expected encoding %v, but got %v", tc.name, readMaker.name, Unknown, enc)
|
||||
}
|
||||
|
||||
output, err := ioutil.ReadAll(sr)
|
||||
if !reflect.DeepEqual(output, tc.output) {
|
||||
t.Fatalf("test %v reader=%s: expected to read %+#v, but got %+#v", tc.name, readMaker.name, tc.output, output)
|
||||
}
|
||||
if err != tc.inputError {
|
||||
t.Fatalf("test %v reader=%s: expected to get %+#v error, but got %+#v", tc.name, readMaker.name, tc.inputError, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSkipOnly(t *testing.T) {
|
||||
for _, tc := range testCases {
|
||||
for _, readMaker := range readMakers {
|
||||
r := readMaker.fn(&sliceReader{tc.input, tc.inputError})
|
||||
|
||||
sr := SkipOnly(r)
|
||||
|
||||
output, err := ioutil.ReadAll(sr)
|
||||
if !reflect.DeepEqual(output, tc.output) {
|
||||
t.Fatalf("test %v reader=%s: expected to read %+#v, but got %+#v", tc.name, readMaker.name, tc.output, output)
|
||||
}
|
||||
if err != tc.inputError {
|
||||
t.Fatalf("test %v reader=%s: expected to get %+#v error, but got %+#v", tc.name, readMaker.name, tc.inputError, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type zeroReader struct{}
|
||||
|
||||
func (zeroReader) Read(p []byte) (int, error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
type readerEncoding struct {
|
||||
Rd *Reader
|
||||
Enc Encoding
|
||||
}
|
||||
|
||||
func TestSkipZeroReader(t *testing.T) {
|
||||
var z zeroReader
|
||||
|
||||
c := make(chan readerEncoding)
|
||||
go func() {
|
||||
r, enc := Skip(z)
|
||||
c <- readerEncoding{r, enc}
|
||||
}()
|
||||
|
||||
select {
|
||||
case re := <-c:
|
||||
if re.Enc != Unknown {
|
||||
t.Error("Unknown encoding expected")
|
||||
} else {
|
||||
var b [1]byte
|
||||
n, err := re.Rd.Read(b[:])
|
||||
if n != 0 {
|
||||
t.Error("unexpected bytes count:", n)
|
||||
}
|
||||
if err != io.ErrNoProgress {
|
||||
t.Error("unexpected error:", err)
|
||||
}
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Error("test timed out (endless loop in Skip?)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSkipOnlyZeroReader(t *testing.T) {
|
||||
var z zeroReader
|
||||
|
||||
c := make(chan *Reader)
|
||||
go func() {
|
||||
r := SkipOnly(z)
|
||||
c <- r
|
||||
}()
|
||||
|
||||
select {
|
||||
case r := <-c:
|
||||
var b [1]byte
|
||||
n, err := r.Read(b[:])
|
||||
if n != 0 {
|
||||
t.Error("unexpected bytes count:", n)
|
||||
}
|
||||
if err != io.ErrNoProgress {
|
||||
t.Error("unexpected error:", err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Error("test timed out (endless loop in Skip?)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReader_ReadEmpty(t *testing.T) {
|
||||
for _, tc := range testCases {
|
||||
for _, readMaker := range readMakers {
|
||||
r := readMaker.fn(&sliceReader{tc.input, tc.inputError})
|
||||
|
||||
sr := SkipOnly(r)
|
||||
|
||||
n, err := sr.Read(nil)
|
||||
if n != 0 {
|
||||
t.Fatalf("test %v reader=%s: expected to read zero bytes, but got %v", tc.name, readMaker.name, n)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("test %v reader=%s: expected to get <nil> error, but got %+#v", tc.name, readMaker.name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user