Replace UTF8 bounds check with utf8reader

This commit is contained in:
Iwasaki Yudai 2015-08-16 23:46:35 -07:00
parent d0f2128996
commit 87f8409068
5 changed files with 209 additions and 29 deletions

4
Godeps/Godeps.json generated
View File

@ -19,6 +19,10 @@
"ImportPath": "github.com/kr/pty", "ImportPath": "github.com/kr/pty",
"Comment": "release.r56-28-g5cf931e", "Comment": "release.r56-28-g5cf931e",
"Rev": "5cf931ef8f76dccd0910001d74a58a7fca84a83d" "Rev": "5cf931ef8f76dccd0910001d74a58a7fca84a83d"
},
{
"ImportPath": "github.com/yudai/utf8reader",
"Rev": "543610cf49fc1279921d6ef1b7f7773ebece14f8"
} }
] ]
} }

View File

@ -0,0 +1,65 @@
# UTF8Reader for Go
UTF8Reader is a simple wrapper Reader that fills the given buffer with a "tail-safe" UTF8 byte sequence.
## Tail-Safe?
Let's say you have a buffer of 7 bytes and your Reader is going to fill your buffer with a UTF8 byte sequence.
```go
buf := make([]byte, 7)
reader := strings.NewReader("いろは")
reader.Read(buf)
```
The byte length of UTF8 characters is not fixed and some characters like the examples above have 3 byte length. There are others which have a single byte, 2 byte and 4 byte length as well. This means your buffer will be sometimes filled with incomplete bytes as an Unicode character at the tail.
By `reader.Read(buf)`, your `buf` will be like below:
```go
[]byte{
// い
byte(0xe3), // 1
byte(0x81), // 2
byte(0x84), // 3
// ろ
byte(0xe3), // 4
byte(0x82), // 5
byte(0x8d), // 6
// は (incomplete)
byte(0xe3), // 7
}
```
The last character `は` is incomplete and the buffer is now invalid as a UTF8 string.
UTF8Reader detects incomplete bytes like above and aborts filling up the buffer in such cases.
```go
buf := make([]byte, 7)
reader := strings.NewReader("いろは")
utfReader := utf8reader.New(reader)
utfReader.Read(buf)
```
Then you will get:
```go
[]byte{
// い
byte(0xe3), // 1
byte(0x81), // 2
byte(0x84), // 3
// ろ
byte(0xe3), // 4
byte(0x82), // 5
byte(0x8d), // 6
}
```
Of course, bytes left behind will be used to fill up the buffer on next `Read()`.
## Note
UTF8Reader just checks incomplete bytes at the tail of the buffer. Even if the original byte sequence given to UTF8Reader is broken, UTF8Reader reports no errors and just fills up the buffer.

View File

@ -0,0 +1,56 @@
package utf8reader
import (
"bytes"
"errors"
"io"
"unicode/utf8"
)
var SmallBufferError = errors.New("Buffer size must be larger than utf8.UTFMax.")
type UTF8Reader struct {
reader io.Reader
buffer *bytes.Buffer
}
func New(reader io.Reader) *UTF8Reader {
return &UTF8Reader{
reader: reader,
buffer: bytes.NewBuffer(make([]byte, 0)),
}
}
func (r *UTF8Reader) Read(p []byte) (n int, err error) {
size := 0
if cap(p) < utf8.UTFMax {
return size, SmallBufferError
}
if r.buffer.Len() > 0 {
n, err = r.buffer.Read(p)
size += n
if err != nil {
return size, err
}
}
n, err = r.reader.Read(p[size:])
size += n
if err != nil {
return size, err
}
leftOver := 0
for ; leftOver < utf8.UTFMax; leftOver++ {
rune, _ := utf8.DecodeLastRune(p[:size-leftOver])
if rune != utf8.RuneError {
break
}
}
r.buffer.Write(p[size-leftOver : size])
return size - leftOver, nil
}

View File

@ -0,0 +1,79 @@
package utf8reader
import (
"testing"
"bytes"
"strings"
)
func TestRead(t *testing.T) {
str := "日本語"
or := strings.NewReader(str)
r := New(or)
buf := make([]byte, 512)
n, err := r.Read(buf)
if err != nil {
t.Errorf("Unexpected error")
}
if bytes.Compare(buf[:n], []byte(str)) != 0 {
t.Errorf("Failed to read bytes")
}
n, err = r.Read(buf)
if err.Error() != "EOF" {
t.Errorf("Unexpected error")
}
// 3 byte runes
str = "いろはにほ"
or = strings.NewReader(str)
r = New(or)
buf = make([]byte, 7) // 7 % 3 = 1
n, err = r.Read(buf)
if err != nil {
t.Errorf("Unexpected error")
}
if n != 6 {
t.Errorf("Read length error")
}
if bytes.Compare(buf[:n], []byte(str)[:6]) != 0 {
t.Errorf("Failed to read bytes")
}
n, err = r.Read(buf)
if err != nil {
t.Errorf("Unexpected error")
}
if n != 6 {
t.Errorf("Read length error")
}
if bytes.Compare(buf[:n], []byte(str)[6:12]) != 0 {
t.Errorf("Failed to read bytes")
}
n, err = r.Read(buf)
if err != nil {
t.Errorf("Unexpected error")
}
if n != 3 {
t.Errorf("Read length error")
}
if bytes.Compare(buf[:n], []byte(str)[12:15]) != 0 {
t.Errorf("Failed to read bytes")
}
}
func TestReadWithSmallBuffer(t *testing.T) {
str := "日本語"
or := strings.NewReader(str)
r := New(or)
buf := make([]byte, 2) // too small
_, err := r.Read(buf)
if err != SmallBufferError {
t.Errorf("Expected error were not returned")
}
}

View File

@ -7,12 +7,12 @@ import (
"os/exec" "os/exec"
"strings" "strings"
"syscall" "syscall"
"unicode/utf8"
"unsafe" "unsafe"
"github.com/elazarl/go-bindata-assetfs" "github.com/elazarl/go-bindata-assetfs"
"github.com/gorilla/websocket" "github.com/gorilla/websocket"
"github.com/kr/pty" "github.com/kr/pty"
"github.com/yudai/utf8reader"
) )
type App struct { type App struct {
@ -83,11 +83,10 @@ func (app *App) generateHandler() func(w http.ResponseWriter, r *http.Request) {
defer func() { exit <- true }() defer func() { exit <- true }()
buf := make([]byte, 1024) buf := make([]byte, 1024)
leftOver := 0 utf8f := utf8reader.New(fio)
for {
size, err := fio.Read(buf[leftOver:])
size += leftOver
for {
size, err := utf8f.Read(buf)
if err != nil { if err != nil {
log.Printf("command exited for: %s", r.RemoteAddr) log.Printf("command exited for: %s", r.RemoteAddr)
return return
@ -98,31 +97,8 @@ func (app *App) generateHandler() func(w http.ResponseWriter, r *http.Request) {
return return
} }
// UTF-8 Boundary check writer.Write(buf[:size])
for leftOver = 0; leftOver < utf8.UTFMax; leftOver++ {
re, _ := utf8.DecodeLastRune(
buf[:size-leftOver],
)
if re != utf8.RuneError {
break
}
// Invalid UTF rune
}
if leftOver == utf8.UTFMax-1 {
re, _ := utf8.DecodeLastRune(buf[:size-leftOver])
if re == utf8.RuneError {
log.Fatal("UTF8 Boundary error.")
}
}
writer.Write(buf[:size-leftOver])
writer.Close() writer.Close()
for i := 0; i < leftOver; i++ {
buf[i] = buf[size-leftOver+i]
}
} }
}() }()