首先给大家推荐一本书Go in Practice，通过一个个超级小巧而又非常实战的案例提升golang内功。
在TECHNIQUE 48 Incrementally saving a file中，向读者讲述了如何处理多个文件同时提交的处理方法
以下代码和文中略有不同：
func uploadBigFile(w http.ResponseWriter, r *http.Request) {
mr, err := r.MultipartReader()
if err != nil {
fmt.Sprintln(err)
fmt.Fprintln(w, err)
return
}
values := make(map[string][]string, 0)
maxValueBytes := int64(10 << 20)
for {
part, err := mr.NextPart()
if err == io.EOF {
break
}
name := part.FormName()
if name == "" {
continue
}
fileName := part.FileName()
var b bytes.Buffer
if fileName == "" {
n, err := io.CopyN(&b, part, maxValueBytes)
if err != nil && err != io.EOF {
fmt.Sprintln(err)
fmt.Fprintln(w, err)
return
}
maxValueBytes -= n
if maxValueBytes <= 0 {
msg := "multipart message too large"
fmt.Fprint(w, msg)
return
}
values[name] = append(values[name], b.String())
}
dst, err := os.Create("/tmp/upload/" + fileName)
defer dst.Close()
for {
buffer := make([]byte, 100000)
cBytes, err := part.Read(buffer)
if err == io.EOF {
break
}
dst.Write(buffer[0:cBytes])
}
}
}
在进行benchmark测试的时候，发现上述方式在处理单个文件上传的时候要比普通的处理方法要快
普通的文件上传处理接口
func upload(w http.ResponseWriter, r *http.Request) {
file, head, err := r.FormFile("my_file")
if err != nil {
fmt.Sprintln(err)
fmt.Fprintln(w, err)
return
}
localFileDir := "/tmp/upload/"
err = os.MkdirAll(localFileDir, 0777)
if err != nil {
fmt.Sprintln(err)
fmt.Fprintln(w, err)
return
}
localFilePath := localFileDir + head.Filename
localFile, err := os.Create(localFilePath)
if err != nil {
fmt.Sprintln(err)
fmt.Fprintln(w, err)
return
}
defer localFile.Close()
io.Copy(localFile, file)
fmt.Fprintln(w, localFilePath)
}
基准测试代码
func BenchmarkUpload(b *testing.B) {
for i := 0; i < b.N; i++ {
path := "/home/kes/code_1.13.1-1497464373_amd64.deb"
file, err := os.Open(path)
if err != nil {
b.Error(err)
}
defer file.Close()
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile("my_file", filepath.Base(path))
if err != nil {
b.Error(err)
}
io.Copy(part, file)
writer.Close()
req := httptest.NewRequest("POST", "/upload", body)
req.Header.Set("Content-Type", writer.FormDataContentType())
res := httptest.NewRecorder()
upload(res, req)
if res.Code != http.StatusOK {
b.Error("not 200")
}
}
// t.Log(res.Body.String())
// t.Log(io.read)
}
func BenchmarkUploadBig(b *testing.B) {
for i := 0; i < b.N; i++ {
path := "/home/kes/code_1.13.1-1497464373_amd64.deb"
file, err := os.Open(path)
if err != nil {
b.Error(err)
}
defer file.Close()
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile("my_file", filepath.Base(path))
if err != nil {
b.Error(err)
}
io.Copy(part, file)
writer.Close()
req := httptest.NewRequest("POST", "/upload", body)
req.Header.Set("Content-Type", writer.FormDataContentType())
res := httptest.NewRecorder()
uploadBigFile(res, req)
if res.Code != http.StatusOK {
b.Error("not 200")
}
}
// t.Log(res.Body.String())
// t.Log(io.read)
}
测试的文件大小大概44M
测试结果
$ go test -bench="." -v
BenchmarkUpload-4 1 1074491528 ns/op
BenchmarkUploadBig-4 2 517001745 ns/op
PASS
ok _/home/kes/test 2.966s
$ go test -bench="." -v
BenchmarkUpload-4 1 1295672144 ns/op
BenchmarkUploadBig-4 2 537212487 ns/op
PASS
ok _/home/kes/test 3.218s
