FormData/Go分片/分块文件上传

OStack程序员社区-中国程序员成长平台 › 门户 › 编程› Go›Go教程

原作者: [db:作者] 来自: [db:来源] 收藏邀请

FormData 接口提供了一种表示表单数据的键值对的构造方式，经过它的数据可以使用 XMLHttpRequest.send() 方法送出，本接口和此方法都相当简单直接。如果送出时的编码类型被设为 "multipart/form-data"，它会使用和表单一样的格式。

如果你想构建一个简单的GET请求，并且通过<form>的形式带有查询参数，可以将它直接传递给URLSearchParams。

分块(分片，统称分块了，确实只是发送一块数据)文件上传主要分2部分。

1. 前端js用file.slice可以从文件中切出一块一块的数据，然后用FormData包装一下，用XMLHttpRequest把切出来的数据块，一块一块send到server.

2. Server接收到的每一块都是一个multipart/form-data Form表单。可以在表单里放很多附属信息，文件名，大小，块大小，块索引，最总带上这块切出来的二进制数据。

multipart/form-data 数据

POST /upload HTTP/1.1
Host: localhost:8080
Content-Length: 2098072
User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36

Content-Type: multipart/form-data; boundary=----WebKitFormBoundarymtng0xrR3ASR7wx7

------WebKitFormBoundaryHdBeczaB5xBq6d55
Content-Disposition: form-data; name="file_name"

apache-maven-3.6.3-bin.zip
------WebKitFormBoundaryHdBeczaB5xBq6d55
Content-Disposition: form-data; name="file_size"

9602303
------WebKitFormBoundaryHdBeczaB5xBq6d55
Content-Disposition: form-data; name="block_size"

2097152
------WebKitFormBoundaryHdBeczaB5xBq6d55
Content-Disposition: form-data; name="total_blocks"

5
------WebKitFormBoundaryHdBeczaB5xBq6d55
Content-Disposition: form-data; name="break_error"

true
------WebKitFormBoundaryHdBeczaB5xBq6d55
Content-Disposition: form-data; name="index"

3
------WebKitFormBoundaryHdBeczaB5xBq6d55
Content-Disposition: form-data; name="data"; filename="blob"
Content-Type: application/octet-stream
(binary)

在Server存储文件，基本也就2种方案：

1. 直接创建一个对应大小的文件，按照每块数据的offset位置，写进去。

2. 每个传过来的数据块，保存成一个单独的数据块文件，最后把所有文件块合并成文件。

我这里只是做了一份简单的演示代码，基本上是不能用于生产环境的。

Index.html，直接把js写进去了

  1 <!DOCTYPE html>
  2 <html>
  3     <head>
  4         <meta charset="utf8">
  5         <title>Multil-Blocks upload</title>
  6     </head>
  7 
  8     <body>
  9         <h2>Multil-Blocks upload</h2>
 10 
 11         <input id="file" type="file" />
 12 
 13         <input type="checkbox" id="multil_block_file">multil block file</input>
 14         <button type="button" onclick="on_block_upload()">Block upload</button>
 15         <button type="button" onclick="on_concurrency_upload()">Concurrency upload</button>
 16         <hr/>
 17 
 18         <div>
 19             <label>File name: </label><span id="file_name"></span>
 20         </div>
 21         <div>
 22             <label>File size: </label><span id="file_size"></span>
 23         </div>
 24         <div>
 25             <label>Split blocks: </label><span id="block_count"></span>
 26         </div>
 27 
 28         <hr/>
 29 
 30         <p id="upload_info"></p>
 31 
 32         <script>
 33             var Block_Size = 1024 * 1024 * 2;
 34 
 35             var el_file = document.getElementById('file');
 36             var el_multil_block_file = document.getElementById('multil_block_file');
 37             var el_file_name = document.getElementById('file_name');
 38             var el_file_size = document.getElementById('file_size');
 39             var el_block_count = document.getElementById('block_count');
 40             var el_upload_info = document.getElementById('upload_info');
 41 
 42             var file = null;
 43             var total_blocks = 0;
 44             var block_index = -1;
 45             var block_index_random_arr = [];
 46             var form_data = null;
 47 
 48 
 49             el_file.onchange = function() {
 50                 if (this.files.length === 0) return;
 51 
 52                 file = this.files[0];
 53                 total_blocks = Math.ceil( file.size / Block_Size );
 54 
 55                 el_file_name.innerText = file.name;
 56                 el_file_size.innerText = file.size;
 57                 el_block_count.innerText = total_blocks;
 58             }
 59 
 60             function print_info(msg) {
 61                 el_upload_info.innerHTML += `${msg}<br/>`;
 62             }
 63 
 64             function done() {
 65                 file = null;
 66                 total_blocks = 0;
 67                 block_index = -1;
 68                 form_data = null;
 69                 
 70                 el_file.value = '';
 71             }
 72 
 73 
 74             function get_base_form_data() {
 75                 var base_data = new FormData();
 76                 base_data.append('file_name', file.name);
 77                 base_data.append('file_size', file.size);
 78                 base_data.append('block_size', Block_Size);
 79                 base_data.append('total_blocks', total_blocks);
 80                 base_data.append('break_error', true);
 81                 base_data.append('index', 0);
 82                 base_data.append('data', null);
 83 
 84                 return base_data
 85             }
 86 
 87 
 88             function build_block_index_random_arr() {
 89                 block_index_random_arr = new Array(total_blocks).fill(0).map((v,i) => i);
 90                 block_index_random_arr.sort((n, m) => Math.random() > .5 ? -1 : 1);
 91 
 92                 print_info(`Upload sequence: ${block_index_random_arr}`);
 93             }
 94 
 95 
 96             function post(index, success_cb, failed_cb) {
 97                 if (!form_data) {
 98                     form_data = get_base_form_data();
 99                 }
100                 var start = index * Block_Size;
101                 var end = Math.min(file.size, start + Block_Size);
102 
103                 form_data.set('index', index);
104                 form_data.set('data', file.slice(start, end));
105 
106                 print_info(`Post ${index}/${total_blocks}, offset: ${start} -- ${end}`);
107 
108 
109                 var xhr = new XMLHttpRequest();
110                 xhr.open('POST', '/upload', true);
111                 /*
112                     Browser-based general content types
113                     Content-Type: multipart/form-data; boundary=----WebKitFormBoundarysXH5DIES2XFMuLXL
114 
115                     Error content type:
116                     xhr.setRequestHeader('Content-Type', 'multipart/form-data');
117                     Content-Type: multipart/form-data;
118                 */
119                 xhr.onreadystatechange = function() {
120 
121                     if (xhr.readyState === XMLHttpRequest.DONE) {
122 
123                         if (xhr.status >= 200 && xhr.status < 300 && success_cb) {
124                             return success_cb();
125                         }
126 
127                         if (xhr.status >= 400 && failed_cb) {
128                             failed_cb();
129                         }
130                     }
131                 }
132 
133                 // xhr.onerror event
134                 xhr.send(form_data);
135             }
136 
137 
138             function block_upload() {
139                 if (!file) {
140                     return;
141                 }
142                 if (block_index + 1 >= total_blocks) {
143                     return done();
144                 }
145 
146                 block_index++;
147                 var index = block_index_random_arr[block_index];
148             
149                 post(index, block_upload);
150             }
151 
152 
153             function concurrency_upload() {
154                 if (!file || total_blocks === 0) {
155                     return;
156                 }
157 
158                 build_block_index_random_arr();
159 
160                 form_data = get_base_form_data();
161                 form_data.set('break_error', false);
162                 form_data.set('multil_block', el_multil_block_file.checked);
163 
164                 for (var i of block_index_random_arr) {
165                     ((idx) => {
166                         post(idx, null, function() {
167                             print_info(`Failed: ${idx}`);
168                             setTimeout(() => post(idx), 1000);
169                         });
170                     })(i);
171                 }
172             }
173 
174 
175             function on_block_upload() {
176                 if (file) {
177                     print_info('Block upload');
178 
179                     form_data = get_base_form_data();
180                     form_data.set('multil_block', el_multil_block_file.checked);
181 
182                     build_block_index_random_arr();
183 
184                     block_index = -1;
185                     block_upload();
186                 }
187             }
188 
189             function on_concurrency_upload() {
190                 if (file) {
191                     print_info('Concurrency upload');
192                     concurrency_upload();
193                 }
194             }
195         </script>
196 
197     </body>
198 </html>

View Code

简单的Go server和保存文件，基本忽略所有的错误处理

  1 package main
  2 
  3 import (
  4     "fmt"
  5     "io/ioutil"
  6     "log"
  7     "net/http"
  8     "os"
  9     "path"
 10     "path/filepath"
 11     "regexp"
 12     "strconv"
 13     "strings"
 14     "syscall"
 15     "text/template"
 16 )
 17 
 18 type MultilBlockFile struct {
 19     FileName    string
 20     Size        int64
 21     BlockSize   int64
 22     TotalBlocks int
 23     Index       int
 24     Bufs        []byte
 25     BreakError  bool
 26 }
 27 
 28 func fileIsExist(f string) bool {
 29     _, err := os.Stat(f)
 30     return err == nil || os.IsExist(err)
 31 }
 32 
 33 func lockFile(f *os.File) error {
 34     err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
 35     if err != nil {
 36         return fmt.Errorf("get flock failed. err: %s", err)
 37     }
 38 
 39     return nil
 40 }
 41 
 42 func unlockFile(f *os.File) error {
 43     defer f.Close()
 44     return syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
 45 }
 46 
 47 func singleFileSave(mbf *MultilBlockFile) error {
 48 
 49     dir, _ := filepath.Abs(filepath.Dir(os.Args[0]))
 50     filePath := path.Join(dir, "tmp", mbf.FileName)
 51 
 52     offset := int64(mbf.Index) * mbf.BlockSize
 53 
 54     fmt.Println(">>> Single file save ---------------------")
 55     fmt.Printf("Save file: %s \n", filePath)
 56     fmt.Printf("File offset: %d \n", offset)
 57 
 58     var f *os.File
 59     var needTruncate bool = false
 60     if !fileIsExist(filePath) {
 61         needTruncate = true
 62     }
 63 
 64     f, _ = os.OpenFile(filePath, syscall.O_CREAT|syscall.O_WRONLY, 0777)
 65 
 66     err := lockFile(f)
 67     if err != nil {
 68         if mbf.BreakError {
 69             log.Fatalf("get flock failed. err: %s", err)
 70         } else {
 71             return err
 72         }
 73     }
 74 
 75     if needTruncate {
 76         f.Truncate(mbf.Size)
 77     }
 78 
 79     f.WriteAt(mbf.Bufs, offset)
 80 
 81     unlockFile(f)
 82 
 83     return nil
 84 }
 85 
 86 func multilBlocksSave(mbf *MultilBlockFile) error {
 87     dir, _ := filepath.Abs(filepath.Dir(os.Args[0]))
 88     tmpFolderPath := path.Join(dir, "tmp")
 89     tmpFileName := fmt.Sprintf("%s.%d", mbf.FileName, mbf.Index)
 90     fileBlockPath := path.Join(tmpFolderPath, tmpFileName)
 91 
 92     f, _ := os.OpenFile(fileBlockPath, syscall.O_CREAT|syscall.O_WRONLY|syscall.O_TRUNC, 0777)
 93     defer f.Close()
 94 
 95     f.Write(mbf.Bufs)
 96     f.Close()
 97 
 98     re := regexp.MustCompile(`(?i:^` + mbf.FileName + `).\d$`)
 99 
100     files, _ := ioutil.ReadDir(tmpFolderPath)
101     matchFiles := make(map[string]bool)
102 
103     for _, file := range files {
104         if file.IsDir() {
105             continue
106         }
107 
108         fname := file.Name()
109         if re.MatchString(fname) {
110             matchFiles[fname] = true
111         }
112     }
113 
114     if len(matchFiles) >= mbf.TotalBlocks {
115         lastFile, _ := os.OpenFile(path.Join(tmpFolderPath, mbf.FileName), syscall.O_CREAT|syscall.O_WRONLY, 0777)
116         lockFile(lastFile)
117 
118         lastFile.Truncate(mbf.Size)
119 
120         for name := range matchFiles {
121             tmpPath := path.Join(tmpFolderPath, name)
122 
123             idxStr := name[strings.LastIndex(name, ".")+1:]
124             idx, _ := strconv.ParseInt(idxStr, 10, 32)
125 
126             fmt.Printf("Match file: %s index: %d \n", name, idx)
127 
128             data, _ := ioutil.ReadFile(tmpPath)
129 
130             lastFile.WriteAt(data, idx*mbf.BlockSize)
131 
132             os.Remove(tmpPath)
133         }
134         unlockFile(lastFile)
135     }
136 
137     return nil
138 }
139 
140 func indexHandle(w http.ResponseWriter, r *http.Request) {
141     tmp, _ := template.ParseFiles("./static/index.html")
142     tmp.Execute(w, "Index")
143 }
144 
145 func uploadHandle(w http.ResponseWriter, r *http.Request) {
146 
147     var mbf MultilBlockFile
148     mbf.FileName = r.FormValue("file_name")
149     mbf.Size, _ = strconv.ParseInt(r.FormValue("file_size"), 10, 64)
150     mbf.BlockSize, _ = strconv.ParseInt(r.FormValue("block_size"), 10, 64)
151     mbf.BreakError, _ = strconv.ParseBool(r.FormValue("break_error"))
152 
153     var i int64
154     i, _ = strconv.ParseInt(r.FormValue("total_blocks"), 10, 32)
155     mbf.TotalBlocks = int(i)
156 
157     i, _ = strconv.ParseInt(r.FormValue("index"), 10, 32)
158     mbf.Index = int(i)
159 
160     d, _, _ := r.FormFile("data")
161     mbf.Bufs, _ = ioutil.ReadAll(d)
162 
163     fmt.Printf(">>> Upload --------------------- \n")
164     fmt.Printf("File name: %s \n", mbf.FileName)
165     fmt.Printf("Size: %d \n", mbf.Size)
166     fmt.Printf("Block size: %d \n", mbf.BlockSize)
167     fmt.Printf("Total blocks: %d \n", mbf.TotalBlocks)
168     fmt.Printf("Index: %d \n", mbf.Index)
169     fmt.Println("Bufs len:", len(mbf.Bufs))
170 
171     multilBlockFile, _ := strconv.ParseBool(r.FormValue("multil_block"))
172 
173     var err error
174     if multilBlockFile {
175         err = multilBlocksSave(&mbf)
176     } else {
177         err = singleFileSave(&mbf)
178     }
179 
180     if !mbf.BreakError && err != nil {
181         w.WriteHeader(400)
182         fmt.Fprintf(w, fmt.Sprintf("%s", err))
183         return
184     }
185 
186     fmt.Fprintf(w, "ok")
187 }
188 
189 func main() {
190     println("Listen on 8080")
191 
192     http.HandleFunc("/", indexHandle)
193     http.HandleFunc("/upload", uploadHandle)
194 
195     log.Fatal(http.ListenAndServe(":8080", nil))
196 }