Golang 实现断点续传功能：服务端与客户端教程

服务端支持断点续传

下载文件时暂停后可以继续接着下载, 在线看视频时可以随意拖动进度条, 这些都是断点续传所实现的应用。

http1.1支持Range属性从而实现断点续传, 客户端在暂停时记录了已经下载的文件范围, 当继续下载时就向服务器发送文件剩余的范围Range, 服务器则根据客户端请求的范围Range返回相应文件的部分数据, 而不用将整个文件返回给客户端。

多线程下载器就是利用这个属性进行多线程下载, 下载器将服务器返回的文件数据范围进行分割, 然后根据分割后的范围向服务器索取数据。

1.首先要告诉客户端支持Range断点续传, 然后客户端才能发起Range请求。

这需要在服务器返回http的head以下内容:

writer.Header().Add("Accept-ranges", "bytes") //告诉客户端支持Range, 并且是bytes级的断点续传writer.Header().Add("Content-Length", 文件大小) //文件的总体大小writer.Header().Add("Content-Disposition", "attachment; filename=文件名称“)

若客户端请求Ranges, 服务器就返回Content-Range,Content-Range的内容是(0~文件大小)-(文件大小−1)/文件大小

例如一个大小为1024的文件, Content-Range内容为0-1023/1024

writer.Header().Add("Content-Range", "bytes 0-1023/1024")//断点writer.Header().Add("Content-Range", "bytes 200-1023/1024")

服务器要从客户端的请求的Range判断是否是有效的范围。

如果是有效的范围就要将相应状态码设为206 PartialContent, 否则就设为416

2.代码实现 -- 编写服务端(基于HTTP 服务端)

server.go

package mainimport (    "fmt"    "io"    "io/ioutil"    "log"    "net/http"    "os"    "strconv"    "strings")/*本案例主要展示: 断点续传服务端编写请求的地址: url = "http://localhost:8080:/download?filename=goland-2020.2.2.dmg"文件存放地址:window: C:/tmp/Linux: /tmp*/// 断点续传封装的方法func sendFile(writer http.ResponseWriter, request *http.Request, f *os.File) {    defer f.Close()    info, err := f.Stat()    if err != nil {        log.Println("sendFile1", err.Error())        http.NotFound(writer, request)        return		}    writer.Header().Add("Accept-Ranges", "bytes")    writer.Header().Add("Content-Disposition", "attachment; filename="+info.Name())    var start, end int64    //fmt.Println(request.Header,"\n")    if r := request.Header.Get("Range"); r != "" {    if strings.Contains(r, "bytes=") && strings.Contains(r, "-") {    fmt.Sscanf(r, "bytes=%d-%d", &start, &end)    if end == 0 {    end = info.Size() - 1    }    if start > end || start < 0 || end < 0 || end >= info.Size() {    writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)    log.Println("sendFile2 start:", start, "end:", end, "size:", info.Size())    return    }    writer.Header().Add("Content-Length", strconv.FormatInt(end-start+1, 10))    writer.Header().Add("Content-Range", fmt.Sprintf("bytes %v-%v/%v", start, end, info.Size()))    writer.WriteHeader(http.StatusPartialContent)    } else {    writer.WriteHeader(http.StatusBadRequest)    return    }    } else {    writer.Header().Add("Content-Length", strconv.FormatInt(info.Size(), 10))    start = 0    end = info.Size() - 1    }    _, err = f.Seek(start, 0)    if err != nil {    log.Println("sendFile3", err.Error())    writer.WriteHeader(http.StatusInternalServerError)    return    }    n := 512    buf := make([]byte, n)    for {    if end-start+1 < int64(n) {    n = int(end - start + 1)    }    _, err := f.Read(buf[:n])    if err != nil {    log.Println("1:", err)    if err != io.EOF {    log.Println("error:", err)    }    return    }    err = nil    _, err = writer.Write(buf[:n])    if err != nil {    //log.Println(err, start, end, info.Size(), n)    return    }    start += int64(n)    if start >= end+1 {    return    }    }}// downloadHandler : 文件下载接口func downloadHandler(w http.ResponseWriter, r *http.Request) {    r.ParseForm()    filename := r.Form.Get("filename") //获取文件的名称    // window    f, err := os.Open("C:/tmp/" + filename) //打开文件    // Linux    // f, err := os.Open("/tmp/" + filename) //打开文件    if err != nil {    w.WriteHeader(http.StatusInternalServerError) // 500    return    }    defer f.Close()    fifo, err := f.Stat()    if err != nil {    w.WriteHeader(http.StatusInternalServerError) // 500    return    }    if fifo.Size() < 160*1024*1024 {    // 一般文件的下载的方式(小文件 160MB)    data, err := ioutil.ReadAll(f) //ReadAll从r读取数据直到EOF或遇到error    if err != nil {    w.WriteHeader(http.StatusInternalServerError)    return    }    //下载响应头设置    w.Header().Set("Content-Type", "application/octect-stream")    w.Header().Set("content-disposition", "attachment; filename=\""+filename+"\"")    w.Write(data)    } else {    // 断点续传的下载方式(大文件)    sendFile(w, r, f)    }}func myHandler(w http.ResponseWriter, r *http.Request) {    fmt.Fprintln(w, "Hello world")}func main() {    http.HandleFunc("/home", myHandler)    // 断点续传    http.HandleFunc("/download", downloadHandler)    // 监听端口    fmt.Println("上传服务正在启动, 监听端口:8080...")    err := http.ListenAndServe(":8080", nil)    if err != nil {    		fmt.Printf("Failed to start server, err:%s", err.Error())    }}

编写客户端

client.go

package mainimport ("crypto/sha256""encoding/hex""errors""fmt""io/ioutil""log""mime""net/http""os""path/filepath""strconv""sync""time")/*本案例主要展示: 断点续传客户端编写请求的地址:curl = "https://download.jetbrains.com/go/goland-2020.2.2.dmg"url = "http://localhost:8080:/download?filename=goland-2020.2.2.dmg"文件存放地址:window: C:/tmp/Linux: /tmp*/func parseFileInfoFrom(resp *http.Response) string {    contentDisposition := resp.Header.Get("Content-Disposition")    if contentDisposition != "" {    		_, params, err := mime.ParseMediaType(contentDisposition)        if err != nil {            panic(err)        }        return params["filename"]    }    filename := filepath.Base(resp.Request.URL.Path)    return filename}//FileDownloader 文件下载器type FileDownloader struct {    fileSize int    url string    outputFileName string    totalPart int //下载线程    outputDir string    doneFilePart []filePart}//NewFileDownloader .func NewFileDownloader(url, outputFileName, outputDir string, totalPart int) *FileDownloader {    if outputDir == "" {    wd, err := os.Getwd() //获取当前工作目录    if err != nil {    		log.Println(err)    }    outputDir = wd    }    return &FileDownloader{        fileSize: 0,        url: url,        outputFileName: outputFileName,        outputDir: outputDir,        totalPart: totalPart,        doneFilePart: make([]filePart, totalPart),    }}//filePart 文件分片type filePart struct {    Index int //文件分片的序号    From int //开始byte    To int //解决byte    Data []byte //http下载得到的文件内容}func main() {    startTime := time.Now()    var url string //下载文件的地址    // url = "https://download.jetbrains.com/go/goland-2020.2.2.dmg"    url = "http://localhost:8080:/download?filename=goland-2020.2.2.dmg"    downloader := NewFileDownloader(url, "", "", 10)    if err := downloader.Run(); err != nil {        // fmt.Printf("\n%s", err)        log.Fatal(err)    }    fmt.Printf("\n 文件下载完成耗时: %f second\n", time.Now().Sub(startTime).Seconds())}//head 获取要下载的文件的基本信息(header) 使用HTTP Method Headfunc (d *FileDownloader) head() (int, error) {    r, err := d.getNewRequest("HEAD")    if err != nil {        return 0, err    }    resp, err := http.DefaultClient.Do(r)    if err != nil {    		return 0, err    }    if resp.StatusCode > 299 {    		return 0, errors.New(fmt.Sprintf("Can't process, response is %v", resp.StatusCode))    }    //检查是否支持 断点续传    //https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Ranges    if resp.Header.Get("Accept-Ranges") != "bytes" {    		return 0, errors.New("服务器不支持文件断点续传")    }    d.outputFileName = parseFileInfoFrom(resp)    //https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Length    return strconv.Atoi(resp.Header.Get("Content-Length"))}//Run 开始下载任务func (d *FileDownloader) Run() error {    fileTotalSize, err := d.head()    if err != nil {    		return err    }    d.fileSize = fileTotalSize    jobs := make([]filePart, d.totalPart)    eachSize := fileTotalSize / d.totalPart    for i := range jobs {    jobs[i].Index = i    if i == 0 {    		jobs[i].From = 0    } else {    		jobs[i].From = jobs[i-1].To + 1    }    if i < d.totalPart-1 {    jobs[i].To = jobs[i].From + eachSize    } else {    //the last filePart    jobs[i].To = fileTotalSize - 1    }    }    var wg sync.WaitGroup    for _, j := range jobs {        wg.Add(1)        go func(job filePart) {            defer wg.Done()            err := d.downloadPart(job)            if err != nil {                log.Println("下载文件失败:", err, job)            }        }(j)    }    wg.Wait()    return d.mergeFileParts()}//下载分片func (d FileDownloader) downloadPart(c filePart) error {    r, err := d.getNewRequest("GET")    if err != nil {        return err    }    log.Printf("开始[%d]下载from:%d to:%d\n", c.Index, c.From, c.To)    r.Header.Set("Range", fmt.Sprintf("bytes=%v-%v", c.From, c.To))    resp, err := http.DefaultClient.Do(r)    if err != nil {        return err    }    if resp.StatusCode > 299 {        return errors.New(fmt.Sprintf("服务器错误状态码: %v", resp.StatusCode))    }    defer resp.Body.Close()    bs, err := ioutil.ReadAll(resp.Body)    if err != nil {        return err    }    if len(bs) != (c.To - c.From + 1) {        return errors.New("下载文件分片长度错误")    }    c.Data = bs    d.doneFilePart[c.Index] = c    return nil}    // getNewRequest 创建一个requestfunc (d FileDownloader) getNewRequest(method string) (*http.Request, error) {    r, err := http.NewRequest(    method,    d.url,    nil,    )    if err != nil {    return nil, err    }    r.Header.Set("User-Agent", "mojocn")    return r, nil}//mergeFileParts 合并下载的文件func (d FileDownloader) mergeFileParts() error {    log.Println("开始合并文件")    path := filepath.Join(d.outputDir, d.outputFileName)    mergedFile, err := os.Create(path)    if err != nil {        return err    }    defer mergedFile.Close()    hash := sha256.New()    totalSize := 0    for _, s := range d.doneFilePart {    		mergedFile.Write(s.Data)        hash.Write(s.Data)        totalSize += len(s.Data)    }    if totalSize != d.fileSize {    		return errors.New("文件不完整")    }    //https://download.jetbrains.com/go/goland-2020.2.2.dmg.sha256?_ga=2.223142619.1968990594.1597453229-1195436307.1493100134    if hex.EncodeToString(hash.Sum(nil)) != "3af4660ef22f805008e6773ac25f9edbc17c2014af18019b7374afbed63d4744" {    		return errors.New("文件损坏")    } else {    		log.Println("文件SHA-256校验成功")    }    return nil}

> # go run duandian.go2022/02/14 15:25:51 开始[2]下载from:83654700 to:1254820492022/02/14 15:25:51 开始[3]下载from:125482050 to:1673093992022/02/14 15:25:51 开始[1]下载from:41827350 to:836546992022/02/14 15:25:51 开始[4]下载from:167309400 to:2091367492022/02/14 15:25:51 开始[5]下载from:209136750 to:2509640992022/02/14 15:25:51 开始[9]下载from:376446150 to:4182734952022/02/14 15:25:51 开始[6]下载from:250964100 to:2927914492022/02/14 15:25:51 开始[0]下载from:0 to:418273492022/02/14 15:25:51 开始[7]下载from:292791450 to:3346187992022/02/14 15:25:51 开始[8]下载from:334618800 to:3764461492022/02/14 15:25:56 开始合并文件2022/02/14 15:25:59 文件SHA-256校验成功文件下载完成耗时: 8.493695 second

注意:

文件存放地址:

window: C:/tmp/

Linux: /tmp