龙空技术网

go|通过HTTP请求分片下载一个大文件

博学的书签v 170

前言:

如今姐妹们对“http下载请求”大概比较关注,兄弟们都想要剖析一些“http下载请求”的相关文章。那么小编同时在网摘上搜集了一些有关“http下载请求””的相关文章,希望兄弟们能喜欢,你们快快来了解一下吧!

核心是计算好分片的起始位置,然后设置好Header的Range字段

p.request.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", p.start, p.end))

分片合并,按序号将分片组装即可,shell的简单实现:

cat (ls |sort -n) >1.zip

借助Goroutine和channel可以实现并发下载分片

完整代码如下:

package mainimport (	"fmt"	"io"	"io/ioutil"	"log"	"math"	"net/http"	"os"	"strconv"	"sync")var (	size    int64 = 5 * 1024 * 1024 // 分块大小	workers       = 3               // 并发下载数)type part struct {	partID  int	start   int64	end     int64	request *http.Request	client  *http.Client}func (p *part) Do() *http.Response {	p.request.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", p.start, p.end))	resp, err := p.client.Do(p.request)	if err != nil {		fmt.Println(err)	}	log.Printf("[%d]Content-Range:%s, %v\n", p.partID, resp.Header.Get("Content-Range"), resp.Request.Header)	return resp}func complatePart(partNum int, filename string) {	// 分块合并	file, err := os.Create(filename)	if err != nil {		log.Println(err)		return	}	var offset int64	defer file.Close()	for x := 0; x < partNum; x++ {		partFile := fmt.Sprintf("data/%d", x)		buf, err := ioutil.ReadFile(partFile)		if err != nil {			log.Println(err)			continue		}		file.WriteAt(buf, offset)		offset += int64(len(buf))		//os.Remove(partFile)	}	log.Println("written to ", filename)}func sendPart(partNum int, length int64, req *http.Request, ch chan part) {	for i := 0; i < partNum; i++ {		start := int64(i) * size		var end int64		if i == partNum-1 {			end = length - 1		} else {			end = start + size - 1		}		log.Println(start, end)		p := part{			partID:  i,			start:   int64(start),			end:     int64(end),			request: req,			client:  &http.Client{},		}		log.Printf("send part %d to queue \n", i)		ch <- p	}	close(ch)}func worker(ch chan part, wg *sync.WaitGroup) {	for d := range ch {		// 下载分片		log.Printf("download part %d\n", d.partID)		resp := d.Do()		defer resp.Body.Close()		// 保存分片		filename := fmt.Sprintf("data/%d", d.partID)		fd, err := os.Create(filename)		if err != nil {			log.Println(err)		}		defer fd.Close()		_, err = io.Copy(fd, resp.Body)		if err != nil {			log.Println(err)		}	}	wg.Done()}func Download(url string) {	client := http.Client{}	request, err := http.NewRequest("GET", url, nil)	if err != nil {		log.Fatal(err)	}	response, err := client.Do(request)	response.Body.Close()	num := response.Header.Get("Content-Length")	length, _ := strconv.ParseInt(num, 10, 64)	log.Println("Conetnt-Length", length)	partNum := int(math.Ceil(float64(length) / float64(size)))	log.Println("partNum: ", partNum)	var wg sync.WaitGroup	ch := make(chan part)	go func(req *http.Request) {		sendPart(partNum, length, req, ch)	}(request)	for i := 0; i < workers; i++ {		wg.Add(1)		go worker(ch, &wg)	}	wg.Wait()	complatePart(partNum, "./data/1.zip")}func main() {	url := ";	Download(url)}

下载如下:

分片情况如下:

标签: #http下载请求