前言:
如今姐妹们对“http下载请求”大概比较关注,兄弟们都想要剖析一些“http下载请求”的相关文章。那么小编同时在网摘上搜集了一些有关“http下载请求””的相关文章,希望兄弟们能喜欢,你们快快来了解一下吧!核心是计算好分片的起始位置,然后设置好Header的Range字段
p.request.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", p.start, p.end))
分片合并,按序号将分片组装即可,shell的简单实现:
cat (ls |sort -n) >1.zip
借助Goroutine和channel可以实现并发下载分片
完整代码如下:
package mainimport ( "fmt" "io" "io/ioutil" "log" "math" "net/http" "os" "strconv" "sync")var ( size int64 = 5 * 1024 * 1024 // 分块大小 workers = 3 // 并发下载数)type part struct { partID int start int64 end int64 request *http.Request client *http.Client}func (p *part) Do() *http.Response { p.request.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", p.start, p.end)) resp, err := p.client.Do(p.request) if err != nil { fmt.Println(err) } log.Printf("[%d]Content-Range:%s, %v\n", p.partID, resp.Header.Get("Content-Range"), resp.Request.Header) return resp}func complatePart(partNum int, filename string) { // 分块合并 file, err := os.Create(filename) if err != nil { log.Println(err) return } var offset int64 defer file.Close() for x := 0; x < partNum; x++ { partFile := fmt.Sprintf("data/%d", x) buf, err := ioutil.ReadFile(partFile) if err != nil { log.Println(err) continue } file.WriteAt(buf, offset) offset += int64(len(buf)) //os.Remove(partFile) } log.Println("written to ", filename)}func sendPart(partNum int, length int64, req *http.Request, ch chan part) { for i := 0; i < partNum; i++ { start := int64(i) * size var end int64 if i == partNum-1 { end = length - 1 } else { end = start + size - 1 } log.Println(start, end) p := part{ partID: i, start: int64(start), end: int64(end), request: req, client: &http.Client{}, } log.Printf("send part %d to queue \n", i) ch <- p } close(ch)}func worker(ch chan part, wg *sync.WaitGroup) { for d := range ch { // 下载分片 log.Printf("download part %d\n", d.partID) resp := d.Do() defer resp.Body.Close() // 保存分片 filename := fmt.Sprintf("data/%d", d.partID) fd, err := os.Create(filename) if err != nil { log.Println(err) } defer fd.Close() _, err = io.Copy(fd, resp.Body) if err != nil { log.Println(err) } } wg.Done()}func Download(url string) { client := http.Client{} request, err := http.NewRequest("GET", url, nil) if err != nil { log.Fatal(err) } response, err := client.Do(request) response.Body.Close() num := response.Header.Get("Content-Length") length, _ := strconv.ParseInt(num, 10, 64) log.Println("Conetnt-Length", length) partNum := int(math.Ceil(float64(length) / float64(size))) log.Println("partNum: ", partNum) var wg sync.WaitGroup ch := make(chan part) go func(req *http.Request) { sendPart(partNum, length, req, ch) }(request) for i := 0; i < workers; i++ { wg.Add(1) go worker(ch, &wg) } wg.Wait() complatePart(partNum, "./data/1.zip")}func main() { url := "; Download(url)}
下载如下:
分片情况如下:
版权声明:
本站文章均来自互联网搜集,如有侵犯您的权益,请联系我们删除,谢谢。
标签: #http下载请求