快捷導(dǎo)航

Go并發(fā)讀寫文件、分片寫、分片下載文件的實(shí)現(xiàn)示例

更新時(shí)間：2024年01月04日 10:34:19 作者：ProblemTerminator

讀寫文件在很多項(xiàng)目中都可以用到,本文主要介紹了Go并發(fā)讀寫文件、分片寫、分片下載文件的實(shí)現(xiàn)示例,具有一定的參考價(jià)值,感興趣的可以了解一下

簡單讀取

func ReadFile(filePath string) (chunks []byte, err error) {
	f, err := os.Open(filePath)
	if err != nil {
		return
	}
    
    defer f.Close()
	reader := bufio.NewReader(f)

	for {
		dataByte := make([]byte, 5*1024)
		var n int
		n, err = reader.Read(dataByte)
		if err != nil || 0 == n {
			break
		}

		chunks = append(chunks, dataByte[:n]...)
		fmt.Printf("file: %s, len(chunks):%v", filePath, len(chunks))
	}

	isEOF := strings.Compare(err.Error(), "EOF")
	if isEOF == 0 {
		err = nil
		fmt.Printf("read %s success: \n, len=%v", filePath, len(chunks))
		return
	}

	fmt.Printf("readFile over")
	return
}

可以看到如文件較大，chunks會(huì)變得很大，此法只適用特定條件下的一般做法。

讀取&分片寫

讀取文件流+分片寫-1

var bufLen = 2 * 1024 * 1024

func DownLoadFileShardByFilePath1(writerFilePath string, body io.Reader) (err error) {

	f, err := os.OpenFile(writerFilePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, os.ModeAppend)
	defer f.Close()
	if err != nil {
		fmt.Println("open err:" + err.Error())
		return
	}

	writer := bufio.NewWriter(f)
	bs := make([]byte, bufLen)
	for {
		var read int
		read, err = body.Read(bs)
		if err != nil || 0 == read {
			break
		}

		_, err = writer.Write(bs[:read])
		if err != nil {
			fmt.Println("write err:" + err.Error())
			break
		}
	}

	if err == io.EOF {
		err = nil
	}

	if err != nil {
		return
	}

	if err = writer.Flush(); err != nil {
		fmt.Println("writer flush err: ", err.Error())
		return
	}

	fmt.Printf("downLoad over")
	return
}

讀取文件流+分片寫-2

var bufLen = 2 * 1024 * 1024

func DownLoadFileShard(writerFilePath string, body io.Reader) (err error) {

	f, err := os.OpenFile(writerFilePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, os.ModeAppend)
	if err != nil {
		fmt.Println("open err:" + err.Error())
		return
	}

    defer f.Close()
	bs := make([]byte, bufLen)
	writer := bufio.NewWriter(f)
	for {
		var read int
		switch read, err = body.Read(bs[:]); true {
		case read < 0:
			fmt.Println("read err: ", err.Error())
			return
		case read == 0, err == io.EOF:
			fmt.Printf("downLoad over")
			return writer.Flush()
		case read > 0:
			_, err = writer.Write(bs[:read])
			if err != nil {
				fmt.Println("write err:" + err.Error())
				return
			}
		}
	}

	return
}

讀取文件流+并發(fā)分片寫

type FileShard struct {
	Data []byte
	Err  error
	Code int // 0-正常  -1=失敗
}

var bufLen = 2 * 1024 * 1024

func DownLoadFileShardCon(writerFilePath string, body io.Reader) (err error) {

	writerFile, err := os.OpenFile(writerFilePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, os.ModeAppend)
	if err != nil {
		fmt.Println("open err:" + err.Error())
		return
	}

    defer writerFile.Close()
	ch, complete := make(chan *FileShard), make(chan struct{})
	go func() {
		writer := bufio.NewWriter(writerFile)

	youKnow:
		for {
			select {
			case data := <-ch:
				if data == nil {
					err = writer.Flush()
					break youKnow
				}

				if data.Code != 0 {
					err = data.Err
					break youKnow
				}

				if _, err = writer.Write(data.Data); err != nil {
					fmt.Println("write err:", err.Error())
				}
			}
		}

		close(complete)
	}()

	go func() {
		bs := make([]byte, bufLen)
		for {
			switch read, readErr := body.Read(bs[:]); true {
			case read < 0:
				ch <- &FileShard{Code: -1, Err: readErr}
				close(ch)
				return
			case read == 0, err == io.EOF:
				close(ch)
				return
			case read > 0:
				ch <- &FileShard{Data: bs[:read], Code: 0}
			}
		}

	}()

	select {
	case <-complete:
		break
	}

	fmt.Printf("downLoad over")
	return
}

并發(fā)思路有很多種，看你代碼怎么寫哦，條條大路通羅馬！

更好用的Copy方法

要提醒的是，還有一個(gè)很不錯(cuò)的方法在io包里，就是io.Copy()，可防止大文件處理時(shí)的內(nèi)存溢出，也可以替換上述主要流程，比如：

func IOCopyExample(writerFilePath string, body io.Reader) (err error) {
	f, err := os.OpenFile(writerFilePath, os.O_CREATE|os.O_TRUNC, 0666)
	if err != nil {
		return
	}
	
	defer f.Close()
	writer := bufio.NewWriter(f)
	_, err = io.Copy(writer, body)
	_ = writer.Flush()
	return
}

http并發(fā)、分片下載

基于http的Range來完成：

func DownloadFileRange(url, writeFile string) error {
	f, err := os.OpenFile(writeFile , os.O_CREATE|os.O_TRUNC, 0666)
	if err != nil {
		return err
	}

	defer f.Close()

	resp, err := http.Head(url)
	if err != nil {
		return err
	}

	size, err := strconv.Atoi(resp.Header.Get("Content-Length"))
	if err != nil {
		return err
	}

	con := getSize(size)  // getSize函數(shù)用來計(jì)算每次的并發(fā)數(shù)，可按自己方式自行指定
	var start, end int64
	for i := 0; i < con; i++ {
		
		start = int64(i) * int64(size/con)
		end = start + int64(size/con) - 1

		go func(n int, offset, end int64) {
			req := &http.Request{}
			req, err = http.NewRequest(http.MethodGet, url, nil)
			req.Header.Set("Range", fmt.Sprintf("bytes=%v-%v", offset, end))

			client := &http.Client{}
			resp, err = client.Do(req)
			if err != nil {
				return
			}

			defer resp.Body.Close()

			f.Seek(offset, 0)
			_, err = io.Copy(f, resp.Body)
			if err != nil {
				// log
			}

		}(i, start, end)
	}

	return nil
}

到此這篇關(guān)于Go并發(fā)讀寫文件、分片寫、分片下載文件的實(shí)現(xiàn)示例的文章就介紹到這了,更多相關(guān)Go并發(fā)讀寫、分片寫、分片下載內(nèi)容請(qǐng)搜索腳本之家以前的文章或繼續(xù)瀏覽下面的相關(guān)文章希望大家以后多多支持腳本之家！

您可能感興趣的文章: