通过gRPC流传输大文件

Question

在接收方传输文件超过1MB时，我收到零指针取消引用错误。

panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x30 pc=0x1454e81]

我尝试将大文件分割成每个小于1MB的小字节，并通过gRPC流及其工作进行传输。还有使用gRPC传输大文件的其他方法吗？

  

pb / transfer.proto

//TODO() generate proto file following command => protoc ./transfer.proto -I. --go_out=plugins=grpc:.

syntax = "proto3";

package pb;

service GuploadService {
    rpc Upload(stream Chunk) returns (UploadStatus) {}
}

message Chunk {
    bytes Content = 1;
    string totalSize = 2;
    string received = 3;
}

enum UploadStatusCode {
    Unknown = 0;
    Ok = 1;
    Failed = 2;
}

message UploadStatus {
    string Message = 1;
    UploadStatusCode Code = 2;
}

  

file_receive.go

package main

import (
    "./pb"
    "fmt"
    "github.com/pkg/errors"
    "github.com/prometheus/common/log"
    "google.golang.org/grpc"
    "io"
    "net"
    "os"
)

const (
    port = "localhost:50051"
)

type server struct {
}

func (s *server) Upload(stream pb.GuploadService_UploadServer) (err error) {
    // open output file
    fo, err := os.Create("./output")
    if err != nil {
        return errors.New("failed to create file")
    }
    // close fo on exit and check for its returned error
    defer func() {
        if err := fo.Close(); err != nil {
            panic(err)
        }
    }()
    var res *pb.Chunk
    for {
        res, err = stream.Recv()

        if err == io.EOF {
            err = stream.SendAndClose(&pb.UploadStatus{
                Message: "Upload received with success",
                Code:    pb.UploadStatusCode_Ok,
            })
            if err != nil {
                err =  errors.New("failed to send status code")
                return err
            }
            return nil
        }
        fmt.Println(res.Received)
        // write a chunk
        if _, err := fo.Write(res.Content); err != nil {
            err =  errors.New(err.Error())
            return err
        }
    }
}

func main(){
    lis, err := net.Listen("tcp", port)
    if err != nil {
        log.Fatalf("failed to listen: %v", err)
    }
    s := grpc.NewServer()

    pb.RegisterGuploadServiceServer(s, &server{})

    if err := s.Serve(lis); err != nil {
        log.Fatalf("failed to serve: %v", err)
    }
}

  

file_send.go

package main

import (
    "auth_server/example/file_transfer/pb"
    "context"
    "fmt"
    "github.com/pkg/errors"
    "github.com/prometheus/common/log"
    "google.golang.org/grpc"
    "io"
    "os"
    "strconv"
)

const (
    address     = "localhost:50051"
    sentValue = 1000000 //limit
)

func main()  {
    conn, err := grpc.Dial(address, grpc.WithInsecure())
    if err != nil{
        log.Fatalf("didn't connect %v", err)
    }
    defer conn.Close()
    c := pb.NewGuploadServiceClient(conn)

    var (
        buf     []byte
        status  *pb.UploadStatus
    )
    // open input file
    fi, err := os.Open("./input.dmg")
    if err != nil {
        fmt.Println("Not able to open")
        return
    }

    stat, err := fi.Stat()
    if err != nil {
        return
    }
    // close fi on exit and check for its returned error
    defer func() {
        if err := fi.Close(); err != nil {
            fmt.Println("Not able to open")
            return
        }
    }()

    ctx := context.Background()
    stream, err := c.Upload(ctx)
    if err != nil {
        err = errors.Wrapf(err,
            "failed to create upload stream for file %s",
            fi)
        return
    }
    defer stream.CloseSend()

    buf = make([]byte,stat.Size())
    for {
        // read a chunk
        n, err := fi.Read(buf)
        if err != nil && err != io.EOF {
            err = errors.Wrapf(err,
                "failed to send chunk via stream")
            return
        }
        if n == 0 {
            break
        }
        var i int64
        for i = 0 ; i < ((stat.Size()/sentValue)*sentValue)  ; i += sentValue {
            err = stream.Send(&pb.Chunk{
                Content: buf[i:i+sentValue],
                TotalSize:strconv.FormatInt(stat.Size(), 10),
                Received:strconv.FormatInt(i+sentValue, 10),
            })
        }
        if stat.Size()%sentValue > 0{
            err = stream.Send(&pb.Chunk{
                Content: buf[((stat.Size()/sentValue)*sentValue):((stat.Size()/sentValue*sentValue)+ (stat.Size()%sentValue))],
                TotalSize:strconv.FormatInt(stat.Size(), 10),
                Received:string(stat.Size()%sentValue),
            })
        }

        if err != nil {
            err = errors.Wrapf(err,
                "failed to send chunk via stream")
            return
        }
    }


    status, err = stream.CloseAndRecv()
    if err != nil {
        err = errors.Wrapf(err,
            "failed to receive upstream status response")
        return
    }

    if status.Code != pb.UploadStatusCode_Ok {
        err = errors.Errorf(
            "upload failed - msg: %s",
            status.Message)
        return
    }

    return
}

检查我的工作代码。