简单实现TCP下的大文件高效传输-白红宇

简单实现TCP下的大文件高效传输

阅读量：5953 次

发布时间：2019-06-19

本文共 11084 字，大约阅读时间需要 36 分钟。

在TCP下进行大文件传输不象小文件那样直接打包个BUFFER发送出去,因为文件比较大所以不可能把文件读到一个BUFFER发送出去.主要有些文件的大小可能是1G,2G或更大,分配这么大的BUFFER对内存来说显然是不现实的事情;针对服务端的设计来说就更需要严紧些,BUFFER大小的限制也是变得很重要.下面介绍使用Beetle简单地实现大文件在TCP的传输应用.

协议制定

既然需要把文件分块来处理,那在TCP传输的过程需要制定一些协议来规范数据有效性,数据协议主要有三个:告诉服务器需要上传文件,文件块上传和返回每个环节处理的结果.

1)上传文件指令

public class Upload:ObjectMessage    {        public string FileMD5        {            get;            set;        }        public string FileName        {            get;            set;        }        public long FileSize        {            get;            set;        }        public override void FromProtocolData(HttpData httpbase)        {            FileName = httpbase[CONSTVALUE.HEADER_NAME];            FileMD5 = httpbase[CONSTVALUE.HEADER_MD5];            FileSize = long.Parse(httpbase[CONSTVALUE.HEADER_FILESIZE]);        }        protected override void OnDisposed()        {                   }        protected override void OnToProtocolData(HttpData httpbase)        {            httpbase.Command = CONSTVALUE.COMMAND_UPLOAD;            httpbase[CONSTVALUE.HEADER_MD5] = FileMD5;            httpbase[CONSTVALUE.HEADER_NAME] = FileName;            httpbase[CONSTVALUE.HEADER_FILESIZE] = FileSize.ToString();        }    }

View Code

2)上传文件块指令

public class UploadData:ObjectMessage    {        public string FileMD5        {            get;            set;        }        public Beetle.ByteArraySegment Data        {            get;            set;        }                public override void FromProtocolData(HttpData httpbase)        {            FileMD5 = httpbase[CONSTVALUE.HEADER_MD5];            Data = httpbase.Content;        }        protected override void OnDisposed()        {            if (Data != null)            {                FileTransferPackage.BufferPool.Push(Data);                Data = null;            }        }        protected override void OnToProtocolData(HttpData httpbase)        {            httpbase.Command = CONSTVALUE.COMMAND_UPLOAD_DATA;            httpbase[CONSTVALUE.HEADER_MD5] = FileMD5;            httpbase.Content = Data;        }    }

View Code

3)返回值指令

public class Result :ObjectMessage    {        public string FileMD5        {            get;            set;        }        public bool Error        {            get;            set;        }        public string ErrorDetail        {            get;            set;        }        public override void FromProtocolData(HttpData httpbase)        {            ErrorDetail = httpbase[CONSTVALUE.HEADER_STATUS_DETAIL];            Error = httpbase[CONSTVALUE.HEADER_STATUS] == CONSTVALUE.VALUE_SUCCESS;            FileMD5 = httpbase[CONSTVALUE.HEADER_MD5];        }        protected override void OnDisposed()        {        }        protected override void OnToProtocolData(HttpData httpbase)        {            httpbase.Command = CONSTVALUE.COMMAND_RESULT;            if (Error)            {                httpbase[CONSTVALUE.HEADER_STATUS] = CONSTVALUE.VALUE_SUCCESS;            }            else            {                httpbase[CONSTVALUE.HEADER_STATUS] = CONSTVALUE.VALUE_ERROR;            }            httpbase[CONSTVALUE.HEADER_STATUS_DETAIL] = ErrorDetail;            httpbase[CONSTVALUE.HEADER_MD5] = FileMD5;        }    }

View Code

ObjectMessage是Beetle一个简化HTTP协议的扩展对象,它提供自定义Header和Body等功能.

文件读写器

既然需要处理文件块,那提供一些简单的文件块读取和写入方法是比较重要的.它不仅从设计解决功能的偶合度,还可以方便今后的利用.

1)UploadReader

public class UploadReader : IDisposable    {        public UploadReader(string file)        {            mStream = System.IO.File.OpenRead(file);            StringBuilder sb = new StringBuilder();            MD5 md5Hasher = MD5.Create();            foreach (Byte b in md5Hasher.ComputeHash(mStream))                sb.Append(b.ToString("x2").ToLower());            FileMD5 = sb.ToString();            mStream.Position = 0;            FileSize = mStream.Length;            FileName = System.IO.Path.GetFileName(file);                        }        private System.IO.FileStream mStream = null;        public string FileName        {            get;            set;        }        public long LastReadLength        {            get;            set;        }        public long ReadLength        {            get;            set;        }        public long FileSize        {            get;            set;        }        public string FileMD5        {            get;            set;        }        public bool Completed        {            get            {                return mStream != null && ReadLength == mStream.Length;            }        }        public void Close()        {            if (mStream != null)            {                mStream.Close();                mStream.Dispose();            }        }        public void Reset()        {            mStream.Position = 0;            LastReadLength = 0;            ReadLength = 0;        }        public void Read(ByteArraySegment segment)        {            int loads = mStream.Read(segment.Array, 0, FileTransferPackage.BUFFER_SIZE);            segment.SetInfo(0, loads);            ReadLength += loads;        }        public void Dispose()        {            mStream.Dispose();        }        public override string ToString()        {            string value= string.Format("{0}(MD5:{4})\r\n\r\n[{1}/{2}({3}/秒)]",FileName,ReadLength,FileSize,ReadLength-LastReadLength,FileMD5);            if (!Completed)            {                LastReadLength = ReadLength;            }            return value;        }    }

View Code

UploadReader的功能主要是把文件流读取到指定大小的Buffer中,并提供方法获取当前的读取情况

2)UploadWriter

public class UploadWriter    {        public UploadWriter(string rootPath, string filename,string fileMD5,long size)        {            mFullName = rootPath + filename;            FileName = filename;            FileMD5 = fileMD5;            Size = size;        }        private string mFullName;        private System.IO.FileStream mStream;        public System.IO.FileStream Stream        {            get            {                if (mStream == null)                {                    mStream = System.IO.File.Create(mFullName+".up");                }                return mStream;            }        }        public long WriteLength        {            get;            set;        }        public long LastWriteLength        {            get;            set;        }        public long Size        {            get;            set;        }        public string FileName        {            get;            set;        }        public string FileMD5        {            get;            set;        }        public bool Write(ByteArraySegment segment)        {            Stream.Write(segment.Array, 0, segment.Count);            WriteLength += segment.Count;            Stream.Flush();            if (WriteLength == Size)            {                Stream.Close();                if (System.IO.File.Exists(mFullName))                    System.IO.File.Delete(mFullName);                System.IO.File.Move(mFullName + ".up", mFullName);                return true;            }            return false;        }    }

View Code

UploadWriter的功能主要是把文件写入到临时文件中,写入完成后再更改相应的名称,为了方便查询同样也提供了一些写入情况信息.

服务端代码

如果有了解过Beetle的服务端制定的话,那服务端的实现是非常简单的,只需要写一个对象承继ServerBase并实现数据接收方法处理即可以,接收的数据会会自动转换成之前定义的消息对象,而服务端内部处理的细节是完全不用关心.

protected override void OnMessageReceive(Beetle.PacketRecieveMessagerArgs e)        {            base.OnMessageReceive(e);            if (e.Message is Protocol.Upload)            {                OnUpload(e.Channel, e.Message as Protocol.Upload);            }            else if (e.Message is Protocol.UploadData)            {                OnUploadData(e.Channel, e.Message as Protocol.UploadData);            }        }        private Protocol.Result GetErrorResult(string detail)        {            Protocol.Result result = new Protocol.Result();            result.Error = true;            result.ErrorDetail = detail;            return result;        }        private void OnUpload(Beetle.TcpChannel channel, Protocol.Upload e)        {            Protocol.Result result;            if (mTask[e.FileMD5] != null)            {                 result = GetErrorResult( "该文件正在上传任务中!");                channel.Send(result);                return;            }            UploadWriter writer = new UploadWriter(mRootPath, e.FileName, e.FileMD5, e.FileSize);            lock (mTask)            {                mTask[e.FileMD5] = writer;            }            result = new Protocol.Result();            channel.Send(result);        }        private void OnUploadData(Beetle.TcpChannel channel, Protocol.UploadData e)        {            using (e)            {                Protocol.Result result;                UploadWriter writer = (UploadWriter)mTask[e.FileMD5];                if (writer == null)                {                    result = GetErrorResult("上传任务不存在!");                    channel.Send(result);                    return;                }                if (writer.Write(e.Data))                {                    lock (mTask)                    {                        mTask.Remove(e.FileMD5);                    }                }                result = new Protocol.Result();                result.FileMD5 = writer.FileMD5;                channel.Send(result);            }        }

当接收到客户求上传请求后会建立对应MD5的文件写入器,后面文件块的上传写入相关对象即可.

客户端代码

Beetle对于Client的支持也是非常简单方便,只需要定义一个TcpChannel直接发送定义的对象消息并获取服务器端返回的消息即可.

1 private void OnUpload(object state) 2         { 3             Lib.UploadReader reader = (Lib.UploadReader)state; 4             try 5             { 6                 IsUpload = true; 7                 Lib.Protocol.Upload upload = new Lib.Protocol.Upload(); 8                 upload.FileMD5 = reader.FileMD5; 9                 upload.FileName = reader.FileName;10                 upload.FileSize = reader.FileSize;11                 Lib.Protocol.Result result = mClient.Send
      
       (upload);12                 if (result.Error)13                 {14                     mLastError = result.ErrorDetail;15                     return;16                 }17                 while (!reader.Completed)18                 {19                     mLastError = "文件上传中...";20                     Lib.Protocol.UploadData data = new Lib.Protocol.UploadData();21                     data.Data = Lib.FileTransferPackage.BufferPool.Pop();22                     data.FileMD5 = reader.FileMD5;23                     reader.Read(data.Data);24                     result = mClient.Send
       
        (data);25                     if (result.Error)26                     {27                         mLastError = result.ErrorDetail;28                         return;29                     }30                 }31                 mLastError = "文件上传完成!";32                 33             }34             catch (Exception e_)35             {36                 mLastError = e_.Message;37             }38             mReader.Reset();39             IsUpload = false;40                41         }

View Code

整个过程只需要一个方法却可完成,首先把需要上传的文件信息发送到服务器,当服务器确认后不停地把文件块信息输送到服务端即可.

使用测试

下载代码

转载地址：http://fgaxx.baihongyu.com/

你可能感兴趣的文章