ftp多线程下载工具

//代码类似https多线程下载,整体实现逻辑类似,区别比较大的是curl_opt的相关参数设置不一样

#include <iostream> #include <fstream> #include <curl/curl.h> #include <pthread.h> #include <sys/mman.h> #include <sys/stat.h> #include <fcntl.h> #include <sys/types.h> #include <string.h> #include <unistd.h> #include <errno.h> #include <signal.h> #include <stdio.h> #include <stdlib.h> using namespace std; #define THREADS_NUMS (10) class FileInfo { public: void * pFile; size_t offset; size_t endpos; char * pUrl; pthread_t tid; size_t used; FILE * file; size_t totalLen; }; FileInfo cFiles[THREADS_NUMS+1]; char buffer[64]={0}; long dwLen=0; //处理下载进度 int progress_callback(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow){ if (dltotal != 0) { //printf("%lf / %lf (%lf %%)\n", dlnow, dltotal, dlnow*100.0 / dltotal); long totalUsedLen=0; //long totalLen=0; for(int i=0;i<THREADS_NUMS+1;i++) { totalUsedLen+=cFiles[i].used; totalUsedLen+=cFiles[i].totalLen; } printf("%ld / %ld (%ld %%)\n",totalUsedLen,dwLen,totalUsedLen*100/dwLen); } return 0; } //获取ftp需要下载文件的大小 static size_t header_callback(char *buffer, size_t size, size_t nitems, void *userdata) { /* received header is nitems * size long in 'buffer' NOT ZERO TERMINATED */ /* 'userdata' is set with CURLOPT_HEADERDATA */ long duLen=0; if(sscanf(buffer,"Content-Length: %ld\n",&duLen)) { *(long *)userdata=duLen; } return nitems*size; } //获取ftp需要下载文件的大小 double getFileLength(char * pUrl,char * usr,char * pwd) { CURL * pCurl=curl_easy_init(); if(NULL==pCurl) { cout<<"curl_easy_init error!"<<endl; return false; } snprintf(buffer,64,"%s:%s",usr,pwd); curl_easy_setopt(pCurl,CURLOPT_URL,pUrl); curl_easy_setopt(pCurl,CURLOPT_HEADER ,1); curl_easy_setopt(pCurl,CURLOPT_NOBODY ,1); curl_easy_setopt(pCurl,CURLOPT_USERPWD,buffer); curl_easy_setopt(pCurl, CURLOPT_HEADERFUNCTION, header_callback);//设置头文件处理函数(获取ftp需要下载的文件大小) curl_easy_setopt(pCurl, CURLOPT_HEADERDATA, &dwLen); CURLcode tRet=curl_easy_perform(pCurl); if(0!=tRet) { cout<<"curl_easy_perform error"<<endl; return false; } curl_easy_cleanup(pCurl); return dwLen; } //各线程分别写自己对应的位置 size_t writeFile(void *pData, size_t dwSize, size_t dwMemb, void * pFile) { FileInfo * pFileInfo=(FileInfo *)pFile; cout<<"id: "<<pFileInfo->tid<<" offset: "<<pFileInfo->offset<<endl; cout<<"dwSize*dwMemb: "<<dwSize*dwMemb<<endl; memcpy((char *)pFileInfo->pFile+pFileInfo->offset,(char *)pData,dwSize*dwMemb); pFileInfo->offset+=dwSize*dwMemb; pFileInfo->used+=dwSize*dwMemb; return dwSize*dwMemb; } void * works(void * arg) { FileInfo * pFile=(FileInfo *)arg; CURL * pCurl=curl_easy_init(); if(NULL==pCurl) { cout<<"curl_easy_init error!"<<endl; return NULL; } if(pFile->file) { cout<<"hello"<<endl; fscanf(pFile->file,"%ld-%ld-%ld",&pFile->offset,&pFile->endpos,&pFile->totalLen); } else { cout<<"downFile open failed"<<endl; } if(pFile->offset>=pFile->endpos-1) { cout<<pFile->tid<<" already downed: "<<pFile->offset<<"--"<<pFile->endpos<<endl; return NULL; } char range[64]={0}; snprintf(range,64,"%ld-%ld",pFile->offset,pFile->endpos); curl_easy_setopt(pCurl,CURLOPT_URL,"ftp://127.0.0.1/boost.rar");//需要下载的ftp文件链接 curl_easy_setopt(pCurl,CURLOPT_USERPWD,buffer); curl_easy_setopt(pCurl,CURLOPT_WRITEDATA,pFile->file); //CURLOPT_WRITEFUNCTION curl_easy_setopt(pCurl,CURLOPT_WRITEFUNCTION,writeFile);//写文件函数 curl_easy_setopt(pCurl,CURLOPT_WRITEDATA ,pFile); curl_easy_setopt(pCurl,CURLOPT_RANGE ,range); curl_easy_setopt(pCurl,CURLOPT_NOPROGRESS ,0L); curl_easy_setopt(pCurl,CURLOPT_PROGRESSFUNCTION,progress_callback);//下载进度显示函数 CURLcode tRet=curl_easy_perform(pCurl); if(0!=tRet) { cout<<"curl_easy_perform error"<<endl; return NULL; } curl_easy_cleanup(pCurl); } void downFtpFile(char * usr,char * pwd) { long lLen=(long)getFileLength("ftp://127.0.0.1/boost.rar",usr,pwd); cout<<lLen<<endl; int fd=open("ftpdown.txt",O_RDWR|O_CREAT,S_IRUSR|S_IWUSR); if(fd==-1) { cout<<"open failed"<<endl; return; } if(lseek(fd,dwLen,SEEK_SET)==-1) { cout<<"lseek failed"<<endl; close(fd); return; } if(write(fd,"",1)!=1) { cout<<"write failed"<<endl; close(fd); return; } //内存映射本地的文件(放置ftp服务器上需要下载的文件) char * filePos=(char *)mmap(NULL,dwLen,PROT_READ|PROT_WRITE,MAP_SHARED,fd,0); if(filePos==MAP_FAILED) { close(fd); cout<<"mmap failed: "<<errno<<endl; return; } int slice=dwLen/THREADS_NUMS; FILE * file=fopen("downTemp.txt","w+"); //FILE * file=NULL; //创建THREADS_NUMS个线程,同时设置各线程在文件中的下载位置 for(int i=0;i<THREADS_NUMS+1;i++) { cFiles[i].offset=i*slice; //cFiles[i].pUrl=pUrl; cFiles[i].pFile=filePos; //cFiles[i].used=0; cFiles[i].file=file; if(i==THREADS_NUMS) { cFiles[i].endpos=dwLen-1; cFiles[i].totalLen=cFiles[i].endpos-cFiles[i].offset+1; } else { cFiles[i].endpos=(i+1)*slice-1; cFiles[i].totalLen=slice; } pthread_create(&cFiles[i].tid,NULL,works,&cFiles[i]); usleep(1); } for(int i=0;i<THREADS_NUMS+1;i++) { cout<<"tid: "<<cFiles[i].tid<<" finished"<<endl; pthread_join(cFiles[i].tid,NULL); } cout<<"00000"<<endl; fclose(file); cout<<"11111"<<endl; munmap(filePos,dwLen); cout<<"22222"<<endl; } void sighandler_func(int arg) { cout<<"arg: "<<arg<<endl; int fd=open("downTemp.txt",O_RDWR|O_CREAT,S_IRUSR|S_IWUSR); for(int i=0;i<THREADS_NUMS+1;i++) { cFiles[i].totalLen=cFiles[i].used; //cout<<"used: "<<cFiles[i].used<<"/"<<cFiles[i].totalLen<<endl; char buffer[64]={0}; snprintf(buffer,64,"%ld-%ld-%ld\n",cFiles[i].offset,cFiles[i].endpos,cFiles[i].totalLen); write(fd,buffer,strlen(buffer)); } close(fd); exit(-1); } int main(int argc,char * * argv) { if(SIG_ERR==signal(SIGINT,sighandler_func)) { cout<<"signal error"<<endl; return 0; } downFtpFile(argv[1],argv[2]); cout<<"end"<<endl; return 0; }

 

热门相关:医道至尊   魅王毒后   最强神话帝皇   绝宠小医妃:王爷,来一针   人族镇守使