Question

我正在使用C ++开发一个应用程序，该应用程序可以线程化并传递一堆线程URL，以便cURL并行下载。

我正在使用一种安全的方法来下载图像和视频等。我使用memcpy而不是假设数据是字符串或字符数组。

我为每个线程传递了一个结构thread_status，用于许多事情。该结构让父进程知道线程已完成下载。它还存储cURL正在下载的数据并跟踪它的大小，因为cURL返回更多用于写入的缓冲区。

我传递一个（void *）指针，该指针指向在初始化时为每个执行下载的线程分配的每个结构。正确下载第一页，之后我不断从realloc（）获取错误。

这是说明我的问题的最简单的例子。此示例不是多线程的，但使用类似的结构来跟踪自身。

#include <string>
#include <assert.h>
#include <iostream>
#include <curl/curl.h>
#include <stdlib.h> 
#include <stdio.h>
#include <string.h>

#define NOT_READY   1
#define READY       0

using namespace std;

struct thread_status {
    int id;
  pthread_t *pid;
    int readyState;
    char *url;
    void *data;
    size_t bufferlen;
    size_t writepos;
    int initialized;
} ;


size_t static 
writefunction(  void *ptr, size_t size, 
                    size_t nmemb, void *userdata)
{
    size_t nbytes = size*nmemb;
        struct thread_status **this_status;
        this_status = (struct thread_status **) userdata;

        if (!(*this_status)->initialized){
                (*this_status)->data = (void *)malloc(1024);
                (*this_status)->bufferlen = 1024;
                (*this_status)->writepos = 0;
                (*this_status)->initialized = true;
        }

        if ((*this_status)->bufferlen < ((*this_status)->writepos + nbytes)){
            (*this_status)->bufferlen = (*this_status)->bufferlen + nbytes;
            (*this_status)->data = realloc((*this_status)->data, (size_t) ((*this_status)->writepos + nbytes));
        }

        assert((*this_status)->data != NULL);
        memcpy((*this_status)->data + (*this_status)->writepos, ptr, nbytes);
        (*this_status)->writepos += nbytes; 
    return nbytes;
}

void *pull_data (void *my_struct){

struct thread_status *this_struct;
this_struct = (struct thread_status *) my_struct;
this_struct->initialized = false;

cout<<(char *)this_struct->url<<"\n";

CURL *curl;
curl = curl_easy_init();
size_t rc = 0;

while(true){

    curl_easy_setopt(curl,
        CURLOPT_WRITEFUNCTION, writefunction);
    curl_easy_setopt(curl,
        CURLOPT_WRITEDATA, (void *) &this_struct);
    curl_easy_setopt(curl,
        CURLOPT_NOSIGNAL, true);
    curl_easy_setopt(curl,
        CURLOPT_URL, (char *)this_struct->url);

    if (curl_easy_perform(curl) != 0){
        cout<<"curl did not perform\n";
        exit(1);
    } else { 
    if (this_struct->data != NULL){
            // Use a binary write.
            rc = fwrite(this_struct->data, this_struct->writepos, 1, stdout);
            free(this_struct->data);
        } else {
            cout<<"Data is NULL\n";
        } 
    }

    // Tell the babysitter the thread is ready.
    this_struct->readyState = READY;
// This would pause the thread until the parent thread has processed the data in it.
//  while(this_struct->readyState == READY){;}

    // Now get ready for another round!
    this_struct->writepos = (size_t) 0;
    this_struct->initialized = false;
    this_struct->bufferlen = (size_t) 0; 

    break;
}

    curl_easy_cleanup(curl);
    return (void *)"a";
}

int main(){

    char *urls[] = { "http://www.example.com/", "http://www.google.com", "http://www.touspassagers.com/", "http://www.facebook.com/" };
    int i=0;
    struct thread_status mystatuses[4];
    for (i=0;i<4;i++){

        struct thread_status my_status; 
        char *data;

        my_status.id = i;
        my_status.readyState = NOT_READY;
        my_status.url = urls[i];
        my_status.data = data;
        my_status.bufferlen = 0;
        my_status.writepos = 0;
        my_status.initialized = false;

        mystatuses[i] = my_status;
    }

    for (i=0;i<4;i++){
        cout<<"pulling #"<<i<<"\n";
        pull_data((void *)&mystatuses[i]);
    }

}

如果有人可以告诉我我的错误来源或补救措施，我将不胜感激。

Answer 1

您可以考虑使用valgrind来帮助找到内存问题的来源。

Answer 2

知道了！

显然1KB没有足够的内存来处理第一个cURL缓冲区。我将1024更改为nbytes，它的工作原理！

在放入缓冲区的内存memcpy运行已分配的内存之前导致损坏。

如果有人关心看到完整的实施，我发了一篇关于它的帖子： http://www.touspassagers.com/2011/01/a-working-curlopt_writefunction-function-for-libcurl/

多线程卷曲应用程序存在内存分配问题

2 个答案: