#include #include #include #include #include #include #include #include #include #include #ifdef DMALLOC #include #endif int http_getpage(const char *url, char *databuf, int max_len) { /* Split domain name */ if (memcmp(url, "http://", 7) != 0) { fprintf(stderr, "Invalid url %s\n", url); return HTTP_INVALIDURL; } int domainlen; domainlen = strstr(url + 7, "/") - (url + 7); if (domainlen < 0) domainlen = strlen(url + 7); struct hostent *host; char *domain = malloc(domainlen + 1); memcpy(domain, url + 7, domainlen); domain[domainlen] = '\0'; /* Check if port included */ int port = 80; char *portpos = strstr(domain, ":"); if (portpos != NULL) { port = atoi(portpos + 1); *portpos = '\0'; } /* Get host address */ host = gethostbyname(domain); if (host == NULL) { fprintf(stderr, "Domain not found %s\n", domain); free(domain); return HTTP_DOMAINNOTFOUND; } free(domain); /* Connect to server */ struct sockaddr_in serv_addr; int s = socket(AF_INET, SOCK_STREAM, 0); FILE *sfile = fdopen(s, "w+"); memset(&serv_addr, 0, sizeof(serv_addr)); serv_addr.sin_family = AF_INET; memcpy(&serv_addr.sin_addr.s_addr, host->h_addr, sizeof(serv_addr.sin_addr.s_addr)); serv_addr.sin_port = htons(port); if (connect(s, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) { fprintf(stderr, "Connect to %s failed", domain); perror(""); return HTTP_CONNECTFAILED; } /* Send request */ fprintf(sfile, "GET %s HTTP/1.0\r\n\r\n", url); /* Get response */ char linebuf[256]; if (fgets(linebuf, 256, sfile) == NULL) { fprintf(stderr, "Read failed\n"); return HTTP_CONNECTFAILED; } /* Get response code */ char *codestart = strstr(linebuf, " "); if (memcmp(linebuf, "HTTP", 4) != 0 || codestart == NULL) { fprintf(stderr, "Protocol error, data %s\n", linebuf); return HTTP_CONNECTFAILED; } int rcode = atoi(codestart); if (rcode != 200) { fprintf(stderr, "Response code %d\n", rcode); return HTTP_STATUSFAIL; } /* Wait for blank line */ int empty = 0; while (fgets(linebuf, 256, sfile)) { int i; empty = 1; for (i = 0; i < strlen(linebuf); i++) { if (!isspace(linebuf[i])) { empty = 0; break; } } if (empty) break; } if (!empty) { fprintf(stderr, "No blank line found\n"); return HTTP_CONNECTFAILED; } int count = fread(databuf, 1, max_len, sfile); fclose(sfile); return count; } /* Asynchronous downloading */ struct dlthread { struct dlthread *next; char *url; char *databuf; int max_len; int complete; int returnval; int timestamp; pthread_t thread; }; static struct dlthread *dlstack = NULL; struct dlthread *dlstack_add(const char *url, int max_len) { struct dlthread *newitem = malloc(sizeof(struct dlthread)); newitem->next = dlstack; dlstack = newitem; newitem->url = malloc(strlen(url) + 1); strcpy(newitem->url, url); newitem->databuf = malloc(max_len); newitem->max_len = max_len; newitem->complete = 0; newitem->returnval = 0; newitem->timestamp = time(NULL); return newitem; } void dlstack_remove(struct dlthread *item) { free(item->url); free(item->databuf); struct dlthread *parent = dlstack; if (dlstack == item) { dlstack = item->next; } else { while (parent->next != item) parent = parent->next; parent->next = item->next; } free(item); } struct dlthread *dlstack_find(const char *url) { struct dlthread *p = dlstack; while (p != NULL && strcmp(p->url, url) != 0) p = p->next; return p; } void *dlthread_getpage(void *arg) { struct dlthread *item = (struct dlthread *) arg; item->returnval = http_getpage(item->url, item->databuf, item->max_len); if (item->returnval < 0) item->databuf[0] = '\0'; item->complete = 1; pthread_exit(NULL); } int http_getpage_async(const char *url, char *databuf, int max_len, int *timestamp) { static int bytes_in_hour = 0, last_hour = 0; const int max_bytes_in_hour = 400000; struct dlthread *p; const int timeout = 20; int len; if (last_hour + 3600 < time(NULL)) { last_hour = time(NULL); bytes_in_hour = 0; } p = dlstack_find(url); if (p && !p->complete && p->timestamp + timeout < time(NULL)) { // Timeouted request fprintf(stderr, "Timeouted async http request at 0x%x\n", (unsigned int) p); p = NULL; } if (p) { if (!p->complete) return HTTP_ASYNCWAIT; if (p->max_len < max_len) max_len = p->max_len; len = strlen(p->databuf); bytes_in_hour += len; strncpy(databuf, p->databuf, max_len); *timestamp = p->timestamp; int r = p->returnval; dlstack_remove(p); return r; } /* Not found, add */ if (bytes_in_hour > max_bytes_in_hour) { fprintf(stderr, "Transfer limit exceeded, %d bytes (%d bytes max)\n", bytes_in_hour, max_bytes_in_hour); return HTTP_RATELIMIT; } struct dlthread *item; item = dlstack_add(url, max_len); if (pthread_create(&item->thread, NULL, &dlthread_getpage, item) != 0) fprintf(stderr, "Error in pthread_create\n"); return HTTP_ASYNCWAIT; }