import gevent
from gevent import monkey
monkey.patch_all()
import requests

def down(url):
    print len(requests.get(url).content)
urls = ['http://www.baidu.com','http://www.sina.com.cn']
spawns = []
for url in urls:
    spawns.append(gevent.spawn(down, url))

gevent.joinall(spawns)

import gevent
from gevent import monkey, Greenlet
monkey.patch_all()
from gevent.pool import Pool
import requests

p = Pool(2)#设置并发数为2
def down(url):
    print len(requests.get(url).content)

urls = ['http://www.baidu.com','http://www.sina.com.cn']
for url in urls:
    p.spawn(down, url)

def buildurl():
    while True:
        gevent.sleep(0)#添加这条后才可以切换到其它任务
        print u"检测下载地址"#这里可以动态添加下载任务

Greenlet.spawn(buildurl)
loop = gevent.core.loop()
loop.run()

#-*- encoding:utf-8 -*-
import gevent
from gevent import monkey
from gevent.pool import Pool
import os, urllib2

monkey.patch_all()

class ComicDown():
    process = 5

    def __init__(self, poolsize=5):
        self.urls = []
        self.pool = Pool(poolsize)#同时并发数

    def addurl(self, url, headers, savefile):
        self.pool.spawn(self.down, url, headers, savefile)#将下载任务添加到任务池

    def down(self, url, headers, savefile):
        try:
            request = urllib2.Request(url)
            for h in headers:
                request.add_header(h[0],h[1])

            opener = urllib2.build_opener()
            img = opener.open(request).read()
            open(savefile, 'wb').write(img)#将下载结果写入文件
            print url,'Complete!'
        except Exception, e:
            print url, 'Error'
            print e

    def join(self):
        self.pool.join()

cd = ComicDown()
def start(path):
    f = open(os.path.join(path, 'link'))


    line = f.readline()
    while line:
        urlinfo = {}
        urlinfo['url'] = line.split('\t')[0]
        urlinfo['headers'] = [['referer',line.split('\t')[1]]]
        urlinfo['savefile'] = os.path.join(path,urlinfo['url'].split('/')[-1:][0])
        line = f.readline()
        if not os.path.exists(urlinfo['savefile']):
            cd.addurl(**urlinfo)

#coding:utf-8
from pyquery import PyQuery as pq
from optparse import OptionParser
import os, re, getImgs

def getparser():
    parser = OptionParser()
    parser.add_option('-u','--url', dest='url',help=u'要采集的漫画地址', metavar='url')
    options, args = parser.parse_args()
    return options, args

def getImgList(url):
    '''获取图片地址列表'''
    content = pq(url=url).html()
    jpgs = re.findall(r"imanhua_\d{3}", content)
    out = []
    d1, d2 = re.findall(r'(\d{4}).*(\d{5})', url)[0]

    for j in jpgs:
        out.append('http://t5.imanhua.com/Files/Images/%s/%s/%s.jpg\thttp://www.imanhua.com/comic/%s/list_%s.html'%\
            (d1,d2,j,d1,d2))
    return out

d = pq(url=getparser()[0].url)
site = '/'.join(getparser()[0].url.split('/')[:3])
print 'site',site
title = d('div.bookInfo h1').text()
if not os.path.exists('./comic/%s'%title):
    os.mkdir('./comic/%s'%title)#新建目录

print 'title', title
for i in d('ul#subBookList li a'):
    tmp = i.values()
    if not os.path.exists('./comic/%s/%s'%(title,tmp[1])):
        os.mkdir('./comic/%s/%s'%(title,tmp[1]))
    if not os.path.exists('./comic/%s/%s/link'%(title, tmp[1])):
        jpglist = getImgList(site+tmp[0])
        f = open('./comic/%s/%s/link'%(title, tmp[1]), 'wb')
        for j in jpglist:
            f.write(j+'\r\n')
        f.close()
    #print ('getImgs.py -p %s/comic/%s/%s'%(os.getcwd().decode('cp936'),title,tmp[1])).encode('cp936')
    #os.system(('getImgs.py -p %s/comic/%s/%s'%(os.getcwd().decode('cp936'),title,tmp[1])).encode('cp936'))
    try:
        getImgs.start(('%s/comic/%s/%s'%(os.getcwd().decode('cp936'),title,tmp[1])).encode('cp936'))
    except:
        print u'%s 错误'%tmp[1]

getImgs.cd.join()