import time
import asyncio

import aiohttp
from bs4 import BeautifulSoup as bs

BASE_URL = "http://www.biqudu.com"
TITLE2URL = dict()
CONTENT = list()


async def fetch(url, callback=None, **kwarags):
    headers = {'User-Agent':'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'}
    sem = asyncio.Semaphore(5)  
    with (await sem):
        async with aiohttp.ClientSession() as session: 
            async with session.get(url, headers=headers) as res:
                page = await res.text()
                if callback:
                    callback(page, **kwarags)
                else:
                    return page


def parse_url(page):
    soup = bs(page, "lxml")
    dd_a_doc = soup.select("dd > a")
    for a_doc in dd_a_doc:
        article_page_url = a_doc['href']
        article_title = a_doc.get_text()
        if article_page_url:
            TITLE2URL[article_title] = article_page_url


def parse_body(page, **kwarags):
    title = kwarags.get('title', '')
    print("{}".format(title))
    soup = bs(page, "lxml")
    content_doc = soup.find("div", id="content")
    content_text = content_doc.get_text().replace('readx();', '').replace('    ', "\r\n")
    content = "%s\n%s\n\n" % (title, content_text)
    CONTENT.append(content)


def main():
    t0 = time.time()
    loop = asyncio.get_event_loop()
    loop.run_until_complete(fetch(BASE_URL+"/43_43074/", callback=parse_url))
    tasks = [fetch(BASE_URL + page_url, callback=parse_body, title=title) for title, page_url in TITLE2URL.items()]
    loop.run_until_complete(asyncio.gather(*tasks[:500]))
    loop.close()
    elapsed = time.time() - t0
    print("cost {}".format(elapsed))


if __name__ == "__main__":
    main()