清华大佬耗费三个月吐血整理的几百G的资源，免费分享！....>>>

# encoding:utf-8
import sys
reload(sys)
sys.setdefaultencoding("utf8")
import urllib, urllib2
from pprint import pprint
from collections import Counter, OrderedDict
from bs4 import BeautifulSoup
from jinja2 import Environment, PackageLoader


class HexunQihuo(object):
    def __init__(self):
        self.all_href = []

    def __get_max_index(self, soup):
        listdh = soup.find('div', class_='listdh')
        s1 = str(listdh).split('maxPage = ')[1]
        s2 = s1.split(';')[0]
        return int(s2)

    def fetch_data(self, qtype, qname):
        '''
          qtype is in [industrynews, agriculturenews, nyzx] 
        # http://futures.hexun.com/industrynews/index.html') # 金属
        # http://futures.hexun.com/agriculturenews/index.html') # 农副资讯 
        # http://futures.hexun.com/nyzx/index.html  # 能源资讯
        '''
        self.qtype = qtype
        self.qname = qname
        f = urllib.urlopen('http://futures.hexun.com/%s/index.html' % qtype) #
        soup = BeautifulSoup(f.read(), 'html.parser')
        keywd = qname.decode('utf-8')
        temp01 = soup.find('div', class_='temp01')
        content = temp01.select('li > a')
        hrefs = [a['href'] +' '+ a.string for a in content if keywd in a.string]
        self.all_href.extend(hrefs)
        max_idx = self.__get_max_index(soup)

        do_range = range(670, max_idx)[::-1]
        for idx in do_range:
            print idx
            f = urllib.urlopen('http://futures.hexun.com/%s/index-%s.html' % (qtype, idx))
            soup = BeautifulSoup(f.read(), 'html.parser')
            temp01 = soup.find('div', class_='temp01')
            content = temp01.select('li > a')
            hrefs = [a['href'] +' '+ a.string for a in content if keywd in a.string]
            self.all_href.extend(hrefs)
        self.all_hrefstr = '\n'.join(self.all_href)
        return self

    def write_txt(self):
        with open('file.txt', 'w') as ff:
            ff.write(str(self.all_hrefstr))

    def write_html(self):
        # hexunqihuo 是py的文件名  在同级目录下建个templates文件夹
        env = Environment(loader=PackageLoader('hexunqihuo', 'templates'))
        data = OrderedDict()
        for href in self.all_href:
            item = href.split(' ')
            data[item[0]] = item[1]
        template = env.get_template('template.html') # templates/template.html
        html = template.render(data=data)
        htmlfile = 'files/%s.html' % self.qname
        with open(htmlfile, 'w') as ff:
            ff.write(html)


if __name__ == '__main__':

    HexunQihuo().fetch_data('industrynews', u'螺纹钢').write_html()