# encoding:utf-8 import sys reload(sys) sys.setdefaultencoding("utf8") import urllib, urllib2 from pprint import pprint from collections import Counter, OrderedDict from bs4 import BeautifulSoup from jinja2 import Environment, PackageLoader class HexunQihuo(object): def __init__(self): self.all_href = [] def __get_max_index(self, soup): listdh = soup.find('div', class_='listdh') s1 = str(listdh).split('maxPage = ')[1] s2 = s1.split(';')[0] return int(s2) def fetch_data(self, qtype, qname): ''' qtype is in [industrynews, agriculturenews, nyzx] # http://www.51sjk.com/Upload/Articles/1/0/320/320962_20220812160414468.html') # 金属 # http://www.51sjk.com/Upload/Articles/1/0/320/320962_20220812160414875.html') # 农副资讯 # http://www.51sjk.com/Upload/Articles/1/0/320/320962_20220812160415000.html # 能源资讯 ''' self.qtype = qtype self.qname = qname f = urllib.urlopen('http://www.51sjk.com/Upload/Articles/1/0/320/320962_20220812160415187.html' % qtype) # soup = BeautifulSoup(f.read(), 'html.parser') keywd = qname.decode('utf-8') temp01 = soup.find('div', class_='temp01') content = temp01.select('li > a') hrefs = [a['href'] +' '+ a.string for a in content if keywd in a.string] self.all_href.extend(hrefs) max_idx = self.__get_max_index(soup) do_range = range(670, max_idx)[::-1] for idx in do_range: print idx f = urllib.urlopen('http://www.51sjk.com/Upload/Articles/1/0/320/320962_20220812160415390.html' % (qtype, idx)) soup = BeautifulSoup(f.read(), 'html.parser') temp01 = soup.find('div', class_='temp01') content = temp01.select('li > a') hrefs = [a['href'] +' '+ a.string for a in content if keywd in a.string] self.all_href.extend(hrefs) self.all_hrefstr = '\n'.join(self.all_href) return self def write_txt(self): with open('file.txt', 'w') as ff: ff.write(str(self.all_hrefstr)) def write_html(self): # hexunqihuo 是py的文件名 在同级目录下建个templates文件夹 env = Environment(loader=PackageLoader('hexunqihuo', 'templates')) data = OrderedDict() for href in self.all_href: item = href.split(' ') data[item[0]] = item[1] template = env.get_template('template.html') # templates/template.html html = template.render(data=data) htmlfile = 'files/%s.html' % self.qname with open(htmlfile, 'w') as ff: ff.write(html) if __name__ == '__main__': HexunQihuo().fetch_data('industrynews', u'螺纹钢').write_html()
用户登录
还没有账号?立即注册
用户注册
投稿取消
文章分类: |
|
还能输入300字
上传中....