97 lines
3.6 KiB
Python
97 lines
3.6 KiB
Python
import scrapy
|
|
from scrapy import Request
|
|
|
|
|
|
class CementSpider(scrapy.Spider):
|
|
name = "cement"
|
|
start_urls = [
|
|
('水泥', "http://www.baiinfo.com/news/newscategory/4596/33/1"),
|
|
('水泥', "http://www.baiinfo.com/news/newscategory/4596/33/2"),
|
|
('水泥', "http://www.baiinfo.com/news/newscategory/4596/33/3"),
|
|
('水泥', "http://www.baiinfo.com/news/newscategory/4596/33/4"),
|
|
('水泥', "http://www.baiinfo.com/news/newscategory/4596/33/5"),
|
|
('水泥', "http://www.baiinfo.com/news/newscategory/4596/33/6"),
|
|
('水泥', "http://www.baiinfo.com/news/newscategory/4596/33/7"),
|
|
('水泥', "http://www.baiinfo.com/news/newscategory/4596/33/8"),
|
|
('水泥', "http://www.baiinfo.com/news/newscategory/4596/33/9"),
|
|
('水泥', "http://www.baiinfo.com/news/newscategory/4596/33/10"),
|
|
]
|
|
cookie = None
|
|
user_agent = None
|
|
|
|
def start_requests(self):
|
|
for source, url in self.start_urls:
|
|
yield Request(
|
|
method='GET',
|
|
url=url,
|
|
headers={
|
|
'User-Agent': self.user_agent,
|
|
},
|
|
cookies=self.cookie,
|
|
meta={'source': source, 'type': 'home'}
|
|
)
|
|
|
|
def parse(self, response, **kwargs):
|
|
if response.meta['type'] == 'home':
|
|
yield from self.parse_home(response)
|
|
elif response.meta['type'] == 'list_page':
|
|
yield from self.parse_list_page(response)
|
|
|
|
def parse_home(self, response):
|
|
for item in response.xpath('//*[@id="__nuxt"]/div/div[5]/div/div[2]/div[1]/div[1]/div[2]/ul/li/a'):
|
|
title = item.xpath('text()').get()
|
|
if '福建水泥市场参考价格' in title:
|
|
print(title, 'http://www.baiinfo.com{}'.format(item.xpath('@href').get()))
|
|
yield Request(
|
|
method='GET',
|
|
url='http://www.baiinfo.com{}'.format(item.xpath('@href').get()),
|
|
headers={
|
|
'User-Agent': self.user_agent,
|
|
},
|
|
cookies=self.cookie,
|
|
meta={'source': title, 'type': 'list_page'}
|
|
)
|
|
|
|
def parse_list_page(self, response):
|
|
date = response.xpath('//*[@id="__nuxt"]/div/div[5]/div/div[2]/div[1]/div[2]/p[1]/span[4]/text()').get()
|
|
date = date.strip('日').replace('年', '-').replace('月', '-')
|
|
|
|
for item in response.xpath('//tr[position()>2]'):
|
|
block_1 = [cell.get() for cell in item.xpath('td/span/span/span/span/span/text()') if cell.get()]
|
|
price, *_ = block_1
|
|
|
|
block_2 = [cell.get() for cell in item.xpath('td/span/span/span/span/span/span/text()') if cell.get()]
|
|
spec, name, pack, source, _, fluctuating = block_2
|
|
|
|
yield {
|
|
'name': name,
|
|
'price': price,
|
|
'spec': spec,
|
|
'pack': pack,
|
|
'date': date,
|
|
'source': source,
|
|
'fluctuating': int(fluctuating)
|
|
}
|
|
|
|
|
|
if __name__ == '__main__':
|
|
import json
|
|
|
|
from spiders import run_spider, BaiinfoCookieTools
|
|
from commons.models.cement import Cement
|
|
from core.factory import ClientApp
|
|
|
|
# cookie 读取
|
|
cookie = BaiinfoCookieTools.get_cookies()
|
|
# 爬取
|
|
CementSpider.cookie = cookie
|
|
CementSpider.user_agent = BaiinfoCookieTools.user_agent
|
|
file_path = run_spider(CementSpider)
|
|
# 入库
|
|
data = json.loads(open(file_path, 'r', encoding='utf-8').read())
|
|
with ClientApp().app_context():
|
|
for item in data:
|
|
print(item)
|
|
Cement(**item).upsert()
|
|
|