81 lines
2.1 KiB
Python
81 lines
2.1 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""
|
||
Created on 2022-06-10 14:30:54
|
||
---------
|
||
@summary:
|
||
---------
|
||
@author: Boris
|
||
"""
|
||
|
||
import feapder
|
||
from feapder import ArgumentParser
|
||
|
||
|
||
class TestTaskSpider(feapder.TaskSpider):
|
||
def add_task(self):
|
||
# 加种子任务 框架会调用这个函数，方便往redis里塞任务，但不能写成死循环。实际业务中可以自己写个脚本往redis里塞任务
|
||
self._redisdb.zadd(self._task_table, {"id": 1, "url": "https://www.baidu.com"})
|
||
|
||
def start_requests(self, task):
|
||
task_id, url = task
|
||
yield feapder.Request(url, task_id=task_id)
|
||
|
||
def parse(self, request, response):
|
||
# 提取网站title
|
||
print(response.xpath("//title/text()").extract_first())
|
||
# 提取网站描述
|
||
print(response.xpath("//meta[@name='description']/@content").extract_first())
|
||
print("网站地址: ", response.url)
|
||
|
||
# mysql 需要更新任务状态为做完 即 state=1
|
||
# yield self.update_task_batch(request.task_id)
|
||
|
||
|
||
def start(args):
|
||
"""
|
||
用mysql做种子表
|
||
"""
|
||
spider = TestTaskSpider(
|
||
task_table="spider_task",
|
||
task_keys=["id", "url"],
|
||
redis_key="test:task_spider",
|
||
keep_alive=True,
|
||
)
|
||
if args == 1:
|
||
spider.start_monitor_task()
|
||
else:
|
||
spider.start()
|
||
|
||
|
||
def start2(args):
|
||
"""
|
||
用redis做种子表
|
||
"""
|
||
spider = TestTaskSpider(
|
||
task_table="spider_task2",
|
||
task_table_type="redis",
|
||
redis_key="test:task_spider",
|
||
keep_alive=True,
|
||
use_mysql=False,
|
||
)
|
||
if args == 1:
|
||
spider.start_monitor_task()
|
||
else:
|
||
spider.start()
|
||
|
||
|
||
if __name__ == "__main__":
|
||
parser = ArgumentParser(description="测试TaskSpider")
|
||
|
||
parser.add_argument(
|
||
"--start", type=int, nargs=1, help="用mysql做种子表 (1|2）", function=start
|
||
)
|
||
parser.add_argument(
|
||
"--start2", type=int, nargs=1, help="用redis做种子表 (1|2）", function=start2
|
||
)
|
||
|
||
parser.start()
|
||
|
||
# 下发任务 python3 test_task_spider.py --start 1
|
||
# 采集 python3 test_task_spider.py --start 2
|