# 自定义下载器 下载器一共分为三种：**普通下载器**、**支持保持session的下载器**以及**浏览器渲染下载器**。默认已经在框架中内置，setting中的配置如下 ``` DOWNLOADER = "feapder.network.downloader.RequestsDownloader" # 请求下载器 SESSION_DOWNLOADER = "feapder.network.downloader.RequestsSessionDownloader" RENDER_DOWNLOADER = "feapder.network.downloader.SeleniumDownloader" # 渲染下载器 ``` - session下载器当配置中`USE_SESSION = True`时会启用 - 渲染下载器当使用浏览器下载功能时会启用 这些下载器均为插件的形式，我们可以自定义 ## 自定义普通下载器 1. 编写下载器。如在 `xxx-spider/downloader/my_downloader.py `下自定义了如下下载器 ``` import requests from feapder.network.downloader.base import Downloader from feapder.network.response import Response class RequestsDownloader(Downloader): def download(self, request) -> Response: response = requests.request( request.method, request.url, **request.requests_kwargs ) # 将requests的response转化为feapder的Response 对象，方便后续解析时使用xpath、re等方法 response = Response(response) return response ``` 注：这里返回的response对象不强制要求为是feapder的Response。返回值会传到解析函数的response参数里，若返回的是文本，则接收到的也是文本。 但为了代码可读性，建议将返回值转为feapder的Response后再返回。 转feapder的Response的方式有如下几种 ``` # 方式1 # response参数为reqeusts的response Response(response) # 方式2 Response.from_text(text="html内容") ``` 2. 在settings中指定下载器 ``` DOWNLOADER = "downloader.my_downloader.RequestsDownloader" ``` ## 自定义session下载器 1. 和普通下载器一样，都是继承`Downloader`，如何保持session，可自定义。代码示例 `xxx-spider/downloader/my_downloader.py ` ``` class RequestsSessionDownloader(Downloader): session = None @property def _session(self): if not self.__class__.session: self.__class__.session = requests.Session() # pool_connections – 缓存的 urllib3 连接池个数 pool_maxsize – 连接池中保存的最大连接数 http_adapter = HTTPAdapter(pool_connections=1000, pool_maxsize=1000) # 任何使用该session会话的 HTTP 请求，只要其 URL 是以给定的前缀开头，该传输适配器就会被使用到。 self.__class__.session.mount("http", http_adapter) return self.__class__.session def download(self, request) -> Response: response = self._session.request( request.method, request.url, **request.requests_kwargs ) response = Response(response) return response ``` 2. 在settings中指定下载器 ``` SESSION_DOWNLOADER = "downloader.my_downloader.RequestsSessionDownloader" ``` 注意，这里要配置 `SESSION_DOWNLOADER` ## 自定义浏览器渲染下载器 1. 编写下载器 `xxx-spider/downloader/my_downloader.py ` **若浏览器框架本身不支持多线程，但想在多线程中使用，如playwright使用，参考如下：** ``` import feapder.setting as setting import feapder.utils.tools as tools from feapder.network.downloader.base import RenderDownloader from feapder.network.response import Response from feapder.utils.webdriver import WebDriverPool, PlaywrightDriver class MyDownloader(RenderDownloader): webdriver_pool: WebDriverPool = None @property def _webdriver_pool(self): if not self.__class__.webdriver_pool: self.__class__.webdriver_pool = WebDriverPool( **setting.PLAYWRIGHT, driver_cls=PlaywrightDriver, thread_safe=True ) return self.__class__.webdriver_pool def download(self, request) -> Response: # 代理优先级 自定义 > 配置文件 > 随机 if request.custom_proxies: proxy = request.get_proxy() elif setting.PLAYWRIGHT.get("proxy"): proxy = setting.PLAYWRIGHT.get("proxy") else: proxy = request.get_proxy() # user_agent优先级 自定义 > 配置文件 > 随机 if request.custom_ua: user_agent = request.get_user_agent() elif setting.PLAYWRIGHT.get("user_agent"): user_agent = setting.PLAYWRIGHT.get("user_agent") else: user_agent = request.get_user_agent() cookies = request.get_cookies() url = request.url render_time = request.render_time or setting.PLAYWRIGHT.get("render_time") wait_until = setting.PLAYWRIGHT.get("wait_until") or "domcontentloaded" if request.get_params(): url = tools.joint_url(url, request.get_params()) driver: PlaywrightDriver = self._webdriver_pool.get( user_agent=user_agent, proxy=proxy ) try: if cookies: driver.url = url driver.cookies = cookies driver.page.goto(url, wait_until=wait_until) if render_time: tools.delay_time(render_time) html = driver.page.content() response = Response.from_dict( { "url": driver.page.url, "cookies": driver.cookies, "_content": html.encode(), "status_code": 200, "elapsed": 666, "headers": { "User-Agent": driver.user_agent, "Cookie": tools.cookies2str(driver.cookies), }, } ) response.driver = driver response.browser = driver return response except Exception as e: self._webdriver_pool.remove(driver) raise e def close(self, driver): if driver: self._webdriver_pool.remove(driver) def put_back(self, driver): """ 释放浏览器对象 """ self._webdriver_pool.put(driver) def close_all(self): """ 关闭所有浏览器 """ # 不支持 # self._webdriver_pool.close() pass ``` 这里使用了WebDriverPool，参数`thread_safe=True`，即要保证使用时的线程安全，确保同个浏览器对象只能被同一个线程调用 **若浏览器框架本身支持多线程，如selenium，则参考如下** ``` import feapder.setting as setting import feapder.utils.tools as tools from feapder.network.downloader.base import RenderDownloader from feapder.network.response import Response from feapder.utils.webdriver import WebDriverPool, SeleniumDriver class MyDownloader(RenderDownloader): webdriver_pool: WebDriverPool = None @property def _webdriver_pool(self): if not self.__class__.webdriver_pool: self.__class__.webdriver_pool = WebDriverPool( **setting.WEBDRIVER, driver=SeleniumDriver ) return self.__class__.webdriver_pool def download(self, request) -> Response: # 代理优先级 自定义 > 配置文件 > 随机 if request.custom_proxies: proxy = request.get_proxy() elif setting.WEBDRIVER.get("proxy"): proxy = setting.WEBDRIVER.get("proxy") else: proxy = request.get_proxy() # user_agent优先级 自定义 > 配置文件 > 随机 if request.custom_ua: user_agent = request.get_user_agent() elif setting.WEBDRIVER.get("user_agent"): user_agent = setting.WEBDRIVER.get("user_agent") else: user_agent = request.get_user_agent() cookies = request.get_cookies() url = request.url render_time = request.render_time or setting.WEBDRIVER.get("render_time") if request.get_params(): url = tools.joint_url(url, request.get_params()) browser: SeleniumDriver = self._webdriver_pool.get( user_agent=user_agent, proxy=proxy ) try: browser.get(url) if cookies: browser.cookies = cookies # 刷新使cookie生效 browser.get(url) if render_time: tools.delay_time(render_time) html = browser.page_source response = Response.from_dict( { "url": browser.current_url, "cookies": browser.cookies, "_content": html.encode(), "status_code": 200, "elapsed": 666, "headers": { "User-Agent": browser.user_agent, "Cookie": tools.cookies2str(browser.cookies), }, } ) response.driver = browser response.browser = browser return response except Exception as e: self._webdriver_pool.remove(browser) raise e def close(self, driver): if driver: self._webdriver_pool.remove(driver) def put_back(self, driver): """ 释放浏览器对象 """ self._webdriver_pool.put(driver) def close_all(self): """ 关闭所有浏览器 """ self._webdriver_pool.close() ``` 2. 在settings中指定下载器 ``` RENDER_DOWNLOADER = "downloader.my_downloader.MyDownloader" ``` 注，这里要写`RENDER_DOWNLOADER`