From 9de1356dfa3cff26a0c144d1380b6ea7a48ccb7c Mon Sep 17 00:00:00 2001 From: DIYgod Date: Mon, 30 Dec 2019 19:17:54 +0800 Subject: [PATCH] docs: English docs sync, #3051 --- docs/en/install/README.md | 278 +++++++-------- docs/en/joinus/README.md | 688 +++++++++++++++++--------------------- docs/install/README.md | 2 +- docs/joinus/README.md | 57 +--- docs/support/README.md | 1 + docs/university.md | 26 +- 6 files changed, 480 insertions(+), 572 deletions(-) diff --git a/docs/en/install/README.md b/docs/en/install/README.md index eeda701b2..b812f1b4a 100644 --- a/docs/en/install/README.md +++ b/docs/en/install/README.md @@ -21,11 +21,94 @@ Deploy for public access may require: 1. [Heroku](https://devcenter.heroku.com/articles/getting-started-with-nodejs) 1. [Google App Engine](https://cloud.google.com/appengine/) +## Docker Compose Deployment + +### Install + +Download [docker-compose.yml](https://github.com/DIYgod/RSSHub/blob/master/docker-compose.yml) + +```bash +wget https://raw.githubusercontent.com/DIYgod/RSSHub/master/docker-compose.yml +``` + +Create a docker volume to persist Redis caches + +```bash +$ docker volume create redis-data +``` + +Launch + +```bash +$ docker-compose up -d +``` + +### Update + +Remove old containers + +```bash +$ docker-compose down +``` + +Then repeat the installation steps + +### Configuration + +Edit `environment` in [docker-compose.yml](https://github.com/DIYgod/RSSHub/blob/master/docker-compose.yml) + +## Docker Deployment + +### Install + +Execute the following command to pull RSSHub's docker image. + +```bash +$ docker pull diygod/rsshub +``` + +Start a RSSHub container + +```bash +$ docker run -d --name rsshub -p 1200:1200 diygod/rsshub +``` + +Visit [http://127.0.0.1:1200/](http://127.0.0.1:1200/), and enjoy it! ✅ + +Execute the following command to stop `RSSHub`. + +```bash +$ docker stop rsshub +``` + +### Update + +Remove old container + +```bash +$ docker stop rsshub +$ docker rm rsshub +``` + +Then repeat the installation steps + +### Configuration + +The simplest way to configure RSSHub container is via system environment variables. + +For example, adding `-e CACHE_EXPIRE=3600` will set the cache time to 1 hour. + +```bash +$ docker run -d --name rsshub -p 1200:1200 -e CACHE_EXPIRE=3600 -e GITHUB_ACCESS_TOKEN=example diygod/rsshub +``` + +To configure more options please refer to [Configuration](#configuration). + ## Manual Deployment -Manually deploy a new `RSSHub` instance to a location of your choice. +The most direct way to deploy `RSSHub`, you can follow the steps below to deploy`RSSHub` on your computer, server or anywhere. -### Download RSSHub +### Install Execute the following commands to download the source code @@ -62,25 +145,19 @@ Or $ yarn start ``` -Visit [http://127.0.0.1:1200/](http://127.0.0.1:1200/), and enjoy it! ✅ - -::: tip tip - -Refer to our [Guide](https://docs.rsshub.app/) for usage. Replace `https://rsshub.app/` with `http://localhost:1200` in any route example to see the effect. - -::: - -### Update - -Under `RSSHub`'s directory, execute the following commands to pull the latest source code for `RSSHub` +Or use [PM2](https://pm2.io/doc/zh/runtime/quick-start/) ```bash -$ git pull +$ pm2 start lib/index.js --name rsshub ``` +Visit [http://127.0.0.1:1200/](http://127.0.0.1:1200/), and enjoy it! ✅ + +Refer to our [Guide](https://docs.rsshub.app/en/) for usage. Replace `https://rsshub.app/` with `http://localhost:1200` in any route example to see the effect. + ### Configuration -`RSSHub` reads its configurations from `lib/config.js` or system environment variables. +RSSHub can be configured by setting environment variables. Create a `.env` file in the root directory of your project. Add environment-specific variables on new lines in the form of `NAME=VALUE`. For example: @@ -91,83 +168,20 @@ CACHE_EXPIRE=600 Please notice that it will not override already existed environment variables, more rules please refer to [dotenv](https://github.com/motdotla/dotenv) -To configure more options please refer to [Configuration](#configuration-3). +This deployment method does not include puppeteer and redis dependencies. Use the Docker Compose deployment method or deploy external dependencies yourself if you need it. -### Use Redis for caching +To configure more options please refer to [Configuration](#configuration). -By default, `RSSHub` caches everything for 5 minutes in RAM. Redis support is built-in. +### Update -::: tip tips - -Unless you are expecting high traffic or deploying in cluster-mode, Redis is not necessary. - -::: - -Change `CACHE_TYPE` to `redis`, RSSHub will try to connect to `redis://localhost:6379/`. For changing the target address, please refer to [Configuration](#configuration-3). - -## Docker Deployment - -Execute the following command to pull RSSHub's docker image. - -```bash -$ docker pull diygod/rsshub -``` - -Start a RSSHub container - -```bash -$ docker run -d --name rsshub -p 1200:1200 diygod/rsshub -``` - -Visit [http://127.0.0.1:1200/](http://127.0.0.1:1200/), and enjoy it! ✅ - -Execute the following command to stop `RSSHub`. - -```bash -$ docker stop rsshub -``` - -### Configuration - -The simplest way to configure RSSHub container is via system environment variables. - -For example, adding `-e CACHE_EXPIRE=3600` will set the cache time to 1 hour. - -```bash -$ docker run -d --name rsshub -p 1200:1200 -e CACHE_EXPIRE=3600 -e GITHUB_ACCESS_TOKEN=example diygod/rsshub -``` - -To configure more options please refer to [Configuration](#configuration-3). - -### docker-compose Deployment - -1. Create a docker volume to persist Redis caches - -```bash -$ docker volume create redis-data -``` - -1. Copy `lib/config.js` to `lib/config/config.js`, to avoid conflicts with master branch. Git will ignore this file as it contains sensitive information. - -1. Change `environment` section in [docker-compose.yml](https://github.com/DIYgod/RSSHub/blob/master/docker-compose.yml) to configure the corresponding option - - - `PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=1` skips puppeteer Chromium installation. Default to 1, requires `puppeteerWSEndpoint` in `lib/config.js` to be set with a remote Chrome Websocket address, otherwise relevant routes will not work. - - `USE_CHINA_NPM_REGISTRY=1` avoids GFW npm registry interference in mainland China. Default to 0. - -1. Deploy - -```bash -$ docker-compose up -d -``` - -1. Update +Under `RSSHub`'s directory, execute the following commands to pull the latest source code for `RSSHub` ```bash $ git pull -$ docker-compose build -$ docker-compose up -d ``` +Then repeat the installation steps + ## Heroku Deployment [![Deploy](https://i.imgur.com/e6ZcmUY.png)](https://heroku.com/deploy?template=https%3A%2F%2Fgithub.com%2FDIYgod%2FRSSHub) @@ -182,8 +196,6 @@ Please note, GAE free tier doesn't support Flexible Environment, please check th Node.js standard environment is still under beta, unknown or unexpected errors might be encountered during the deployment. -### Pull - Execute `git clone https://github.com/DIYgod/RSSHub.git` to pull the latest code ### app.yaml Settings @@ -236,7 +248,7 @@ env_variables: # [END app_yaml] ``` -### Launch +### Install Under RSSHub's root directory, execute the following commands to launch RSSHub @@ -250,19 +262,11 @@ You can access your `Google App Engine URL` to check the deployment status ## Configuration -### Application Configurations +Configure RSSHub by setting environment variables -`RSSHub` reads its configurations from `lib/config.js` or environment variables. +### Cache Configurations -::: tip - -Use environment variables is recommended to avoid conflicts during upgrade. - -::: - -`PORT`: listening port, default to `1200` - -`SOCKET`: listening Unix Socket, default to `null` +RSSHub supports two caching methods: memory and redis `CACHE_TYPE`: cache type, `memory` or `redis`, empty this value will disable caching, default to `memory` @@ -270,23 +274,13 @@ Use environment variables is recommended to avoid conflicts during upgrade. `CACHE_CONTENT_EXPIRE`: content cache expiry time in seconds, it will be recalculated when it is accessed, default to `1 * 60 * 60` -`LISTEN_INADDR_ANY`: open up for external access, default to `1` - -`REQUEST_RETRY`: retries allowed for failed requests, default to `2` - -`DEBUG_INFO`: display route information on homepage for debugging purpose, default to `true` - -`TITLE_LENGTH_LIMIT`: limit the length of feed title generated in bytes, an English alphabet counts as 1 byte, the rest such as Chinese, Japanese, Korean or Arabic counts as 2 bytes by design, default to `100` - `REDIS_URL`: Redis target address（invalid when `CACHE_TYPE` is set to memory）, default to `redis://localhost:6379/` `REDIS_PASSWORD`: Redis password（invalid when `CACHE_TYPE` is set to memory) -`HTTP_BASIC_AUTH_NAME`: Http basic authentication username, default to `usernam3`, please change asap +### Proxy Configurations -`HTTP_BASIC_AUTH_PASS`: Http basic authentication password, default to `passw0rd`, please change asap - -`LOGGER_LEVEL`: specifies the maximum [level](https://github.com/winstonjs/winston#logging-levels) of messages to the console and log file, default to `info` +Partial routes have a strict anti-crawler policy, and can be configured to use proxy `PROXY_PROTOCOL`: Using proxy, Supports socks, http, https @@ -298,31 +292,59 @@ Use environment variables is recommended to avoid conflicts during upgrade. `PROXY_URL_REGEX`: regex for url of enabling proxy, default to `.*` +### User Authentication Configurations + +Routes in `protected_route.js` will be protected using HTTP Basic Authentication. + +When adding feeds using RSS readers with HTTP Basic Authentication support, authentication information is required, eg：http://usernam3:passw0rd@localhost:1200/protected/rsshub/rss. + +`HTTP_BASIC_AUTH_NAME`: Http basic authentication username, default to `usernam3`, please change asap + +`HTTP_BASIC_AUTH_PASS`: Http basic authentication password, default to `passw0rd`, please change asap + +### Access Control Configuration + +Access control includes a whitelist and a blacklist, support IP and route, use `,` as the delimiter to separate multiple values. When both are defined, values in `BLACKLIST` will be disregarded. + +- `BLACKLIST`: the blacklist + +- `WHITELIST`: the blacklist. When set, values in `BLACKLIST` are disregarded. + +### Other Application Configurations + +`PORT`: listening port, default to `1200` + +`SOCKET`: listening Unix Socket, default to `null` + +`LISTEN_INADDR_ANY`: open up for external access, default to `1` + +`TITLE_LENGTH_LIMIT`: limit the length of feed title generated in bytes, an English alphabet counts as 1 byte, the rest such as Chinese, Japanese, Korean or Arabic counts as 2 bytes by design, default to `100` + +`REQUEST_RETRY`: retries allowed for failed requests, default to `2` + +`DEBUG_INFO`: display route information on homepage for debugging purpose, default to `true` + +`LOGGER_LEVEL`: specifies the maximum [level](https://github.com/winstonjs/winston#logging-levels) of messages to the console and log file, default to `info` + `NODE_NAME`: node name, used for load balancing, identify current node `PUPPETEER_WS_ENDPOINT`: Browser websocket endpoint which can be used as an argument to puppeteer.connect, refer to [browserWSEndpoint](https://pptr.dev/#?product=Puppeteer&version=v1.14.0&show=api-browserwsendpoint) `SENTRY`: [Sentry](https://sentry.io) dsn, used for error tracking -### User Authentication - -Routes in `protected_route.js` will be protected using HTTP Basic Authentication. - -When adding feeds using RSS readers with HTTP Basic Authentication support, authentication information is required, eg：http://usernam3:passw0rd@localhost:1200/protected/rsshub/rss. - ### Route-specific Configurations -- `pixiv`: [Registration](https://accounts.pixiv.net/signup) +- pixiv: [Registration](https://accounts.pixiv.net/signup) - `PIXIV_USERNAME`: Pixiv username - `PIXIV_PASSWORD`: Pixiv password -- `disqus`: [API Key application](https://disqus.com/api/applications/) +- disqus: [API Key application](https://disqus.com/api/applications/) - `DISQUS_API_KEY`: Disqus API -- `twitter`: [Application creation](https://apps.twitter.com) +- twitter: [Application creation](https://apps.twitter.com) - `TWITTER_CONSUMER_KEY`: Twitter Consumer Key, support multiple keys, split them with `,` @@ -330,28 +352,18 @@ When adding feeds using RSS readers with HTTP Basic Authentication support, auth - `TWITTER_TOKEN_{id}`: Twitter token's corresponding id, replace `{id}` with the id, the value is a combination of `consumer_key consumer_secret access_token access_token_secret` by a comma `,`. Eg. `{consumer_key},{consumer_secret},{access_token},{access_token_secret}`. -- `youtube`: [API Key application](https://console.developers.google.com/) +- youtube: [API Key application](https://console.developers.google.com/) - `YOUTUBE_KEY`: YouTube API Key -- `telegram`: [Bot application](https://telegram.org/blog/bot-revolution) +- telegram: [Bot application](https://telegram.org/blog/bot-revolution) - `TELEGRAM_TOKEN`: Telegram bot token -- `github`: [Access Token application](https://github.com/settings/tokens) +- github: [Access Token application](https://github.com/settings/tokens) - `GITHUB_ACCESS_TOKEN`: GitHub Access Token -- `mail`: +- mail: - `EMAIL_CONFIG_{email}`: Mail setting, replace `{email}` with the email account, replace `@` in email account with `.`, eg. `EMAIL_CONFIG_xxx.gmail.com`. the value is in the format of `password=password&host=server&port=port`, eg. `password=123456&host=imap.gmail.com&port=993` - -### Access Control - -Access control includes a whitelist and a blacklist, which is configured via `middleware/access-control.js` or environment variables. - -Support `IP` and `Route`, use `,` as the delimiter to separate multiple values. When both are defined, values in `BLACKLIST` will be disregarded. - -- `BLACKLIST`: the blacklist - -- `WHITELIST`: the blacklist. When set, values in `BLACKLIST` are disregarded. diff --git a/docs/en/joinus/README.md b/docs/en/joinus/README.md index 6fd625477..d4c23ac76 100644 --- a/docs/en/joinus/README.md +++ b/docs/en/joinus/README.md @@ -6,11 +6,26 @@ sidebar: auto We welcome all pull requests. Suggestions and feedback are also welcomed [here](https://github.com/DIYgod/RSSHub/issues). -## Submit new RSS source +## Join the discussion -### Step 1: Code the script +1. [Telegram Group](https://t.me/rsshub) +2. [GitHub Issues](https://github.com/DIYgod/RSSHub/issues) -Firstly, add a .js file for the new route in [/lib/router.js](https://github.com/DIYgod/RSSHub/blob/master/lib/router.js) +## Submit new RSS rule + +Before you start writing RSS rule, please make sure that the source site does not provide RSS. Some web pages will include a link element with type `application/atom+xml` or `application/rss+xml` in the HTML header to indicate the RSS link. + +### Debug + +First `yarn` or`npm install` to install dependencies, then execute `yarn dev` or`npm run dev`, open `http://localhost:1200` to see the effect, and the page will refresh automatically if files modified. + +### Add route + +Firstly, add a route in [/lib/router.js](https://github.com/DIYgod/RSSHub/blob/master/lib/router.js) + +### Code the script + +Create a new js script in [/lib/routes/](https://github.com/DIYgod/RSSHub/tree/master/lib/routes) corresponding to route path #### Acquiring Data @@ -22,285 +37,279 @@ Firstly, add a .js file for the new route in [/lib/router.js](https://github.com - Below is a list of data acquisition methods, ordered by the **「level of recommendation」** + 1. **Acquire data via API using got** - 1. **Acquire data via API using got** + Example：[/lib/routes/bilibili/coin.js](https://github.com/DIYgod/RSSHub/blob/master/lib/routes/bilibili/coin.js)。 - Example：[/lib/routes/bilibili/coin.js](https://github.com/DIYgod/RSSHub/blob/master/lib/routes/bilibili/coin.js)。 + Acquiring data via the official API provided by the data source using got: - Acquiring data via the official API provided by the data source using got: + ```js + // Initiate a HTTP GET request + const response = await got({ + method: 'get', + url: `https://api.bilibili.com/x/space/coin/video?vmid=${uid}&jsonp=jsonp`, + headers: { + Referer: `https://space.bilibili.com/${uid}/`, + }, + }); - ```js - // Initiate a HTTP GET request - const response = await got({ - method: 'get', - url: `https://api.bilibili.com/x/space/coin/video?vmid=${uid}&jsonp=jsonp`, - }); + const data = response.data.data; // response.data is the data object returned from the previous request + // The object contains a nested object called data, thus response.data.data is the actual data needed here + ``` - const data = response.data.data; // response.data is the data object returned from the previous request - // The object contains a nested object called data, thus response.data.data is the actual data needed here - ``` + One of the leaf objects (response.data.data[0])： - One of the leaf objects (response.data.data[0])： + ```json + { + "aid": 33614333, + "videos": 2, + "tid": 20, + "tname": "宅舞", + "copyright": 1, + "pic": "http://i0.hdslb.com/bfs/archive/5649d7fe6ff7f7b431300fc1a0db80d3f174cacd.jpg", + "title": "【赤九玖】响喜乱舞【和我一起狂舞吧，团长大人(✧◡✧)】", + "pubdate": 1539259203, + "ctime": 1539249536, + "desc": "编舞出处：av31984673\n真心好喜欢这个舞和这首歌，居然恰巧被邀请跳了，感谢《苍之纪元》官方的邀请。这次cos的是游戏的新角色缪斯。然而时间有限很多地方还有很多不足。也没跳够，以后私下还会继续练习，希望能学到更多动作，也能为了有机会把它跳的更好。 \n摄影：绯山圣瞳九命猫 \n后期：炉火" + // some more data.... + } + ``` - ```json - { - "aid": 33614333, - "videos": 2, - "tid": 20, - "tname": "宅舞", - "copyright": 1, - "pic": "http://i0.hdslb.com/bfs/archive/5649d7fe6ff7f7b431300fc1a0db80d3f174cacd.jpg", - "title": "【赤九玖】响喜乱舞【和我一起狂舞吧，团长大人(✧◡✧)】", - "pubdate": 1539259203, - "ctime": 1539249536, - "desc": "编舞出处：av31984673\n真心好喜欢这个舞和这首歌，居然恰巧被邀请跳了，感谢《苍之纪元》官方的邀请。这次cos的是游戏的新角色缪斯。然而时间有限很多地方还有很多不足。也没跳够，以后私下还会继续练习，希望能学到更多动作，也能为了有机会把它跳的更好。 \n摄影：绯山圣瞳九命猫 \n后期：炉火" - // some more data.... - } - ``` + Processing the data further to generate objects in accordance with RSS specification, mainly title, link, description, publish time, then assign them to ctx.state.data, [produce RSS feed](#produce-rss-feed)： - Processing the data further to generate objects in accordance with RSS specification, mainly title, link, description, publish time, then assign them to ctx.state.data, [produce RSS feed](#produce-rss-feed)： - - ```js - ctx.state.data = { - // the source title - title: `${name} 的 bilibili 投币视频`, - // the source link - link: `https://space.bilibili.com/${uid}`, - // the source description - description: `${name} 的 bilibili 投币视频`, - // iterate through all leaf objects - item: data.map((item) => ({ - // the article title - title: item.title, - // the article content - description: `${item.desc}
`, - // the article publish time - pubDate: new Date(item.time * 1000).toUTCString(), - // the article link - link: `https://www.bilibili.com/video/av${item.aid}`, - })), - }; - - // the route is now done - ``` - - 2. **Acquire data via HTML webpage using got** - - Data have to be acquired via HTML webpage if **no API was provided**, for example: [/lib/routes/jianshu/home.js](https://github.com/DIYgod/RSSHub/blob/master/lib/routes/jianshu/home.js)。 - - Acquiring data by scrapping the HTML using got: - - ```js - // Initiate a HTTP GET request - const response = await got({ - method: 'get', - url: 'https://www.jianshu.com', - }); - - const data = response.data; // response.data is the entire HTML source of the target page, returned from the previous request - ``` - - Parsing the HTML using cheerio: - - ```js - const $ = cheerio.load(data); // Load the HTML returned into cheerio - const list = $('.note-list li').get(); - // use cheerio selector, select all 'li' elements with 'class="note-list"', the result is an array of cheerio node objects - // use cheerio get() method to transform a cheerio node object array into a node array - - // PS：every cheerio node is a HTML DOM - // PPS：cheerio selector is almost identical to jquery selector - // Refer to cheerio docs：https://cheerio.js.org/ - ``` - - Use /jianshu/utils.js class to extract full-text: - - ```js - const result = await util.ProcessFeed(list, ctx.cache); - ``` - - The logic for full-text extraction in /jianshu/utils.js class： - - ```js - // define a function to load the article content - async function load(link) { - // get the article asynchronously - const response = await got.get(link); - // load the article content - const $ = cheerio.load(response.data); - - // parse the date - const date = new Date( - $('.publish-time') - .text() - .match(/\d{4}.\d{2}.\d{2} \d{2}:\d{2}/) - ); - - // handle the timezone - const timeZone = 8; - const serverOffset = date.getTimezoneOffset() / 60; - const pubDate = new Date(date.getTime() - 60 * 60 * 1000 * (timeZone + serverOffset)).toUTCString(); - - // extract the full-text - const description = $('.show-content-free').html(); - - // return the parsed result - return { description, pubDate }; - } - - // use Promise.all() to initiate requests in parallel - const result = await Promise.all( - // loop through every article - list.map(async (item) => { - const $ = cheerio.load(item); - - const $title = $('.title'); - // resolve the absolute URL - const itemUrl = url.resolve(host, $title.attr('href')); - - // form a new object to hold the data - const single = { - title: $title.text(), - link: itemUrl, - author: $('.nickname').text(), - guid: itemUrl, - }; - - // use tryGet() to query the cache - // if the query returns no result, query the data source via load() to get article content - const other = await caches.tryGet(itemUrl, async () => await load(itemUrl)); - - // merge two objects to form the final output - return Promise.resolve(Object.assign({}, single, other)); - }) - ); - ``` - - Assign the value of `result` to `ctx.state.data` - - ```js - ctx.state.data = { - title: '简书首页', - link: 'https://www.jianshu.com', - // select "content" property of - description: $('meta[name="description"]').attr('content'), - item: result, - }; - - // the route is now done - ``` - - 3. **Acquire data via page rendering using puppeteer** - - ::: tip tips - - This method consumes more resources and is less performant, use only when the above methods failed to acquire data, otherwise your pull requests will be rejected! - - ::: - - Seldomly, data source **provides no API and the page requires rendering** to acquire data, for example: [/lib/routes/sspai/series.js](https://github.com/DIYgod/RSSHub/blob/master/lib/routes/sspai/series.js) - - ```js - // use puppeteer util class, initialise a browser instance - const browser = await require('@/utils/puppeteer')(); - // open a new page - const page = await browser.newPage(); - // access the target link - const link = 'https://sspai.com/series'; - await page.goto(link); - // render the page - const html = await page.evaluate( - () => - // process on the rendered page - document.querySelector('div.new-series-wrapper').innerHTML - ); - // shutdown the browser - browser.close(); - ``` - - Parsing the HTML using cheerio: - - ```js - const $ = cheerio.load(html); // Load the HTML returned into cheerio - const list = $('div.item'); // // use cheerio selector, select all 'div class="item"' elements, the result is an array of cheerio node objects - - ``` - - Assign the value to `ctx.state.data` - - ```js - ctx.state.data = { - title: '少数派 -- 最新上架付费专栏', - link, - description: '少数派 -- 最新上架付费专栏', - item: list - .map((i, item) => ({ + ```js + ctx.state.data = { + // the source title + title: `${name} 的 bilibili 投币视频`, + // the source link + link: `https://space.bilibili.com/${uid}`, + // the source description + description: `${name} 的 bilibili 投币视频`, + // iterate through all leaf objects + item: data.map((item) => ({ // the article title - title: $(item) - .find('.item-title a') - .text() - .trim(), + title: item.title, + // the article content + description: `${item.desc}
`, + // the article publish time + pubDate: new Date(item.time * 1000).toUTCString(), // the article link - link: url.resolve( - link, - $(item) + link: `https://www.bilibili.com/video/av${item.aid}`, + })), + }; + + // the route is now done + ``` + + 2. **Acquire data via HTML webpage using got** + + Data have to be acquired via HTML webpage if **no API was provided**, for example: [/lib/routes/douban/explore.js](https://github.com/DIYgod/RSSHub/blob/master/lib/routes/douban/explore.js) + + Acquiring data by scrapping the HTML using got: + + ```js + // Initiate a HTTP GET request + const response = await got({ + method: 'get', + url: 'https://www.douban.com/explore', + }); + + const data = response.data; // response.data is the entire HTML source of the target page, returned from the previous request + ``` + + Parsing the HTML using cheerio: + + ```js + const $ = cheerio.load(data); // Load the HTML returned into cheerio + const list = $('div[data-item_id]'); + // use cheerio selector, select all 'div' elements with 'data-item_id' attribute, the result is an array of cheerio node objects + // use cheerio get() method to transform a cheerio node object array into a node array + + // PS：every cheerio node is a HTML DOM + // PPS：cheerio selector is almost identical to jquery selector + // Refer to cheerio docs：https://cheerio.js.org/ + ``` + + Use map to traverse the array and parse out the result of each item + + ```js + ctx.state.data = { + title: '豆瓣-浏览发现', + link: 'https://www.douban.com/explore', + item: + list && + list + .map((index, item) => { + item = $(item); + itemPicUrl = `${item.find('a.cover').attr('style')}`.replace('background-image:url(', '').replace(')', ''); + return { + title: item + .find('.title a') + .first() + .text(), + description: `作者：${item + .find('.usr-pic a') + .last() + .text()}
描述：${item.find('.content p').text()}
`, + link: item.find('.title a').attr('href'), + }; + }) + .get(), + }; + + // the route is now done + ``` + + 3. **Acquire data via page rendering using puppeteer** + + ::: tip tips + + This method consumes more resources and is less performant, use only when the above methods failed to acquire data, otherwise your pull requests will be rejected! + + ::: + + Seldomly, data source **provides no API and the page requires rendering** to acquire data, for example: [/lib/routes/sspai/series.js](https://github.com/DIYgod/RSSHub/blob/master/lib/routes/sspai/series.js) + + ```js + // use puppeteer util class, initialise a browser instance + const browser = await require('@/utils/puppeteer')(); + // open a new page + const page = await browser.newPage(); + // access the target link + const link = 'https://sspai.com/series'; + await page.goto(link); + // render the page + const html = await page.evaluate( + () => + // process on the rendered page + document.querySelector('div.new-series-wrapper').innerHTML + ); + // shutdown the browser + browser.close(); + ``` + + Parsing the HTML using cheerio: + + ```js + const $ = cheerio.load(html); // Load the HTML returned into cheerio + const list = $('div.item'); // // use cheerio selector, select all 'div class="item"' elements, the result is an array of cheerio node objects + + ``` + + Assign the value to `ctx.state.data` + + ```js + ctx.state.data = { + title: '少数派 -- 最新上架付费专栏', + link, + description: '少数派 -- 最新上架付费专栏', + item: list + .map((i, item) => ({ + // the article title + title: $(item) .find('.item-title a') - .attr('href') - ), - // the article author - author: $(item) - .find('.item-author') - .text() - .trim(), - })) - .get(), // use cheerio get() method to transform a cheerio node object array into a node array + .text() + .trim(), + // the article link + link: url.resolve( + link, + $(item) + .find('.item-title a') + .attr('href') + ), + // the article author + author: $(item) + .find('.item-author') + .text() + .trim(), + })) + .get(), // use cheerio get() method to transform a cheerio node object array into a node array + }; + + // the route is now done + + // PS: the route acts as a notifier of new articles, it does not provide access to the content behind the paywall, thus not content were fetched + ``` + + 4. **Use general configuration routing** + + A large number of websites can generate RSS through a configuration paradigm. + + The general configuration is to easily generate RSS by reading json data through cheerio (**CSS selector, jQuery function**) + + First we need a few data: + +1. RSS source link +    2. Data source link +    3. RSS title (not item title) + + ```js + const buildData = require('@/utils/common-config'); + module.exports = async (ctx) => { + ctx.state.data = await buildData({ + link: '', // RSS source link + url: '', // Data source link + title: '%title%', // Variables are used here, such as **% xxx%** will be parsed into variables with values of the same name under **params** + params: { + title: '', // RSS title + }, + }); }; - - // the route is now done - - // PS: the route acts as a notifier of new articles, it does not provide access to the content behind the paywall, thus not content were fetched ``` + Our RSS does not have any content for now, the content needs to be completed by `item` + Here is an example + + ```js + const buildData = require('@/utils/common-config'); + + module.exports = async (ctx) => { + const link = `https://www.uraaka-joshi.com/`; + ctx.state.data = await buildData({ + link, + url: link, + title: `%title%`, + params: { + title: '裏垢女子まとめ', + }, + item: { + item: '.content-main .stream .stream-item', + title: `$('.post-account-group').text() + ' - %title%'`, // Only supports js statements like $().xxx() + link: `$('.post-account-group').attr('href')`, // .text() means get the text of the element, .attr() means get the specified attribute + description: `$('.post .context').html()`, // .text() means get the text of the the html code + pubDate: `new Date($('.post-time').attr('datetime')).toUTCString()`, + guid: `new Date($('.post-time').attr('datetime')).getTime()`, + }, + }); + }; + ``` + + So far we have completed a simplest route + --- -#### Enable Caching +#### Use Cache -By default there is a global caching period set in `lib/config.js`, some sources might have a low update frequency, a longer caching period should be set. +All routes have a cache, the global cache time is set in `lib/config.js`, but the content returned by some interfaces is updated less frequently. At this time, you should set a longer cache time for these data. -- Save to cache: +For example, the bilibili column needs to get the full text of the article：[/lib/routes/bilibili/followings_article.js](https://github.com/DIYgod/RSSHub/blob/master/lib/routes/bilibili/followings_article.js) + +Since the full text of all articles cannot be got from one API, each article needs to be requested once, and these data are generally unchanged, so these data should be stored in the cache to avoid requesting repeatedly ```js -ctx.cache.set((key: string), (value: string)); // time is the caching period in seconds. +const description = await ctx.cache.tryGet(link, async () => { + const result = await got.get(link); + + const $ = cheerio.load(result.data); + $('img').each(function(i, e) { + $(e).attr('src', $(e).attr('data-src')); + }); + + return $('.article-holder').html(); +}); ``` -- Access the cache: - -```js -const value = await ctx.cache.get((key: string)); -``` - -For example: [/lib/routes/zhihu/daily.js](https://github.com/DIYgod/RSSHub/blob/master/lib/routes/zhihu/daily.js), the full-text extraction will be triggered even when the article was not updated. - -Given the update frequency is known, set the appropriate caching period to reuse the cache, will save resources and improve performance. - -```js -const key = 'daily' + story.id; // story.id is the unique identifier of each article -ctx.cache.set(key, item.description); // set cache -``` - -When the identical requests come in, reuse the cache： - -```js -const key = 'daily' + story.id; -const value = await ctx.cache.get(key); // query the cache to find the unique identifier -if (value) { - // return the cached data - item.description = value; // assign the cached data -} else { - // no cache found - // initiate request to the data source -} -``` +The implementation of tryGet can be seen [here](https://github.com/DIYgod/RSSHub/blob/master/lib/middleware/cache.js#L128). The first parameter is the cache key, the second parameter is the cache data acquisition method, and the third parameter is the cache time, it should not be passed in normally. The cache time defaults to [CACHE_CONTENT_EXPIRE](/en/install/#cache-configurations), and each time accessing the cache will recalculate the expiration time --- @@ -393,54 +402,7 @@ ctx.state.data = { --- -### Step 2: Add the script into router - -Add the script into [/lib/router.js](https://github.com/DIYgod/RSSHub/blob/master/lib/router.js) - -#### Example - -1. [bilibili/bangumi](https://github.com/DIYgod/RSSHub/blob/master/lib/routes/bilibili/bangumi.js) - -| Name | Description | -| ---------------------------------- | ---------------------------------------------------------------------------------- | -| Route | `/bilibili/bangumi/:seasonid` | -| Data Source | bilibili | -| Route Name | bangumi | -| Parameter 1 | :seasonid required | -| Parameter 2 | n/a | -| Parameter 3 | n/a | -| Route Path | `./routes/bilibili/bangumi` | -| the complete code in lib/router.js | `router.get('/bilibili/bangumi/:seasonid', require('./routes/bilibili/bangumi'));` | - -2. [github/issue](https://github.com/DIYgod/RSSHub/blob/master/lib/routes/github/issue.js) - -| Name | Description | -| ---------------------------------- | ---------------------------------------------------------------------------- | -| Route | `/github/issue/:user/:repo` | -| Data Source | github | -| Route Name | issue | -| Parameter 1 | :user, required | -| Parameter 2 | :repo, required | -| Parameter 3 | n/a | -| Route Path | `./routes/github/issue` | -| the complete code in lib/router.js | `router.get('/github/issue/:user/:repo', require('./routes/github/issue'));` | - -3. [embassy](https://github.com/DIYgod/RSSHub/blob/master/lib/routes/embassy/index.js) - -| Name | Description | -| ---------------------------------- | ---------------------------------------------------------------------------- | -| Route | `/embassy/:country/:city?` | -| Data Source | embassy | -| Route Name | n/a | -| Parameter 1 | :country, required | -| Parameter 2 | ?city, optional | -| Parameter 3 | n/a | -| Route Path | `./routes/embassy/index` | -| the complete code in lib/router.js | `router.get('/embassy/:country/:city?', require('./routes/embassy/index'));` | - ---- - -### Step 3: Add the documentation +### Add the documentation 1. Update [Documentation (/docs/en/README.md) ](https://github.com/DIYgod/RSSHub/blob/master/docs/en/README.md), preview the docs via `npm run docs:dev` @@ -455,88 +417,64 @@ Add the script into [/lib/router.js](https://github.com/DIYgod/RSSHub/blob/maste 1. it's redundant to indicate `optional/required` as the component will prepend based on `?` - Documentation examples: - - Multiple parameters: + 1. No parameter: ```vue - + ``` - + Preview: - - Use component slot for complicated description: + *** + + + + *** + + 2. Multiple parameters: ```vue - - This route returns a list of flight deals (in most cases, 6 flight deals) for a period defined by Hopper's algorithm, which means the travel date will be totally random (could be tomorrow or 10 months from now). For - airport IATA code please refer to [Wikipedia List of airports by IATA code](https://en.wikipedia.org/wiki/List_of_airports_by_IATA_code:_A) + + ``` + + Preview: + + *** + + + + *** + + 3. Use component slot for complicated description: + + ```vue + + + | 前端 | Android | iOS | 后端 | 设计 | 产品 | 工具资源 | 阅读 | 人工智能 | + | -------- | ------- | --- | ------- | ------ | ------- | -------- | ------- | -------- | + | frontend | android | ios | backend | design | product | freebie | article | ai | + ``` - + Preview: - This route returns a list of flight deals (in most cases, 6 flight deals) for a period defined by Hopper's algorithm, which means the travel date will be totally random (could be tomorrow or 10 months from now). + *** - For airport IATA code please refer to [Wikipedia List of airports by IATA code](https://en.wikipedia.org/wiki/List_of_airports_by_IATA_code:_A) + + + | 前端 | Android | iOS | 后端 | 设计 | 产品 | 工具资源 | 阅读 | 人工智能 | + | -------- | ------- | --- | ------- | ------ | ------- | -------- | ------- | -------- | + | frontend | android | ios | backend | design | product | freebie | article | ai | + *** + +1. Please be sure to close the tag of ``! + 1. Execute `npm run format` to lint the code before you commit and open a pull request ---- +## ## Submit new RSSHub Radar rule -## Join the discussion - -1. [Telegram Group](https://t.me/rsshub) -2. [GitHub Issues](https://github.com/DIYgod/RSSHub/issues) - -## Some Tips for Development - -### VS Code debug configuration - -`.vscode/launch.js` - -#### Debugging with nodemon - -In terminal, run `npm run dev` or `yarn dev` to start debugging. - -```json -{ - "version": "0.2.0", - "configurations": [ - { - "type": "node", - "request": "attach", - "name": "Node: Nodemon", - "processId": "${command:PickProcess}", - "restart": true, - "protocol": "inspector" - } - ] -} -``` - -#### Debugging without nodemon - -```json -{ - "version": "0.2.0", - "configurations": [ - { - "type": "node", - "request": "launch", - "name": "Launch Program", - "program": "${workspaceFolder}/lib/index.js", - "env": { "NODE_ENV": "dev" } - } - ] -} -``` +TODO diff --git a/docs/install/README.md b/docs/install/README.md index 8f154ebbe..095dc51ca 100644 --- a/docs/install/README.md +++ b/docs/install/README.md @@ -354,7 +354,7 @@ RSSHub 支持 `memory` 和 `redis` 两种缓存方式 - `TWITTER_CONSUMER_SECRET`: Twitter Consumer Secret，支持多个 key，用英文逗号 `,` 隔开，顺序与 key 对应 - - `TWITTER_TOKEN_{id}`: 对应 id 的 Twitter token，`{id}` 替换为 id，值为 consumer_key consumer_secret access_token access_token_secret 用逗号隔开，即：`{consumer_key},{consumer_secret},{access_token},{access_token_secret}` + - `TWITTER_TOKEN_{id}`: 对应 id 的 Twitter token，`{id}` 替换为 id，值为 `consumer_key consumer_secret access_token access_token_secret` 用逗号隔开，即：`{consumer_key},{consumer_secret},{access_token},{access_token_secret}` - youtube 全部路由: [申请地址](https://console.developers.google.com/) diff --git a/docs/joinus/README.md b/docs/joinus/README.md index 8e62ac4e9..21af8d62f 100644 --- a/docs/joinus/README.md +++ b/docs/joinus/README.md @@ -23,49 +23,6 @@ sidebar: auto 在 [/lib/router.js](https://github.com/DIYgod/RSSHub/blob/master/lib/router.js) 里添加路由 -#### 举例 - -1. [bilibili/bangumi](https://github.com/DIYgod/RSSHub/blob/master/lib/routes/bilibili/bangumi.js) - -| 名称 | 说明 | -| -------------------------- | ---------------------------------------------------------------------------------- | -| 路由 | `/bilibili/bangumi/:seasonid` | -| 数据来源 | bilibili | -| 路由名称 | bangumi | -| 参数 1 | :seasonid 必选 | -| 参数 2 | 无 | -| 参数 3 | 无 | -| 脚本路径 | `./routes/bilibili/bangumi` | -| lib/router.js 中的完整代码 | `router.get('/bilibili/bangumi/:seasonid', require('./routes/bilibili/bangumi'));` | - -2. [github/issue](https://github.com/DIYgod/RSSHub/blob/master/lib/routes/github/issue.js) - -| 名称 | 说明 | -| -------------------------- | ---------------------------------------------------------------------------- | -| 路由 | `/github/issue/:user/:repo` | -| 数据来源 | github | -| 路由名称 | issue | -| 参数 1 | :user 必选 | -| 参数 2 | :repo 必选 | -| 参数 3 | 无 | -| 脚本路径 | `./routes/github/issue` | -| lib/router.js 中的完整代码 | `router.get('/github/issue/:user/:repo', require('./routes/github/issue'));` | - -3. [embassy](https://github.com/DIYgod/RSSHub/blob/master/lib/routes/embassy/index.js) - -| 名称 | 说明 | -| -------------------------- | ---------------------------------------------------------------------------- | -| 路由 | `/embassy/:country/:city?` | -| 数据来源 | embassy | -| 路由名称 | 无 | -| 参数 1 | :country 必选 | -| 参数 2 | ?city 可选 | -| 参数 3 | 无 | -| 脚本路径 | `./routes/embassy/index` | -| lib/router.js 中的完整代码 | `router.get('/embassy/:country/:city?', require('./routes/embassy/index'));` | - ---- - ### 编写脚本 在 [/lib/routes/](https://github.com/DIYgod/RSSHub/tree/master/lib/routes) 中的路由对应路径下创建新的 js 脚本： @@ -165,7 +122,7 @@ sidebar: auto ```js const $ = cheerio.load(data); // 使用 cheerio 加载返回的 HTML const list = $('div[data-item_id]'); - // 使用 cheerio 选择器，选择 class="list-item" 的所有元素，返回 cheerio node 对象数组 + // 使用 cheerio 选择器，选择带有 data-item_id 属性的所有 div 元素，返回 cheerio node 对象数组 // 注：每一个 cheerio node 对应一个 HTML DOM // 注：cheerio 选择器与 jquery 选择器几乎相同 @@ -288,11 +245,11 @@ sidebar: auto const buildData = require('@/utils/common-config'); module.exports = async (ctx) => { ctx.state.data = await buildData({ - link: RSS来源链接, - url: 数据来源链接, - title: '%title%', //这里使用了变量，形如 **%xxx%** 这样的会被解析为变量，值为 **params** 下的同名值 + link: '', // RSS来源链接 + url: '', // 数据来源链接 + title: '%title%', // 这里使用了变量，形如 **%xxx%** 这样的会被解析为变量，值为 **params** 下的同名值 params: { - title: RSS标题, + title: '', // RSS标题 }, }); }; @@ -315,8 +272,8 @@ sidebar: auto }, item: { item: '.content-main .stream .stream-item', - title: `$('.post-account-group').text() + ' - %title%'`, //只支持$().xxx()这样的js语句，也足够使用 - link: `$('.post-account-group').attr('href')`, //.text()代表获取元素的文本，attr()表示获取指定属性 + title: `$('.post-account-group').text() + ' - %title%'`, // 只支持$().xxx()这样的js语句，也足够使用 + link: `$('.post-account-group').attr('href')`, // .text()代表获取元素的文本，.attr()表示获取指定属性 description: `$('.post .context').html()`, // .html()代表获取元素的html代码 pubDate: `new Date($('.post-time').attr('datetime')).toUTCString()`, // 日期的格式多种多样，可以尝试使用**/utils/date** guid: `new Date($('.post-time').attr('datetime')).getTime()`, // guid必须唯一，这是RSS的不同item的标志 diff --git a/docs/support/README.md b/docs/support/README.md index 2f82b26f8..f0543cbe3 100644 --- a/docs/support/README.md +++ b/docs/support/README.md @@ -21,6 +21,7 @@ RSSHub 是采用 MIT 许可的开源项目, 使用完全免费. 但是随着项 - [微信支付](https://i.loli.net/2019/03/23/5c950ebbc373e.png) - [支付宝](https://i.loli.net/2019/03/23/5c950ebbc980e.png) +- [Paypal](https://www.paypal.me/DIYgod) ## 周边产品 diff --git a/docs/university.md b/docs/university.md index bf1dbe8d2..9f10d9a3d 100644 --- a/docs/university.md +++ b/docs/university.md @@ -934,6 +934,19 @@ https://rsshub.app/**nuist**/`bulletin` 或 https://rsshub.app/**nuist**/`bullet +## 武汉大学 + +### 计算机学院公告 + + + +| 公告类型 | 新闻动态 | 学术讲座 | 学院通知 | 公示公告 | +| -------- | -------- | -------- | -------- | -------- | +| 参数 | 0 | 1 | 2 | 3 | + + + ## 西安电子科技大学 ### 教务处 @@ -1170,16 +1183,3 @@ https://rsshub.app/**nuist**/`bulletin` 或 https://rsshub.app/**nuist**/`bullet ### 数据科学与计算机学院动态 - -## 武汉大学 - -### 计算机学院公告 - - - -| 公告类型 | 新闻动态 | 学术讲座 | 学院通知 | 公示公告 | -| -------- | -------- | -------- | -------- | -------- | -| 参数 | 0 | 1 | 2 | 3 | - -