xiaoxue_admin
/
RSSHub
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: support setting expire time for content cache

This commit is contained in:
DIYgod 2019-05-09 01:11:19 +08:00
parent f08f3823e4
commit 075a8d1864
No known key found for this signature in database
GPG Key ID: EC0B76A252D3EF67
10 changed files with 28 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
docs/en/joinus/README.md
View File

@ -168,7 +168,7 @@ Firstly, add a .js file for the new route in [/lib/router.js](https://github.com
// use tryGet() to query the cache
// if the query returns no result, query the data source via load() to get article content
const other = await caches.tryGet(itemUrl, async () => await load(itemUrl), 3 * 60 * 60);
const other = await caches.tryGet(itemUrl, async () => await load(itemUrl));
// merge two objects to form the final output
return Promise.resolve(Object.assign({}, single, other));
@ -270,7 +270,7 @@ By default there is a global caching period set in `lib/config.js`, some sources
- Save to cache:
```js
ctx.cache.set((key: string), (value: string), (time: number)); // time is the caching period in seconds.
ctx.cache.set((key: string), (value: string)); // time is the caching period in seconds.
```
- Access the cache:
@ -285,7 +285,7 @@ Given the update frequency is known, set the appropriate caching period to reuse
```js
const key = 'daily' + story.id; // story.id is the unique identifier of each article
ctx.cache.set(key, item.description, 24 * 60 * 60); // set the caching period to 24 hours * 60 minutes * 60 seconds = 86,400 seconds = 1 day
ctx.cache.set(key, item.description); // set cache
```
When the identical requests come in, reuse the cache：

4
docs/install/README.md
View File

@ -311,7 +311,9 @@ $ docker run -d --name rsshub -p 1200:1200 rsshub:arm32v7
`CACHE_TYPE`: 缓存类型, 可为 `memory` 和 `redis`, 设为空可以禁止缓存, 默认为 `memory`
`CACHE_EXPIRE`: 缓存过期时间, 单位为秒, 默认 `300`
`CACHE_EXPIRE`: 路由缓存过期时间, 单位为秒, 默认 `5 * 60`
`CACHE_CONTENT_EXPIRE`: 内容缓存过期时间，单位为秒, 默认 `24 * 60 * 60`
`LISTEN_INADDR_ANY`: 是否允许公网连接, 默认 `1`

6
docs/joinus/README.md
View File

@ -170,7 +170,7 @@ sidebar: auto
// 使用 tryGet() 方法从缓存获取内容
// 当缓存中无法获取到链接内容的时候，则使用 load() 方法加载文章内容
const other = await caches.tryGet(itemUrl, async () => await load(itemUrl), 3 * 60 * 60);
const other = await caches.tryGet(itemUrl, async () => await load(itemUrl));
// 合并解析后的结果集作为该篇文章最终的输出结果
return Promise.resolve(Object.assign({}, single, other));
@ -327,7 +327,7 @@ sidebar: auto
- 添加缓存:
```js
ctx.cache.set((key: string), (value: string), (time: number)); // time 为缓存时间。单位为秒。
ctx.cache.set((key: string), (value: string)); // time 为缓存时间。单位为秒。
```
- 获取缓存:
@ -342,7 +342,7 @@ const value = await ctx.cache.get((key: string));
```js
const key = 'daily' + story.id; // story.id 为知乎日报返回的文章唯一识别符
ctx.cache.set(key, item.description, 24 * 60 * 60); // 设置缓存时间为 24小时 * 60分钟 * 60秒 = 86400秒 = 1天
ctx.cache.set(key, item.description); // 设置缓存
```
当同样的请求被发起时，优先使用未过期的缓存：

7
lib/config.js
View File

@ -3,8 +3,11 @@ module.exports = {
port: process.env.PORT || 1200, // 监听端口
socket: process.env.SOCKET || null, // 监听 Unix Socket, null 为禁用
},
cacheType: process.env.CACHE_TYPE || 'memory', // 缓存类型，支持 'memory' 和 'redis'，设为空可以禁止缓存
cacheExpire: parseInt(process.env.CACHE_EXPIRE) || 5 * 60, // 缓存时间，单位为秒
cache: {
type: process.env.CACHE_TYPE || 'memory', // 缓存类型，支持 'memory' 和 'redis'，设为空可以禁止缓存
routeExpire: parseInt(process.env.CACHE_EXPIRE) || 5 * 60, // 路由缓存时间，单位为秒
contentExpire: parseInt(process.env.CACHE_CONTENT_EXPIRE) || 24 * 60 * 60, // 不变内容缓存时间，单位为秒
},
ua: process.env.UA || 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36',
listenInaddrAny: parseInt(process.env.LISTEN_INADDR_ANY) || 1, // 是否允许公网连接，取值 0 1
requestRetry: parseInt(process.env.REQUEST_RETRY) || 2, // 请求失败重试次数

18
lib/middleware/cache.js
View File

@ -9,13 +9,13 @@ const pathToRegExp = require('path-to-regexp');
module.exports = function(app, options = {}) {
let available = false;
const { prefix = 'koa-redis-cache:', expire = config.cacheExpire, routes = ['(.*)'], exclude = ['/'], passParam = '', maxLength = Infinity, ignoreQuery = true } = options;
const { prefix = 'koa-redis-cache:', expire = config.cache.routeExpire, routes = ['(.*)'], exclude = ['/'], passParam = '', maxLength = Infinity, ignoreQuery = true } = options;
const globalCache = {
get: null,
set: null,
};
if (config.cacheType === 'redis') {
if (config.cache.type === 'redis') {
const { host: redisHost = 'localhost', port: redisPort = 6379, url: redisUrl = `redis://${redisHost}:${redisPort}/`, options: redisOptions = {} } = config.redis || {};
if (!redisOptions.password) {
delete redisOptions.password;
@ -38,13 +38,13 @@ module.exports = function(app, options = {}) {
if (key) {
let value = await redisClient.get(key);
if (value) {
await redisClient.expire(key, 24 * 60 * 60);
await redisClient.expire(key, config.cache.routeExpire);
value = value + '';
}
return value;
}
},
set: async function(key, value, maxAge = 24 * 60 * 60) {
set: async function(key, value, maxAge = config.cache.routeExpire) {
if (await redisClient.exists(key)) {
logger.warn(`repeated key: ${key}, ${value}`);
return;
@ -67,7 +67,7 @@ module.exports = function(app, options = {}) {
}
};
globalCache.set = app.context.cache.set;
} else if (config.cacheType === 'memory') {
} else if (config.cache.type === 'memory') {
const pageCache = new Lru({
maxAge: expire * 1000,
max: maxLength,
@ -89,7 +89,7 @@ module.exports = function(app, options = {}) {
return value;
}
},
set: (key, value, maxAge = 24 * 60 * 60) => {
set: (key, value, maxAge = config.cache.routeExpire) => {
if (!value || value === 'undefined') {
value = '';
}
@ -125,7 +125,7 @@ module.exports = function(app, options = {}) {
};
}
app.context.cache.tryGet = async function(key, getValueFunc, maxAge = 24 * 60 * 60) {
app.context.cache.tryGet = async function(key, getValueFunc, maxAge = config.cache.routeExpire) {
let v = await this.get(key);
if (!v) {
v = await getValueFunc();
@ -157,12 +157,12 @@ module.exports = function(app, options = {}) {
if (Buffer.isBuffer(type)) {
type = type.toString();
}
if (config.cacheType === 'redis') {
if (config.cache.type === 'redis') {
ctx.response.set({
'X-Koa-Redis-Cache': 'true',
'Content-Type': type,
});
} else if (config.cacheType === 'memory') {
} else if (config.cache.type === 'memory') {
ctx.response.set({
'X-Koa-Memory-Cache': 'true',
'Content-Type': type,

2
lib/middleware/header.js
View File

@ -4,7 +4,7 @@ const config = require('../config');
const headers = {
'Access-Control-Allow-Methods': 'GET',
'Content-Type': 'application/xml; charset=utf-8',
'Cache-Control': `public, max-age=${config.cacheExpire}`,
'Cache-Control': `public, max-age=${config.cache.routeExpire}`,
};
module.exports = async (ctx, next) => {

2
lib/middleware/template.js
View File

@ -54,7 +54,7 @@ module.exports = async (ctx, next) => {
const data = {
lastBuildDate: new Date().toUTCString(),
updated: new Date().toISOString(),
ttl: config.cacheExpire,
ttl: config.cache.routeExpire,
...ctx.state.data,
};
if (template) {

2
lib/routes/test/index.js
View File

@ -15,7 +15,7 @@ module.exports = async (ctx) => {
author: `DIYgod0`,
});
} else if (ctx.params.id === 'cache') {
const description = await ctx.cache.tryGet('test', () => `Cache${++cacheIndex}`, config.cacheExpire * 2);
const description = await ctx.cache.tryGet('test', () => `Cache${++cacheIndex}`, config.cache.routeExpire * 2);
item.push({
title: 'Cache Title',
description: description,

2
test/middleware/header.js
View File

@ -13,7 +13,7 @@ describe('header', () => {
expect(response.headers['access-control-allow-origin']).toBe('127.0.0.1:1200');
expect(response.headers['access-control-allow-methods']).toBe('GET');
expect(response.headers['content-type']).toBe('application/xml; charset=utf-8');
expect(response.headers['cache-control']).toBe(`public, max-age=${config.cacheExpire}`);
expect(response.headers['cache-control']).toBe(`public, max-age=${config.cache.routeExpire}`);
expect(response.headers['last-modified']).toBe(response.text.match(/<lastBuildDate>(.*)<\/lastBuildDate>/)[1]);
// expect(response.headers.etag).toBe('"b37-MORyrF0tJ8BFw0xLLZL/zBYAFPY"');
});

2
test/router.js
View File

@ -21,7 +21,7 @@ async function checkRSS(response) {
expect(parsed.description).toEqual(expect.any(String));
expect(parsed.link).toEqual(expect.any(String));
expect(parsed.lastBuildDate).toEqual(expect.any(String));
expect(parsed.ttl).toEqual(config.cacheExpire + '');
expect(parsed.ttl).toEqual(config.cache.routeExpire + '');
expect(parsed.items).toEqual(expect.any(Array));
checkDate(parsed.lastBuildDate);