提案: 添加一个`tryGet`方法到cache工具中。 (#1130)

在抓取一些博客或者论坛的时候,有时候会系统抓取内容页面作为desc。
大多数的做法是获取子页面的地址,然后判断是否已经缓存结果,如果没有则获取,然后设置到缓存中。
这做法固然没有问题,这里只是提供一种新的思路,以将子页面的抓取逻辑独立出去便于逻辑实现。

使用方法:
```
const loadContent = async (link) => {
    // 执行页面加载,解析内容等操作。
    // 最终返回一个结果,该结果将被缓存。
    return result;
}
const result = await ctx.cache.tryGet(link, async () => loadContent(link), 3 * 60 * 60);
```
This commit is contained in:
cnzgray 2018-11-13 10:57:13 +08:00 committed by DIYgod
parent 96eac564cb
commit d1c50b0c7e
3 changed files with 45 additions and 13 deletions

View File

@ -36,6 +36,27 @@ module.exports = function(options = {}) {
memoryCache.set(key, value, maxAge * 1000);
}
},
/**
*
* try get from cache.
* if not exists use `getValue` function to get value, and put into cahche.
*
* @param key cache key
* @param getValueFunc a function to get value. call it when key not exists.
* @param maxAge
*
* @returns {Promise<void>}
*/
tryGet: async function(key, getValueFunc, maxAge) {
let v = await this.get(key);
if (!v) {
v = await getValueFunc();
this.set(key, v, maxAge);
}
return v;
},
};
return async function cache(ctx, next) {

View File

@ -57,6 +57,27 @@ module.exports = function(options = {}) {
await redisClient.setex(key, maxAge, value);
}
},
/**
*
* try get from cache.
* if not exists use `getValue` function to get value, and put into cahche.
*
* @param key cache key
* @param getValueFunc a function to get value. call it when key not exists.
* @param maxAge
*
* @returns {Promise<void>}
*/
tryGet: async function(key, getValueFunc, maxAge) {
let v = await this.get(key);
if (!v) {
v = await getValueFunc();
this.set(key, v, maxAge);
}
return v;
},
};
return async function cache(ctx, next) {

View File

@ -14,13 +14,7 @@ module.exports = async (ctx) => {
const $ = cheerio.load(response.data);
const parseContent = async (link) => {
// Check cache
const cache = await ctx.cache.get(link);
if (cache) {
return Promise.resolve(JSON.parse(cache));
}
const loadContent = async (link) => {
const response = await axios({
method: 'get',
url: link,
@ -36,17 +30,13 @@ module.exports = async (ctx) => {
// const author = $('.post-meta span').text();
const html = $('#pagecontainer').html();
const result = {
return {
title: title,
link: link,
guid: link,
pubDate: pubDate,
description: html,
};
ctx.cache.set(link, JSON.stringify(result), 3 * 60 * 60);
return result;
};
const items = await Promise.all(
@ -55,7 +45,7 @@ module.exports = async (ctx) => {
.map(async (item) => {
const node = $('a', item);
const link = node.attr('href');
const result = await parseContent(link);
const result = await ctx.cache.tryGet(link, async () => loadContent(link), 3 * 60 * 60);
return Promise.resolve(result);
})