提案: 添加一个`tryGet`方法到cache工具中。 (#1130)
在抓取一些博客或者论坛的时候,有时候会系统抓取内容页面作为desc。
大多数的做法是获取子页面的地址,然后判断是否已经缓存结果,如果没有则获取,然后设置到缓存中。
这做法固然没有问题,这里只是提供一种新的思路,以将子页面的抓取逻辑独立出去便于逻辑实现。
使用方法:
```
const loadContent = async (link) => {
// 执行页面加载,解析内容等操作。
// 最终返回一个结果,该结果将被缓存。
return result;
}
const result = await ctx.cache.tryGet(link, async () => loadContent(link), 3 * 60 * 60);
```
This commit is contained in:
parent
96eac564cb
commit
d1c50b0c7e
|
|
@ -36,6 +36,27 @@ module.exports = function(options = {}) {
|
|||
memoryCache.set(key, value, maxAge * 1000);
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
*
|
||||
* try get from cache.
|
||||
* if not exists use `getValue` function to get value, and put into cahche.
|
||||
*
|
||||
* @param key cache key
|
||||
* @param getValueFunc a function to get value. call it when key not exists.
|
||||
* @param maxAge
|
||||
*
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
tryGet: async function(key, getValueFunc, maxAge) {
|
||||
let v = await this.get(key);
|
||||
if (!v) {
|
||||
v = await getValueFunc();
|
||||
this.set(key, v, maxAge);
|
||||
}
|
||||
|
||||
return v;
|
||||
},
|
||||
};
|
||||
|
||||
return async function cache(ctx, next) {
|
||||
|
|
|
|||
|
|
@ -57,6 +57,27 @@ module.exports = function(options = {}) {
|
|||
await redisClient.setex(key, maxAge, value);
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
*
|
||||
* try get from cache.
|
||||
* if not exists use `getValue` function to get value, and put into cahche.
|
||||
*
|
||||
* @param key cache key
|
||||
* @param getValueFunc a function to get value. call it when key not exists.
|
||||
* @param maxAge
|
||||
*
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
tryGet: async function(key, getValueFunc, maxAge) {
|
||||
let v = await this.get(key);
|
||||
if (!v) {
|
||||
v = await getValueFunc();
|
||||
this.set(key, v, maxAge);
|
||||
}
|
||||
|
||||
return v;
|
||||
},
|
||||
};
|
||||
|
||||
return async function cache(ctx, next) {
|
||||
|
|
|
|||
|
|
@ -14,13 +14,7 @@ module.exports = async (ctx) => {
|
|||
|
||||
const $ = cheerio.load(response.data);
|
||||
|
||||
const parseContent = async (link) => {
|
||||
// Check cache
|
||||
const cache = await ctx.cache.get(link);
|
||||
if (cache) {
|
||||
return Promise.resolve(JSON.parse(cache));
|
||||
}
|
||||
|
||||
const loadContent = async (link) => {
|
||||
const response = await axios({
|
||||
method: 'get',
|
||||
url: link,
|
||||
|
|
@ -36,17 +30,13 @@ module.exports = async (ctx) => {
|
|||
// const author = $('.post-meta span').text();
|
||||
const html = $('#pagecontainer').html();
|
||||
|
||||
const result = {
|
||||
return {
|
||||
title: title,
|
||||
link: link,
|
||||
guid: link,
|
||||
pubDate: pubDate,
|
||||
description: html,
|
||||
};
|
||||
|
||||
ctx.cache.set(link, JSON.stringify(result), 3 * 60 * 60);
|
||||
|
||||
return result;
|
||||
};
|
||||
|
||||
const items = await Promise.all(
|
||||
|
|
@ -55,7 +45,7 @@ module.exports = async (ctx) => {
|
|||
.map(async (item) => {
|
||||
const node = $('a', item);
|
||||
const link = node.attr('href');
|
||||
const result = await parseContent(link);
|
||||
const result = await ctx.cache.tryGet(link, async () => loadContent(link), 3 * 60 * 60);
|
||||
|
||||
return Promise.resolve(result);
|
||||
})
|
||||
|
|
|
|||
Loading…
Reference in New Issue