From e4478a366ce179671c4f1bd2f0cff95985da44ea Mon Sep 17 00:00:00 2001 From: NeverBehave Date: Wed, 22 Jun 2022 12:25:30 -0700 Subject: [PATCH] feat(core): regex engine config (#10013) --- docs/en/install/README.md | 2 ++ docs/en/parameter.md | 12 +++++++++++- docs/install/README.md | 2 ++ docs/parameter.md | 8 ++++++++ lib/config.js | 1 + lib/middleware/parameter.js | 19 +++++++++++++++++-- 6 files changed, 41 insertions(+), 3 deletions(-) diff --git a/docs/en/install/README.md b/docs/en/install/README.md index 52618e994..9c8f5baab 100644 --- a/docs/en/install/README.md +++ b/docs/en/install/README.md @@ -557,6 +557,8 @@ Configs in this sections are in beta stage, and are turn off by default. Please `ALLOW_USER_HOTLINK_TEMPLATE`: [Parameters->Multimedia processing](/en/parameter.html#multimedia-processing) +`FILTER_REGEX_ENGINE`: Define Regex engine used in [Parameters->filtering](/en/parameter.html#filtering). Valid value are `[re, regexp]`. Default value is `re`. We suggest public instance should leave this value to default, and this option right now is mainly for backward compatibility. + ### Other Application Configurations `DISALLOW_ROBOT`: prevent indexing by search engine, default to enable, set false or 0 to disable diff --git a/docs/en/parameter.md b/docs/en/parameter.md index 612d682b1..420534c32 100644 --- a/docs/en/parameter.md +++ b/docs/en/parameter.md @@ -26,6 +26,16 @@ Please make sure you've [fully URL-encoded](https://gchq.github.io/CyberChef/#re ::: +::: warning Warning + +filter supports Regex, and due to the fact that some Regex are vulnerable to DoS (ReDoS), default engine `re` blocks some of these functionalities available in node `Regexp`. These two engines also behaves a bit different in some corner cases. [Details](https://github.com/uhop/node-re2#limitations-things-re2-does-not-support) + + +If you need to use a different engine, please refer to [Deploy->Features->FILTER_REGEX_ENGINE](/en/install/#configuration-features). + +::: + + The following URL query parameters are supported, Regex support is built-in. Set `filter` to include the content @@ -102,7 +112,7 @@ E.g. ## Multimedia processing -::: warning 注意 +::: warning Warning This is an experimental API diff --git a/docs/install/README.md b/docs/install/README.md index 400c5856a..15884e627 100644 --- a/docs/install/README.md +++ b/docs/install/README.md @@ -563,6 +563,8 @@ RSSHub 支持使用访问密钥 / 码，白名单和黑名单三种方式进行 `ALLOW_USER_HOTLINK_TEMPLATE`: [通用参数 -> 多媒体处理](/parameter.html#duo-mei-ti-chu-li)特性控制 +`FILTER_REGEX_ENGINE`: 控制 [通用参数 -> 内容过滤](/parameter.html#nei-rong-guo-lu) 使用的正则引擎。可选`[re, regexp]`，默认`re`。我们推荐公开实例不要调整这个选项，这个选项目前主要用于向后兼容。 + ### 其他应用配置 `DISALLOW_ROBOT`: 阻止搜索引擎收录，默认开启，设置 false 或 0 关闭 diff --git a/docs/parameter.md b/docs/parameter.md index 15c1d1e7a..5189837b2 100644 --- a/docs/parameter.md +++ b/docs/parameter.md @@ -26,6 +26,14 @@ ::: +::: warning 注意 + +filter 支持正则表达式。由于正则部分特性可被利用于 DoS (ReDOS)，默认引擎`RE`屏蔽了部分`Regexp`功能，且在部分情况下表现不一致。具体差异可以[查看文档](https://github.com/uhop/node-re2#limitations-things-re2-does-not-support) + +如果需要指定不同的引擎，请参考[功能特性 -> FILTER_REGEX_ENGINE](install/#pei-zhi-gong-neng-te-xing)。 + +::: + 可以使用以下 URL query 过滤内容，支持通过正则表达式过滤 `filter` 选出想要的内容 diff --git a/lib/config.js b/lib/config.js index f5f27a024..b3347fa4b 100644 --- a/lib/config.js +++ b/lib/config.js @@ -93,6 +93,7 @@ const calculateValue = () => { }, feature: { allow_user_hotlink_template: envs.ALLOW_USER_HOTLINK_TEMPLATE === 'true', + filter_regex_engine: envs.FILTER_REGEX_ENGINE || 're', }, suffix: envs.SUFFIX, titleLengthLimit: parseInt(envs.TITLE_LENGTH_LIMIT) || 150, diff --git a/lib/middleware/parameter.js b/lib/middleware/parameter.js index 5c8f91fab..a599cc2b8 100644 --- a/lib/middleware/parameter.js +++ b/lib/middleware/parameter.js @@ -152,10 +152,25 @@ module.exports = async (ctx, next) => { // filter const makeRegex = (string) => { // default: case_senstivie = true + const engine = config.feature.filter_regex_engine; if (ctx.query.filter_case_sensitive === 'false') { - return new RE2(string, 'i'); + switch (engine) { + case 'regexp': + return new RegExp(string, 'i'); + case 're': + return new RE2(string, 'i'); + default: + throw Error(`Invalid Engine Value: ${engine}, please check your config.`); + } } else { - return new RE2(string); + switch (engine) { + case 'regexp': + return new RegExp(string); + case 're': + return new RE2(string); + default: + throw Error(`Invalid Engine Value: ${engine}, please check your config.`); + } } };