diff options
Diffstat (limited to 'src/utils/request.rs')
| -rw-r--r-- | src/utils/request.rs | 38 |
1 files changed, 23 insertions, 15 deletions
diff --git a/src/utils/request.rs b/src/utils/request.rs index 4516806..8cee0a0 100644 --- a/src/utils/request.rs +++ b/src/utils/request.rs @@ -8,6 +8,7 @@ use reqwest::Url; use scraper::{Html, Selector}; use serde::Deserialize; use serde_json::Value; +use std::sync::LazyLock; use std::{ collections::{HashMap, HashSet}, sync::Arc, @@ -26,28 +27,35 @@ const MAX_CRAWLS: usize = 5; const BREAK_ON_ERROR: bool = false; const USER_AGENT: &str = "curl/8.6.0"; -lazy_static::lazy_static! { - static ref CLIENT: Result<reqwest::Client> = { - let builder = reqwest::ClientBuilder::new().timeout(Duration::from_secs(16)); - let client = builder.build()?; - Ok(client) - }; +static CLIENT: LazyLock<Result<reqwest::Client>> = LazyLock::new(|| { + let builder = reqwest::ClientBuilder::new().timeout(Duration::from_secs(16)); + let client = builder.build()?; + Ok(client) +}); - static ref PRESET: Vec<(Regex, CrawlOptions)> = vec![ +static PRESET: LazyLock<Vec<(Regex, CrawlOptions)>> = LazyLock::new(|| { + vec![ ( Regex::new(r"github.com/([^/]+)/([^/]+)/tree/([^/]+)").unwrap(), - CrawlOptions { exclude: vec!["changelog".into(), "changes".into(), "license".into()], ..Default::default() } + CrawlOptions { + exclude: vec!["changelog".into(), "changes".into(), "license".into()], + ..Default::default() + }, ), ( Regex::new(r"github.com/([^/]+)/([^/]+)/wiki").unwrap(), - CrawlOptions { exclude: vec!["_history".into()], extract: Some("#wiki-body".into()), ..Default::default() } - - ) - ]; + CrawlOptions { + exclude: vec!["_history".into()], + extract: Some("#wiki-body".into()), + ..Default::default() + }, + ), + ] +}); - static ref EXTENSION_RE: Regex = Regex::new(r"\.[^.]+$").unwrap(); - static ref GITHUB_REPO_RE: Regex = Regex::new(r"^https://github\.com/([^/]+)/([^/]+)/tree/([^/]+)").unwrap(); -} +static EXTENSION_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\.[^.]+$").unwrap()); +static GITHUB_REPO_RE: LazyLock<Regex> = + LazyLock::new(|| Regex::new(r"^https://github\.com/([^/]+)/([^/]+)/tree/([^/]+)").unwrap()); pub async fn fetch(url: &str) -> Result<String> { let client = match *CLIENT { |
