summaryrefslogtreecommitdiffstats
path: root/src/utils/request.rs
diff options
context:
space:
mode:
Diffstat (limited to 'src/utils/request.rs')
-rw-r--r--src/utils/request.rs38
1 files changed, 23 insertions, 15 deletions
diff --git a/src/utils/request.rs b/src/utils/request.rs
index 4516806..8cee0a0 100644
--- a/src/utils/request.rs
+++ b/src/utils/request.rs
@@ -8,6 +8,7 @@ use reqwest::Url;
use scraper::{Html, Selector};
use serde::Deserialize;
use serde_json::Value;
+use std::sync::LazyLock;
use std::{
collections::{HashMap, HashSet},
sync::Arc,
@@ -26,28 +27,35 @@ const MAX_CRAWLS: usize = 5;
const BREAK_ON_ERROR: bool = false;
const USER_AGENT: &str = "curl/8.6.0";
-lazy_static::lazy_static! {
- static ref CLIENT: Result<reqwest::Client> = {
- let builder = reqwest::ClientBuilder::new().timeout(Duration::from_secs(16));
- let client = builder.build()?;
- Ok(client)
- };
+static CLIENT: LazyLock<Result<reqwest::Client>> = LazyLock::new(|| {
+ let builder = reqwest::ClientBuilder::new().timeout(Duration::from_secs(16));
+ let client = builder.build()?;
+ Ok(client)
+});
- static ref PRESET: Vec<(Regex, CrawlOptions)> = vec![
+static PRESET: LazyLock<Vec<(Regex, CrawlOptions)>> = LazyLock::new(|| {
+ vec![
(
Regex::new(r"github.com/([^/]+)/([^/]+)/tree/([^/]+)").unwrap(),
- CrawlOptions { exclude: vec!["changelog".into(), "changes".into(), "license".into()], ..Default::default() }
+ CrawlOptions {
+ exclude: vec!["changelog".into(), "changes".into(), "license".into()],
+ ..Default::default()
+ },
),
(
Regex::new(r"github.com/([^/]+)/([^/]+)/wiki").unwrap(),
- CrawlOptions { exclude: vec!["_history".into()], extract: Some("#wiki-body".into()), ..Default::default() }
-
- )
- ];
+ CrawlOptions {
+ exclude: vec!["_history".into()],
+ extract: Some("#wiki-body".into()),
+ ..Default::default()
+ },
+ ),
+ ]
+});
- static ref EXTENSION_RE: Regex = Regex::new(r"\.[^.]+$").unwrap();
- static ref GITHUB_REPO_RE: Regex = Regex::new(r"^https://github\.com/([^/]+)/([^/]+)/tree/([^/]+)").unwrap();
-}
+static EXTENSION_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\.[^.]+$").unwrap());
+static GITHUB_REPO_RE: LazyLock<Regex> =
+ LazyLock::new(|| Regex::new(r"^https://github\.com/([^/]+)/([^/]+)/tree/([^/]+)").unwrap());
pub async fn fetch(url: &str) -> Result<String> {
let client = match *CLIENT {