feat: better html to markdown converter (#840)

3 files changed, 26 insertions, 10 deletions

diff --git a/src/utils/html_to_md.rs b/src/utils/html_to_md.rs
new file mode 100644
index 0000000..2bc5e54
--- /dev/null
+++ b/src/utils/html_to_md.rs
@@ -0,0 +1,18 @@
+use std::{cell::RefCell, rc::Rc};
+
+use html_to_markdown::{markdown, TagHandler};
+
+pub fn html_to_md(html: &str) -> String {
+    let mut handlers: Vec<TagHandler> = vec![
+        Rc::new(RefCell::new(markdown::ParagraphHandler)),
+        Rc::new(RefCell::new(markdown::HeadingHandler)),
+        Rc::new(RefCell::new(markdown::ListHandler)),
+        Rc::new(RefCell::new(markdown::TableHandler::new())),
+        Rc::new(RefCell::new(markdown::StyledTextHandler)),
+        Rc::new(RefCell::new(markdown::CodeHandler)),
+        Rc::new(RefCell::new(markdown::WebpageChromeRemover)),
+    ];
+
+    html_to_markdown::convert_html_to_markdown(html.as_bytes(), &mut handlers)
+        .unwrap_or_else(|_| html.to_string())
+}
diff --git a/src/utils/mod.rs b/src/utils/mod.rs
index d9d6617..1090c4e 100644
--- a/src/utils/mod.rs
+++ b/src/utils/mod.rs
@@ -2,6 +2,7 @@ mod abort_signal;
 mod clipboard;
 mod command;
 mod crypto;
+mod html_to_md;
 mod path;
 mod prompt_input;
 mod render_prompt;
@@ -12,6 +13,7 @@ pub use self::abort_signal::*;
 pub use self::clipboard::set_text;
 pub use self::command::*;
 pub use self::crypto::*;
+pub use self::html_to_md::*;
 pub use self::path::*;
 pub use self::prompt_input::*;
 pub use self::render_prompt::render_prompt;
diff --git a/src/utils/request.rs b/src/utils/request.rs
index 73919d8..9f2804b 100644
--- a/src/utils/request.rs
+++ b/src/utils/request.rs
@@ -8,8 +8,11 @@ use reqwest::Url;
 use scraper::{Html, Selector};
 use serde::Deserialize;
 use serde_json::Value;
-use std::{collections::HashMap, time::Duration};
-use std::{collections::HashSet, sync::Arc};
+use std::{
+    collections::{HashMap, HashSet},
+    sync::Arc,
+    time::Duration,
+};
 use tokio::io::AsyncWriteExt;
 use tokio::sync::Semaphore;
 
@@ -136,10 +139,7 @@ pub async fn fetch(
             None => {
                 let contents = res.text().await?;
                 if extension == "html" {
-                    (
-                        html2text::from_read(contents.as_bytes(), usize::MAX),
-                        "md".into(),
-                    )
+                    (html_to_md(&contents), "md".into())
                 } else {
                     (contents, extension)
                 }
@@ -387,10 +387,6 @@ async fn crawl_page(
     Ok((path.to_string(), text, links.into_iter().collect()))
 }
 
-fn html_to_md(html: &str) -> String {
-    html2text::from_read(html.as_bytes(), usize::MAX)
-}
-
 fn should_exclude_link(link: &str, exclude: &[String]) -> bool {
     if link.contains("#") {
         return true;