From 555f4f5d128d018912144b0e7f2b3a50223bf713 Mon Sep 17 00:00:00 2001
From: sigoden <sigoden@gmail.com>
Date: Fri, 6 Sep 2024 08:08:55 +0800
Subject: feat: better html to markdown converter (#840)

---
 Cargo.lock              | 50 +++++++++++++++++++++++++++++++++++--------------
 Cargo.toml              |  2 +-
 src/utils/html_to_md.rs | 18 ++++++++++++++++++
 src/utils/mod.rs        |  2 ++
 src/utils/request.rs    | 16 ++++++----------
 5 files changed, 63 insertions(+), 25 deletions(-)
 create mode 100644 src/utils/html_to_md.rs

diff --git a/Cargo.lock b/Cargo.lock
index 4702621..4ec3312 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -67,7 +67,7 @@ dependencies = [
  "futures-util",
  "hmac",
  "hnsw_rs",
- "html2text",
+ "html_to_markdown",
  "http",
  "http-body-util",
  "hyper",
@@ -1152,19 +1152,6 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
-[[package]]
-name = "html2text"
-version = "0.12.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c66ee488a63a92237d5b48875b7e05bb293be8fb2894641c8118b60c08ab5ef"
-dependencies = [
- "html5ever",
- "markup5ever",
- "tendril",
- "thiserror",
- "unicode-width",
-]
-
 [[package]]
 name = "html5ever"
 version = "0.27.0"
@@ -1179,6 +1166,18 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "html_to_markdown"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e608e8dd0939bfb6b516d96a5919751b835297a02230aecb88d2fc84ebebaa8a"
+dependencies = [
+ "anyhow",
+ "html5ever",
+ "markup5ever_rcdom",
+ "regex",
+]
+
 [[package]]
 name = "http"
 version = "1.1.0"
@@ -1521,6 +1520,18 @@ dependencies = [
  "tendril",
 ]
 
+[[package]]
+name = "markup5ever_rcdom"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "edaa21ab3701bfee5099ade5f7e1f84553fd19228cf332f13cd6e964bf59be18"
+dependencies = [
+ "html5ever",
+ "markup5ever",
+ "tendril",
+ "xml5ever",
+]
+
 [[package]]
 name = "matchers"
 version = "0.1.0"
@@ -3913,6 +3924,17 @@ version = "0.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ec107c4503ea0b4a98ef47356329af139c0a4f7750e621cf2973cd3385ebcb3d"
 
+[[package]]
+name = "xml5ever"
+version = "0.18.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9bbb26405d8e919bc1547a5aa9abc95cbfa438f04844f5fdd9dc7596b748bf69"
+dependencies = [
+ "log",
+ "mac",
+ "markup5ever",
+]
+
 [[package]]
 name = "yansi"
 version = "0.5.1"
diff --git a/Cargo.toml b/Cargo.toml
index d97e537..9ee6cb9 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -59,9 +59,9 @@ path-absolutize = "3.1.1"
 hnsw_rs = "0.3.0"
 rayon = "1.10.0"
 uuid = { version = "1.9.1", features = ["v4"] }
-html2text = "0.12.5"
 scraper = { version = "0.20.0", default-features = false, features = ["deterministic"] }
 sys-locale = "0.3.1"
+html_to_markdown = "0.1.0"
 
 [dependencies.reqwest]
 version = "0.12.0"
diff --git a/src/utils/html_to_md.rs b/src/utils/html_to_md.rs
new file mode 100644
index 0000000..2bc5e54
--- /dev/null
+++ b/src/utils/html_to_md.rs
@@ -0,0 +1,18 @@
+use std::{cell::RefCell, rc::Rc};
+
+use html_to_markdown::{markdown, TagHandler};
+
+pub fn html_to_md(html: &str) -> String {
+    let mut handlers: Vec<TagHandler> = vec![
+        Rc::new(RefCell::new(markdown::ParagraphHandler)),
+        Rc::new(RefCell::new(markdown::HeadingHandler)),
+        Rc::new(RefCell::new(markdown::ListHandler)),
+        Rc::new(RefCell::new(markdown::TableHandler::new())),
+        Rc::new(RefCell::new(markdown::StyledTextHandler)),
+        Rc::new(RefCell::new(markdown::CodeHandler)),
+        Rc::new(RefCell::new(markdown::WebpageChromeRemover)),
+    ];
+
+    html_to_markdown::convert_html_to_markdown(html.as_bytes(), &mut handlers)
+        .unwrap_or_else(|_| html.to_string())
+}
diff --git a/src/utils/mod.rs b/src/utils/mod.rs
index d9d6617..1090c4e 100644
--- a/src/utils/mod.rs
+++ b/src/utils/mod.rs
@@ -2,6 +2,7 @@ mod abort_signal;
 mod clipboard;
 mod command;
 mod crypto;
+mod html_to_md;
 mod path;
 mod prompt_input;
 mod render_prompt;
@@ -12,6 +13,7 @@ pub use self::abort_signal::*;
 pub use self::clipboard::set_text;
 pub use self::command::*;
 pub use self::crypto::*;
+pub use self::html_to_md::*;
 pub use self::path::*;
 pub use self::prompt_input::*;
 pub use self::render_prompt::render_prompt;
diff --git a/src/utils/request.rs b/src/utils/request.rs
index 73919d8..9f2804b 100644
--- a/src/utils/request.rs
+++ b/src/utils/request.rs
@@ -8,8 +8,11 @@ use reqwest::Url;
 use scraper::{Html, Selector};
 use serde::Deserialize;
 use serde_json::Value;
-use std::{collections::HashMap, time::Duration};
-use std::{collections::HashSet, sync::Arc};
+use std::{
+    collections::{HashMap, HashSet},
+    sync::Arc,
+    time::Duration,
+};
 use tokio::io::AsyncWriteExt;
 use tokio::sync::Semaphore;
 
@@ -136,10 +139,7 @@ pub async fn fetch(
             None => {
                 let contents = res.text().await?;
                 if extension == "html" {
-                    (
-                        html2text::from_read(contents.as_bytes(), usize::MAX),
-                        "md".into(),
-                    )
+                    (html_to_md(&contents), "md".into())
                 } else {
                     (contents, extension)
                 }
@@ -387,10 +387,6 @@ async fn crawl_page(
     Ok((path.to_string(), text, links.into_iter().collect()))
 }
 
-fn html_to_md(html: &str) -> String {
-    html2text::from_read(html.as_bytes(), usize::MAX)
-}
-
 fn should_exclude_link(link: &str, exclude: &[String]) -> bool {
     if link.contains("#") {
         return true;
-- 
cgit v1.2.3