From bfb7d75fe47ea588495c5be8157a67b1fec67a13 Mon Sep 17 00:00:00 2001 From: sigoden Date: Thu, 6 Jun 2024 08:29:00 +0800 Subject: refactor: move tools/$lang/** to tools/** (#17) --- tools/get_web_page.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100755 tools/get_web_page.sh (limited to 'tools/get_web_page.sh') diff --git a/tools/get_web_page.sh b/tools/get_web_page.sh new file mode 100755 index 0000000..65e6d2a --- /dev/null +++ b/tools/get_web_page.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -e + +# @describe Fetches the HTML content from a specified webpage URL and converts it to Markdown format. +# Use it to answer user questions that require up-to-date content from web pages. +# @meta require-tools curl,pandoc,sed +# @option --url! The URL to scrape. + +main() { + # span and div tags are dropped from the HTML https://pandoc.org/MANUAL.html#raw-htmltex and sed removes any inline SVG images in image tags from the Markdown content. + curl -fsSL "$argc_url" | pandoc -f html-native_divs-native_spans -t gfm-raw_html | sed -E 's/!\[.*?\]\((data:image\/svg\+xml[^)]+)\)//g' +} + +eval "$(argc --argc-eval "$0" "$@")" -- cgit v1.2.3