From 5600b68e044b4cd9bc55664ce3f41e9a339fc805 Mon Sep 17 00:00:00 2001 From: sigoden Date: Thu, 11 Jul 2024 21:43:31 +0800 Subject: feat: add jina tools (#73) --- tools/fetch_url_via_curl.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100755 tools/fetch_url_via_curl.sh (limited to 'tools/fetch_url_via_curl.sh') diff --git a/tools/fetch_url_via_curl.sh b/tools/fetch_url_via_curl.sh new file mode 100755 index 0000000..2f2efdc --- /dev/null +++ b/tools/fetch_url_via_curl.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -e + +# @describe Extract the content from a given URL. +# @meta require-tools pandoc +# @option --url! The URL to scrape. + +main() { + # span and div tags are dropped from the HTML https://pandoc.org/MANUAL.html#raw-htmltex and sed removes any inline SVG images in image tags from the Markdown content. + curl -fsSL "$argc_url" | \ + pandoc -f html-native_divs-native_spans -t gfm-raw_html | \ + sed -E 's/!\[.*?\]\((data:image\/svg\+xml[^)]+)\)//g' \ + >> "$LLM_OUTPUT" +} + +eval "$(argc --argc-eval "$0" "$@")" -- cgit v1.2.3