aboutsummaryrefslogtreecommitdiffstats
path: root/tools/get_web_page.sh
diff options
context:
space:
mode:
authorsigoden <sigoden@gmail.com>2024-07-11 21:43:31 +0800
committerGitHub <noreply@github.com>2024-07-11 21:43:31 +0800
commit5600b68e044b4cd9bc55664ce3f41e9a339fc805 (patch)
tree053bc2a7cd88d6a30e1db161a4152d7cd4c1d7af /tools/get_web_page.sh
parentb56159903bb16dbabec1a8767702c3c0865ef256 (diff)
downloadllm-functions-docker-5600b68e044b4cd9bc55664ce3f41e9a339fc805.tar.gz
feat: add jina tools (#73)
Diffstat (limited to 'tools/get_web_page.sh')
-rwxr-xr-xtools/get_web_page.sh16
1 files changed, 0 insertions, 16 deletions
diff --git a/tools/get_web_page.sh b/tools/get_web_page.sh
deleted file mode 100755
index 7a516bb..0000000
--- a/tools/get_web_page.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-# @describe Get webpage content from a given URL.
-# @meta require-tools pandoc
-# @option --url! The URL to scrape.
-
-main() {
- # span and div tags are dropped from the HTML https://pandoc.org/MANUAL.html#raw-htmltex and sed removes any inline SVG images in image tags from the Markdown content.
- curl -fsSL "$argc_url" | \
- pandoc -f html-native_divs-native_spans -t gfm-raw_html | \
- sed -E 's/!\[.*?\]\((data:image\/svg\+xml[^)]+)\)//g' \
- >> "$LLM_OUTPUT"
-}
-
-eval "$(argc --argc-eval "$0" "$@")"