diff options
| author | sigoden <sigoden@gmail.com> | 2024-07-11 21:43:31 +0800 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2024-07-11 21:43:31 +0800 |
| commit | 5600b68e044b4cd9bc55664ce3f41e9a339fc805 (patch) | |
| tree | 053bc2a7cd88d6a30e1db161a4152d7cd4c1d7af /tools/get_web_page.sh | |
| parent | b56159903bb16dbabec1a8767702c3c0865ef256 (diff) | |
| download | llm-functions-docker-5600b68e044b4cd9bc55664ce3f41e9a339fc805.tar.gz | |
feat: add jina tools (#73)
Diffstat (limited to 'tools/get_web_page.sh')
| -rwxr-xr-x | tools/get_web_page.sh | 16 |
1 files changed, 0 insertions, 16 deletions
diff --git a/tools/get_web_page.sh b/tools/get_web_page.sh deleted file mode 100755 index 7a516bb..0000000 --- a/tools/get_web_page.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash -set -e - -# @describe Get webpage content from a given URL. -# @meta require-tools pandoc -# @option --url! The URL to scrape. - -main() { - # span and div tags are dropped from the HTML https://pandoc.org/MANUAL.html#raw-htmltex and sed removes any inline SVG images in image tags from the Markdown content. - curl -fsSL "$argc_url" | \ - pandoc -f html-native_divs-native_spans -t gfm-raw_html | \ - sed -E 's/!\[.*?\]\((data:image\/svg\+xml[^)]+)\)//g' \ - >> "$LLM_OUTPUT" -} - -eval "$(argc --argc-eval "$0" "$@")" |
