gpt - annna - Annna the nice friendly bot.
HTML git clone git://bitreich.org/annna/ git://enlrupgkhuxnvlhsf6lc3fziv5h2hhfrinws65d7roiv6bfj7d652fid.onion/annna/
DIR Log
DIR Files
DIR Refs
DIR Tags
DIR README
---
gpt (1954B)
---
1 #!/bin/sh
2
3 export PATH="$HOME/bin:$PATH"
4
5 function local_llama() {
6 #ggmlbase="/br/ai/ggml"
7 ggmlbase="/br/ai/llama.cpp"
8 #ggmlbin="./build/bin/gpt-2"
9 ggmlbin="./build/bin/llama-cli"
10 #ggmlmodel="models/gpt-2-1558M/ggml-model.bin"
11 ggmlmodel="models/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf"
12 ggmlntokens="$((69 * 2))"
13
14 cd $ggmlbase
15 if mountpoint -q /br/ai/tmp;
16 then
17 ggmlbasename="$(basename "${ggmlmodel}")"
18 ggmltmppath="/br/ai/tmp/${ggmlbasename}"
19 [ ! -r "${ggmltmppath}" ] && cp "$ggmlmodel" /br/ai/tmp
20 [ -r "${ggmltmppath}" ] && ggmlmodel="${ggmltmppath}"
21 fi
22
23 prompt="$1"
24 if [ -z "$prompt" ];
25 then
26 cat \
27 | $ggmlbin -m $ggmlmodel -n $ggmlntokens -t 3 \
28 --no-warmup --simple-io --no-display-prompt --grammar 'root ::= ([^\x00-\x1F])*' \
29 -cnv 2>/dev/null \
30 | sed -E '/^$/d;s/^>[[:blank:]]+//;q' \
31 | sed -e 's/^"//;s/"$//;'
32 else
33 printf "%s\n" "${prompt}" \
34 | $ggmlbin -m $ggmlmodel -n $ggmlntokens -t 3 \
35 --no-warmup --simple-io --no-display-prompt --grammar 'root ::= ([^\x00-\x1F])*' \
36 -cnv 2>/dev/null \
37 | sed -E '/^$/d;s/^>[[:blank:]]+//;q' \
38 | sed -e 's/^"//;s/"$//;'
39 fi
40 #$ggmlbin -m $ggmlmodel -n $ggmlntokens \
41 # --simple-io --no-display-prompt --grammar 'root ::= ([^\x00-\x1F])*' \
42 # -p "$1" 2>/dev/null \
43 # | head -n1 \
44 # | sed -E 's/^[[:blank:]]+//;s/[[:blank:]]*\[end of text\]$//' \
45 # | tr -d '"'
46 }
47
48 function remote_llama() {
49 prompt="$1"
50 if [ -z "$prompt" ];
51 then
52 cat \
53 | ollama-gpu \
54 ollama run \
55 --hidethinking \
56 --nowordwrap \
57 llama3:8b \
58 "${prompt}" \
59 | head -n 1 \
60 | sed -e 's/^"//;s/"$//;'
61 else
62 printf "%s\n" "${prompt}" \
63 | ollama-gpu \
64 ollama run \
65 --hidethinking \
66 --nowordwrap \
67 llama3:8b \
68 "${prompt}" \
69 | head -n 1 \
70 | sed -e 's/^"//;s/"$//;'
71 fi
72 }
73
74 prompt="$1"
75 response="$(remote_llama "${prompt}")"
76 [ -z "${response}" ] && response="$(local_llama "${prompt}")"
77 [ -n "${response}" ] && printf "%s\n" "${response}"
78