annna-add-hashtag - annna - Annna the nice friendly bot.
HTML git clone git://bitreich.org/annna/ git://enlrupgkhuxnvlhsf6lc3fziv5h2hhfrinws65d7roiv6bfj7d652fid.onion/annna/
DIR Log
DIR Files
DIR Refs
DIR Tags
DIR README
---
annna-add-hashtag (9548B)
---
1 #!/bin/sh
2
3 export PATH="/home/annna/bin:$PATH"
4
5 memecachedir="/br/gopher/memecache"
6 hashtagdb="/home/annna/bin/modules/hashtags/hashtags.txt"
7 unprocessed="/home/annna/bin/modules/hashtags/unprocessed.txt"
8 memeindex="/br/gopher/memecache/index.meme"
9
10 [ ! -e "${hashtagdb}" ] && touch "${hashtagdb}"
11
12 dltag() {
13 tag="$1"
14 uri="$2"
15
16 grep "${tag} " $hashtagdb >/dev/null
17 if [ $? -eq 0 ];
18 then
19 printf "tag '%s' already in db.\n" "${tag}" >&2
20 printf "%s %s\n" "${tag}" "${uri}" >> "${unprocessed}"
21 return 1
22 fi
23 stag="$(printf "${tag}\n" | cut -c 2-)"
24
25 cd "${memecachedir}"
26 error=0
27 case "$uri" in
28 *4chan.org*|\
29 *afreecatv.com*|\
30 *artstation.com*|\
31 *asiansister.com*|\
32 *asmhentai.com*|\
33 *avgle.com*|\
34 *bandcamp.com*|\
35 *baraag.net*|\
36 *bcy.net*|\
37 *bdsmlr.com*|\
38 *bilibili.com*|\
39 *comic-walker.com*|\
40 *coub.com*|\
41 *danbooru.donmai.us*|\
42 *dropbox.com*|\
43 *dumpert.nl*|\
44 *webtoon.kakao.com*|\
45 *deviantart.com*|\
46 *e-hentai.org*|\
47 *exhentai.org*|\
48 *facebook.com*|\
49 *video.fc2.com*|\
50 *flickr.com*|\
51 *gelbooru.com*|\
52 *syosetu.org*|\
53 *hanime.tv*|\
54 *hentai-foundry.com*|\
55 *hitomi.la*|\
56 *hiyobi.me*|\
57 *instagram.com*|\
58 *iwara.tv*|\
59 *ecchi.iwara.tv*|\
60 *jmana.net*|\
61 *kakuyomu.jp*|\
62 *loveheaven.net*|\
63 *likee.video*|\
64 *luscious.net*|\
65 *myreadingmanga.info*|\
66 *blog.naver.com*|\
67 *cafe.naver.com*|\
68 *post.naver.com*|\
69 *comic.naver.com*|\
70 *tv.naver.com*|\
71 *nhentai.net*|\
72 *nhentai.com*|\
73 *nicovideo.jp*|\
74 *nijie.info*|\
75 *nozomi.la*|\
76 *pawoo.net*|\
77 *pinterest.com*|\
78 *pixiv.net*|\
79 *comic.pixiv.net*|\
80 *pornhub.com*|\
81 *pornhubpremium.com*|\
82 *rule34.xxx*|\
83 *www.sankakucomplex.com*|\
84 *chan.sankakucomplex.com*|\
85 *idol.sankakucomplex.com*|\
86 *soundcloud.com*|\
87 *syosetu.com*|\
88 *tokyomotion.net*|\
89 *tumblr.com*|\
90 *twitch.tv*|\
91 *twitter.com*|\
92 *vimeo.com*|\
93 *vlive.tv*|\
94 *archive.org*|\
95 *weibo.com*|\
96 *www.wikiart.org*|\
97 *xhamster.com*|\
98 *xnxx.com*|\
99 *xvideos.com*|\
100 *yande.re*|\
101 *youku.com*|\
102 *youtube.com*|\
103 *youtu.be*|\
104 *yewtu.be*|\
105 *vimeo.com*|\
106 *twitter.com*|\
107 *reddit.com*|\
108 *v.redd.it*|\
109 *invidious*|\
110 *gfycat.com*|\
111 *xvideos.com*|\
112 *youporn.com*|\
113 *tiktok.com*|\
114 *invidious-us.kavin.rocks*|\
115 *vid.puffyan.us*|\
116 *invidious.snopyta.org*|\
117 *youtube.076.ne.jp*|\
118 *inv.riverside.rocks*|\
119 *yewtu.be*|\
120 *invidio.xamh.de*|\
121 *invidious.osi.kr*|\
122 *invidious.namazso.eu*|\
123 *inv.cthd.icu*|\
124 *yt.artemislena.eu*|\
125 *invidious.kavin.rocks*|\
126 *dl.dropboxusercontent.com/*|\
127 *dl-eu.dropboxusercontent.com*|\
128 *.dl-eu.dropboxusercontent.com*|\
129 *dl-au.dropboxusercontent.com*|\
130 *.dl-au.dropboxusercontent.com*|\
131 *dl-jp.dropboxusercontent.com*|\
132 *.dl-jp.dropboxusercontent.com*|\
133 *dl-uk.dropboxusercontent.com*|\
134 *.dl-uk.dropboxusercontent.com*|\
135 *dl.dropbox.com*|\
136 *dl-eu.dropbox.com*|\
137 *dl-au.dropbox.com*|\
138 *dl-jp.dropbox.com*|\
139 *dl-uk.dropbox.com*|\
140 *dl-web.dropbox.co/*|\
141 *dl-web-eu.dropbox.com*|\
142 *dl-web-au.dropbox.com*|\
143 *dl-web-jp.dropbox.com*|\
144 *dl-web-uk.dropbox.com*|\
145 *dl-client.dropbox.com/*|\
146 *dl-doc.dropbox.com/*|\
147 *dl-doc.dropboxusercontent.com/*|\
148 *api-content.dropbox.com/*|\
149 *api-content-eu.dropbox.com*|\
150 *api-content1-eu.dropbox.com*|\
151 *api-content-au.dropbox.com*|\
152 *api-content1-au.dropbox.com*|\
153 *api-content-jp.dropbox.com*|\
154 *api-content1-jp.dropbox.com*|\
155 *api-content-uk.dropbox.com*|\
156 *api-content1-uk.dropbox.com*|\
157 *content.dropboxapi.com/*|\
158 *content-eu.dropboxapi.com*|\
159 *content-au.dropboxapi.com*|\
160 *content-jp.dropboxapi.com*|\
161 *content-uk.dropboxapi.com*|\
162 *api-content-photos.dropbox.com/*|\
163 *files-eu.dropbox.com*|\
164 *files-au.dropbox.com*|\
165 *files-jp.dropbox.com*|\
166 *files-uk.dropbox.com*|\
167 *showbox-tr.dropbox.com/*|\
168 *showbox-eu.dropbox.com*|\
169 *showbox-au.dropbox.com*|\
170 *showbox-jp.dropbox.com*|\
171 *showbox-uk.dropbox.com*|\
172 *photos.dropbox.com/*|\
173 *photos-*.dropbox.com/*|\
174 *dl-debug*.dropbox.com/*|\
175 *bolt.dropbox.com/*|\
176 *.dropbox.com/content_link/*|\
177 *.dropbox.com/content_link_zip/*|\
178 *.dropbox.com/content_link_htmlify/*|\
179 *.dropbox.com/content_link_preview/*|\
180 *.dropbox.com/link_token_dl/*|\
181 *.dropbox.com/link_token_dl_zip/*|\
182 *.dropbox.com/zip_collection/*|\
183 *.dropbox.com/transcode_video/*|\
184 *.dropbox.com/hls_transcode/*|\
185 *.dropbox.com/t/*|\
186 *.dropbox.com/u/|\
187 *.dropbox.com/s/*|\
188 *.dropbox.com/sh/*|\
189 *.dropbox.com/sc/*|\
190 *.dropbox.com/photos/album/*|\
191 *.dropbox.com/photos/c/*|\
192 *.dropbox.com/photos/cc/*|\
193 *.dropbox.com/photos/cl/*|\
194 *.dropbox.com/photos/cp/*|\
195 *.dropbox.com/photos/conversation_add/*|\
196 *.dropbox.com/photos/conversation_invite/*|\
197 *.dropbox.com/photos/conversation_preview/*|\
198 *.dropbox.com/photos/post/*|\
199 *.dropbox.com/photos/c/permanent_collection_thumb/*|\
200 *ext-twitch.tv*|\
201 *jtvnw.net*|\
202 *live-video.net*|\
203 *ttvnw.net*|\
204 *twitch.okta.com*|\
205 *twitch.tv*|\
206 *twitchcdn.net*|\
207 *twitchsvc.net*|\
208 *nitter.42l.fr*|\
209 *nitter.pussthecat.org*|\
210 *nitter.nixnet.services*|\
211 *nitter.fdn.fr*|\
212 *nitter.1d4.us*|\
213 *nitter.kavin.rocks*|\
214 *nitter.unixfox.eu*|\
215 *nitter.domain.glass*|\
216 *nitter.eu*|\
217 *nitter.namazso.eu*|\
218 *nitter.actionsack.com*|\
219 *birdsite.xanny.family*|\
220 *nitter.hu*|\
221 *twitr.gq*|\
222 *nitter.moomoo.me*|\
223 *nittereu.moomoo.me*|\
224 *bird.trom.tf*|\
225 *nitter.it*|\
226 *twitter.censors.us*|\
227 *nitter.grimneko.de*|\
228 *nitter.alefvanoon.xyz*|\
229 *n.hyperborea.cloud*|\
230 *nitter.ca*|\
231 *twitter.076.ne.jp*|\
232 *nitter.mstdn.social*|\
233 *nitter.fly.dev*|\
234 *notabird.site*|\
235 *nitter.weiler.rocks*|\
236 *nitter.silkky.cloud*|\
237 *nitter.sethforprivacy.com*|\
238 *nttr.stream*|\
239 *nitter.cutelab.space*|\
240 *nitter.nl*|\
241 *nitter.mint.lgbt*|\
242 *nitter.tokhmi.xyz*|\
243 *nitter.bus-hit.me*|\
244 *fuckthesacklers.network*|\
245 *nitter.govt.land*|\
246 *nitter.datatunnel.xyz*|\
247 *nitter.esmailelbob.xyz*|\
248 *tw.artemislena.eu*|\
249 *ardmediathek.de*|\
250 *nitter.eu.org*)
251 youtube-dl -o "${stag}.%(ext)s" "${uri}"
252 if [ $? -gt 0 ];
253 then
254 error=1
255 printf "youtube-dl tag '%s' had problems.\n" "${tag}" >&2
256 printf "%s %s\n" "${tag}" "${uri}" >> "${unprocessed}"
257 fi
258 ;;
259 *)
260 ext="$(basename "${uri}" | sed 's/^.*\.//')"
261 curl -Lo "${stag}.${ext}" "${uri}"
262 if [ $? -gt 0 ];
263 then
264 error=1
265 printf "curl tag '%s' had problems.\n" "${tag}" >&2
266 printf "%s %s\n" "${tag}" "${uri}" >> "${unprocessed}"
267 fi
268 ;;
269 esac
270 if [ $error -eq 0 ];
271 then
272 printf "tag '%s' downloaded.\n" "${tag}" >&2
273 printf "%s\n" "${uri}" > "${stag}.orig"
274 fi
275
276 return $error
277 }
278
279 processtag() {
280 tag="$1"
281 uri="$2"
282
283 cd "${memecachedir}"
284 tagfile="$(basename "$(find . -name "${stag}.*" | grep -v '\.orig$')")"
285
286 # In case youtube-dl mangled with the extension, rename to what file
287 # suggests us as ending.
288 case "${tagfile}" in
289 *.m3u8|*.M3U8)
290 realext="$(file --mime-type -b "${tagfile}" | cut -d'/' -f 2)"
291 if [ "${realext}" != "m3u8" ];
292 then
293 newtagfile="$(printf "%s\n" "${tagfile}" \
294 | tr '[:upper:]' '[:lower:]' \
295 | sed "s,m3u8,${realext},")"
296 printf "Renaming %s to %s.\n" "${tagfile}" "${newtagfile}"
297 mv "${tagfile}" "${newtagfile}"
298 tagfile="${newtagfile}"
299 fi
300 ;;
301 esac
302
303 tagfilenew="$(/br/bin/quinq-size "${tagfile}" </dev/null | sed 's/.* -> //')"
304 if [ -e "${tagfilenew}" ] && [ $? -eq 0 ];
305 then
306 mv "${tagfilenew}" "${tagfile}"
307 fi
308
309 case "${tagfile}" in
310 *.MP4|*.mp4|*.MKV|*.mkv|*.webm|*.WEBM|*.mov|*.MOV)
311 printf "Making %s streamable by adding faststart.\n" "${tagfile}"
312 ffmpeg -i "${tagfile}" -c copy -movflags faststart "faststart_${tagfile}" </dev/null
313 mv "faststart_${tagfile}" "${tagfile}"
314 ;;
315 esac
316 }
317
318 inserttag() {
319 tag="$1"
320 uri="$2"
321
322 grep "${tag} " $hashtagdb >/dev/null
323 if [ $? -eq 0 ];
324 then
325 printf "tag '%s' already in db.\n" "${tag}" >&2
326 printf "%s %s\n" "${tag}" "${uri}" >> "${unprocessed}"
327 return 1
328 fi
329
330 stag="$(printf "${tag}\n" | cut -c 2-)"
331 cd "${memecachedir}"
332 tagfile="$(basename "$(find . -name "${stag}.*" | grep -v '\.orig$')")"
333 tagtype="$(get-tagtype "${tagfile}")"
334 memecacheuri="gophers://bitreich.org/${tagtype}/memecache/${tagfile}"
335
336 printf "%s %s\n" "${tag}" "${memecacheuri}" >> "${hashtagdb}"
337 printf "tag '%s' added as '%s' to hashtags db.\n" "${tag}" "${memecacheuri}" >&2
338 LC_ALL=C sort -t ' ' -k1,1 "${hashtagdb}" > "${hashtagdb}.sort"
339 mv "$hashtagdb.sort" "${hashtagdb}"
340 # keep permissions shared.
341 chown ':bitreich' "${hashtagdb}"
342 chmod 664 "${hashtagdb}"
343 }
344
345 indextag() {
346 tag="$1"
347 uri="$2"
348
349 grep "${tag} " $memeindex >/dev/null
350 if [ $? -eq 0 ];
351 then
352 printf "tag '%s' already in index.\n" "${tag}" >&2
353 return 1
354 fi
355
356 stag="$(printf "${tag}\n" | cut -c 2-)"
357 cd "${memecachedir}"
358 tagfile="$(basename "$(find . -name "${stag}.*" | grep -v '\.orig$')")"
359 checksum="$(sha512sum "${tagfile}" | cut -d' ' -f 1)"
360 tagtype="$(get-tagtype "${tagfile}")"
361 memecacheuri="gophers://bitreich.org/${tagtype}/memecache/${tagfile}"
362
363 printf "%s %s %s\n" "${tag}" "${memecacheuri}" "${checksum}" >> "${memeindex}"
364 printf "tag '%s' added as '%s' to meme index.\n" "${tag}" "${memecacheuri}" >&2
365 tail -n +2 "${memeindex}" | LC_ALL=C sort -t ' ' -k1,1 > "${memeindex}.sort"
366
367 {
368 printf "meme2\n"
369 cat "${memeindex}.sort"
370 } > "${memeindex}"
371 rm "${memeindex}.sort"
372 # keep permissions shared.
373 chown ':bitreich' "${memeindex}"
374 chmod 664 "${memeindex}"
375 }
376
377 addtag() {
378 tag="$1"
379 uri="$2"
380
381 if printf '%s' "$1" | cut -c 2- | grep -qE "[# ',?!]";
382 then
383 printf 'error: tag cannot contain the following characters: # '\'',?!\n' >&2
384 exit 1
385 fi
386
387 dltag "$1" "$2"
388 [ $? -gt 0 ] && return
389 processtag "$1" "$2"
390 [ $? -gt 0 ] && return
391 inserttag "$1" "$2"
392 [ $? -gt 0 ] && return
393 indextag "$1" "$2"
394 }
395
396 if [ $# -eq 2 ];
397 then
398 tag="$1"
399 uri="$2"
400
401 addtag "${tag}" "${uri}"
402 else
403 if [ $# -eq 1 ];
404 then
405 [ -e "$1" ] && cat "$1" \
406 | while read -r tag uri;
407 do
408 addtag "${tag}" "${uri}"
409 done
410 else
411 while read -r tag uri;
412 do
413 addtag "${tag}" "${uri}"
414 done
415 fi
416 fi