filtron.sh 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562
  1. #!/usr/bin/env bash
  2. # -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
  3. # SPDX-License-Identifier: AGPL-3.0-or-later
  4. # shellcheck disable=SC2119,SC2001
  5. # shellcheck source=utils/lib.sh
  6. source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
  7. # shellcheck source=utils/brand.env
  8. source "${REPO_ROOT}/utils/brand.env"
  9. source_dot_config
  10. source "${REPO_ROOT}/utils/lxc-searx.env"
  11. in_container && lxc_set_suite_env
  12. # ----------------------------------------------------------------------------
  13. # config
  14. # ----------------------------------------------------------------------------
  15. PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}"
  16. PUBLIC_HOST="${PUBLIC_HOST:-$(echo "$PUBLIC_URL" | sed -e 's/[^/]*\/\/\([^@]*@\)\?\([^:/]*\).*/\2/')}"
  17. FILTRON_URL_PATH="${FILTRON_URL_PATH:-$(echo "${PUBLIC_URL}" \
  18. | sed -e 's,^.*://[^/]*\(/.*\),\1,g')}"
  19. [[ "${FILTRON_URL_PATH}" == "${PUBLIC_URL}" ]] && FILTRON_URL_PATH=/
  20. FILTRON_ETC="/etc/filtron"
  21. FILTRON_RULES="$FILTRON_ETC/rules.json"
  22. FILTRON_API="${FILTRON_API:-127.0.0.1:4005}"
  23. FILTRON_LISTEN="${FILTRON_LISTEN:-127.0.0.1:4004}"
  24. FILTRON_TARGET="${FILTRON_TARGET:-127.0.0.1:8888}"
  25. SERVICE_NAME="filtron"
  26. SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}"
  27. SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}"
  28. SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}"
  29. SERVICE_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${SERVICE_NAME}.service"
  30. # shellcheck disable=SC2034
  31. SERVICE_GROUP="${SERVICE_USER}"
  32. # shellcheck disable=SC2034
  33. SERVICE_GROUP="${SERVICE_USER}"
  34. GO_ENV="${SERVICE_HOME}/.go_env"
  35. GO_PKG_URL="https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz"
  36. GO_TAR=$(basename "$GO_PKG_URL")
  37. APACHE_FILTRON_SITE="searx.conf"
  38. NGINX_FILTRON_SITE="searx.conf"
  39. # shellcheck disable=SC2034
  40. CONFIG_FILES=(
  41. "${FILTRON_RULES}"
  42. "${SERVICE_SYSTEMD_UNIT}"
  43. )
  44. # ----------------------------------------------------------------------------
  45. usage() {
  46. # ----------------------------------------------------------------------------
  47. # shellcheck disable=SC1117
  48. cat <<EOF
  49. usage::
  50. $(basename "$0") shell
  51. $(basename "$0") install [all|user|rules]
  52. $(basename "$0") update [filtron]
  53. $(basename "$0") remove [all]
  54. $(basename "$0") activate [service]
  55. $(basename "$0") deactivate [service]
  56. $(basename "$0") inspect [service]
  57. $(basename "$0") option [debug-on|debug-off]
  58. $(basename "$0") apache [install|remove]
  59. $(basename "$0") nginx [install|remove]
  60. shell
  61. start interactive shell from user ${SERVICE_USER}
  62. install / remove
  63. :all: complete setup of filtron service
  64. :user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME)
  65. :rules: reinstall filtron rules $FILTRON_RULES
  66. update filtron
  67. Update filtron installation ($SERVICE_HOME)
  68. activate service
  69. activate and start service daemon (systemd unit)
  70. deactivate service
  71. stop and deactivate service daemon (systemd unit)
  72. inspect service
  73. show service status and log
  74. option
  75. set one of the available options
  76. apache (${PUBLIC_URL})
  77. :install: apache site with a reverse proxy (ProxyPass)
  78. :remove: apache site ${APACHE_FILTRON_SITE}
  79. nginx (${PUBLIC_URL})
  80. :install: nginx site with a reverse proxy (ProxyPass)
  81. :remove: nginx site ${NGINX_FILTRON_SITE}
  82. filtron rules: ${FILTRON_RULES}
  83. If needed, set PUBLIC_URL of your WEB service in the '${DOT_CONFIG#"$REPO_ROOT/"}' file::
  84. PUBLIC_URL : ${PUBLIC_URL}
  85. PUBLIC_HOST : ${PUBLIC_HOST}
  86. SERVICE_USER : ${SERVICE_USER}
  87. FILTRON_TARGET : ${FILTRON_TARGET}
  88. FILTRON_API : ${FILTRON_API}
  89. FILTRON_LISTEN : ${FILTRON_LISTEN}
  90. EOF
  91. if in_container; then
  92. # in containers the service is listening on 0.0.0.0 (see lxc-searx.env)
  93. for ip in $(global_IPs) ; do
  94. if [[ $ip =~ .*:.* ]]; then
  95. echo " container URL (IPv6): http://[${ip#*|}]:4005/"
  96. else
  97. # IPv4:
  98. echo " container URL (IPv4): http://${ip#*|}:4005/"
  99. fi
  100. done
  101. fi
  102. [[ -n ${1} ]] && err_msg "$1"
  103. }
  104. main() {
  105. required_commands \
  106. sudo install git wget curl \
  107. || exit
  108. local _usage="unknown or missing $1 command $2"
  109. case $1 in
  110. --getenv) var="$2"; echo "${!var}"; exit 0;;
  111. -h|--help) usage; exit 0;;
  112. shell)
  113. sudo_or_exit
  114. interactive_shell "${SERVICE_USER}"
  115. ;;
  116. inspect)
  117. case $2 in
  118. service)
  119. sudo_or_exit
  120. inspect_service
  121. ;;
  122. *) usage "$_usage"; exit 42;;
  123. esac ;;
  124. install)
  125. rst_title "$SERVICE_NAME" part
  126. sudo_or_exit
  127. case $2 in
  128. all) install_all ;;
  129. user) assert_user ;;
  130. rules)
  131. rst_title "Re-Install filtron rules"
  132. echo
  133. install_template --no-eval "$FILTRON_RULES" root root 644
  134. systemd_restart_service "${SERVICE_NAME}"
  135. ;;
  136. *) usage "$_usage"; exit 42;;
  137. esac ;;
  138. update)
  139. sudo_or_exit
  140. case $2 in
  141. filtron) update_filtron ;;
  142. *) usage "$_usage"; exit 42;;
  143. esac ;;
  144. remove)
  145. sudo_or_exit
  146. case $2 in
  147. all) remove_all;;
  148. user) drop_service_account "${SERVICE_USER}" ;;
  149. *) usage "$_usage"; exit 42;;
  150. esac ;;
  151. activate)
  152. sudo_or_exit
  153. case $2 in
  154. service) systemd_activate_service "${SERVICE_NAME}" ;;
  155. *) usage "$_usage"; exit 42;;
  156. esac ;;
  157. deactivate)
  158. sudo_or_exit
  159. case $2 in
  160. service) systemd_deactivate_service "${SERVICE_NAME}" ;;
  161. *) usage "$_usage"; exit 42;;
  162. esac ;;
  163. apache)
  164. sudo_or_exit
  165. case $2 in
  166. install) install_apache_site ;;
  167. remove) remove_apache_site ;;
  168. *) usage "$_usage"; exit 42;;
  169. esac ;;
  170. nginx)
  171. sudo_or_exit
  172. case $2 in
  173. install) install_nginx_site ;;
  174. remove) remove_nginx_site ;;
  175. *) usage "$_usage"; exit 42;;
  176. esac ;;
  177. option)
  178. sudo_or_exit
  179. case $2 in
  180. debug-on) echo; enable_debug ;;
  181. debug-off) echo; disable_debug ;;
  182. *) usage "$_usage"; exit 42;;
  183. esac ;;
  184. doc) rst-doc ;;
  185. *) usage "unknown or missing command $1"; exit 42;;
  186. esac
  187. }
  188. install_all() {
  189. rst_title "Install $SERVICE_NAME (service)"
  190. assert_user
  191. wait_key
  192. install_go "${GO_PKG_URL}" "${GO_TAR}" "${SERVICE_USER}"
  193. wait_key
  194. install_filtron
  195. wait_key
  196. systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"
  197. wait_key
  198. echo
  199. if ! service_is_available "http://${FILTRON_LISTEN}" ; then
  200. err_msg "Filtron does not listening on: http://${FILTRON_LISTEN}"
  201. fi
  202. if apache_is_installed; then
  203. info_msg "Apache is installed on this host."
  204. if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then
  205. install_apache_site
  206. fi
  207. elif nginx_is_installed; then
  208. info_msg "nginx is installed on this host."
  209. if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then
  210. install_nginx_site
  211. fi
  212. fi
  213. if ask_yn "Do you want to inspect the installation?" Ny; then
  214. inspect_service
  215. fi
  216. }
  217. remove_all() {
  218. rst_title "De-Install $SERVICE_NAME (service)"
  219. rst_para "\
  220. It goes without saying that this script can only be used to remove
  221. installations that were installed with this script."
  222. if ! systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"; then
  223. return 42
  224. fi
  225. drop_service_account "${SERVICE_USER}"
  226. rm -r "$FILTRON_ETC" 2>&1 | prefix_stdout
  227. if service_is_available "${PUBLIC_URL}"; then
  228. MSG="** Don't forget to remove your public site! (${PUBLIC_URL}) **" wait_key 10
  229. fi
  230. }
  231. assert_user() {
  232. rst_title "user $SERVICE_USER" section
  233. echo
  234. tee_stderr 1 <<EOF | bash | prefix_stdout
  235. useradd --shell /bin/bash --system \
  236. --home-dir "$SERVICE_HOME" \
  237. --comment 'Reverse HTTP proxy to filter requests' $SERVICE_USER
  238. mkdir "$SERVICE_HOME"
  239. chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME"
  240. groups $SERVICE_USER
  241. EOF
  242. SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)"
  243. export SERVICE_HOME
  244. echo "export SERVICE_HOME=$SERVICE_HOME"
  245. cat > "$GO_ENV" <<EOF
  246. export GOPATH=\$HOME/go-apps
  247. export PATH=\$PATH:\$HOME/local/go/bin:\$GOPATH/bin
  248. EOF
  249. echo "Environment $GO_ENV has been setup."
  250. tee_stderr <<EOF | sudo -i -u "$SERVICE_USER"
  251. grep -qFs -- 'source $GO_ENV' ~/.profile || echo 'source $GO_ENV' >> ~/.profile
  252. EOF
  253. }
  254. filtron_is_installed() {
  255. [[ -f $SERVICE_HOME/go-apps/bin/filtron ]]
  256. }
  257. _svcpr=" ${_Yellow}|${SERVICE_USER}|${_creset} "
  258. install_filtron() {
  259. rst_title "Install filtron in user's ~/go-apps" section
  260. echo
  261. tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr"
  262. go get -v -u github.com/asciimoo/filtron
  263. EOF
  264. install_template --no-eval "$FILTRON_RULES" root root 644
  265. }
  266. update_filtron() {
  267. rst_title "Update filtron" section
  268. echo
  269. tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr"
  270. go get -v -u github.com/asciimoo/filtron
  271. EOF
  272. }
  273. inspect_service() {
  274. rst_title "service status & log"
  275. cat <<EOF
  276. sourced ${DOT_CONFIG#"$REPO_ROOT/"} :
  277. PUBLIC_URL : ${PUBLIC_URL}
  278. PUBLIC_HOST : ${PUBLIC_HOST}
  279. FILTRON_URL_PATH : ${FILTRON_URL_PATH}
  280. FILTRON_API : ${FILTRON_API}
  281. FILTRON_LISTEN : ${FILTRON_LISTEN}
  282. FILTRON_TARGET : ${FILTRON_TARGET}
  283. EOF
  284. if service_account_is_available "$SERVICE_USER"; then
  285. info_msg "service account $SERVICE_USER available."
  286. else
  287. err_msg "service account $SERVICE_USER not available!"
  288. fi
  289. if go_is_available "$SERVICE_USER"; then
  290. info_msg "~$SERVICE_USER: go is installed"
  291. else
  292. err_msg "~$SERVICE_USER: go is not installed"
  293. fi
  294. if filtron_is_installed; then
  295. info_msg "~$SERVICE_USER: filtron app is installed"
  296. else
  297. err_msg "~$SERVICE_USER: filtron app is not installed!"
  298. fi
  299. if ! service_is_available "http://${FILTRON_API}"; then
  300. err_msg "API not available at: http://${FILTRON_API}"
  301. fi
  302. if ! service_is_available "http://${FILTRON_LISTEN}" ; then
  303. err_msg "Filtron does not listening on: http://${FILTRON_LISTEN}"
  304. fi
  305. if service_is_available "http://${FILTRON_TARGET}" ; then
  306. info_msg "Filtron's target is available at: http://${FILTRON_TARGET}"
  307. fi
  308. if ! service_is_available "${PUBLIC_URL}"; then
  309. warn_msg "Public service at ${PUBLIC_URL} is not available!"
  310. if ! in_container; then
  311. warn_msg "Check if public name is correct and routed or use the public IP from above."
  312. fi
  313. fi
  314. if in_container; then
  315. lxc_suite_info
  316. else
  317. info_msg "public URL --> ${PUBLIC_URL}"
  318. info_msg "internal URL --> http://${FILTRON_LISTEN}"
  319. fi
  320. local _debug_on
  321. if ask_yn "Enable filtron debug mode?"; then
  322. enable_debug
  323. _debug_on=1
  324. fi
  325. echo
  326. systemctl --no-pager -l status "${SERVICE_NAME}"
  327. echo
  328. info_msg "public URL --> ${PUBLIC_URL}"
  329. # shellcheck disable=SC2059
  330. printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log"
  331. read -r -s -n1 -t 5
  332. echo
  333. while true; do
  334. trap break 2
  335. journalctl -f -u "${SERVICE_NAME}"
  336. done
  337. if [[ $_debug_on == 1 ]]; then
  338. disable_debug
  339. fi
  340. return 0
  341. }
  342. enable_debug() {
  343. info_msg "try to enable debug mode ..."
  344. python <<EOF
  345. import sys, json
  346. debug = {
  347. u'name': u'debug request'
  348. , u'filters': []
  349. , u'interval': 0
  350. , u'limit': 0
  351. , u'actions': [{u'name': u'log'}]
  352. }
  353. with open('$FILTRON_RULES') as rules:
  354. j = json.load(rules)
  355. pos = None
  356. for i in range(len(j)):
  357. if j[i].get('name') == 'debug request':
  358. pos = i
  359. break
  360. if pos is not None:
  361. j[pos] = debug
  362. else:
  363. j.append(debug)
  364. with open('$FILTRON_RULES', 'w') as rules:
  365. json.dump(j, rules, indent=2, sort_keys=True)
  366. EOF
  367. systemctl restart "${SERVICE_NAME}.service"
  368. }
  369. disable_debug() {
  370. info_msg "try to disable debug mode ..."
  371. python <<EOF
  372. import sys, json
  373. with open('$FILTRON_RULES') as rules:
  374. j = json.load(rules)
  375. pos = None
  376. for i in range(len(j)):
  377. if j[i].get('name') == 'debug request':
  378. pos = i
  379. break
  380. if pos is not None:
  381. del j[pos]
  382. with open('$FILTRON_RULES', 'w') as rules:
  383. json.dump(j, rules, indent=2, sort_keys=True)
  384. EOF
  385. systemctl restart "${SERVICE_NAME}.service"
  386. }
  387. install_apache_site() {
  388. rst_title "Install Apache site $APACHE_FILTRON_SITE"
  389. rst_para "\
  390. This installs a reverse proxy (ProxyPass) into apache site (${APACHE_FILTRON_SITE})"
  391. ! apache_is_installed && info_msg "Apache is not installed."
  392. if ! ask_yn "Do you really want to continue?" Yn; then
  393. return
  394. else
  395. install_apache
  396. fi
  397. "${REPO_ROOT}/utils/searx.sh" install uwsgi
  398. apache_install_site --variant=filtron "${APACHE_FILTRON_SITE}"
  399. info_msg "testing public url .."
  400. if ! service_is_available "${PUBLIC_URL}"; then
  401. err_msg "Public service at ${PUBLIC_URL} is not available!"
  402. fi
  403. }
  404. remove_apache_site() {
  405. rst_title "Remove Apache site $APACHE_FILTRON_SITE"
  406. rst_para "\
  407. This removes apache site ${APACHE_FILTRON_SITE}."
  408. ! apache_is_installed && err_msg "Apache is not installed."
  409. if ! ask_yn "Do you really want to continue?" Yn; then
  410. return
  411. fi
  412. apache_remove_site "$APACHE_FILTRON_SITE"
  413. }
  414. install_nginx_site() {
  415. rst_title "Install nginx site $NGINX_FILTRON_SITE"
  416. rst_para "\
  417. This installs a reverse proxy (ProxyPass) into nginx site (${NGINX_FILTRON_SITE})"
  418. ! nginx_is_installed && info_msg "nginx is not installed."
  419. if ! ask_yn "Do you really want to continue?" Yn; then
  420. return
  421. else
  422. install_nginx
  423. fi
  424. "${REPO_ROOT}/utils/searx.sh" install uwsgi
  425. # shellcheck disable=SC2034
  426. SEARX_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SRC)
  427. # shellcheck disable=SC2034
  428. SEARX_URL_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_URL_PATH)
  429. nginx_install_app --variant=filtron "${NGINX_FILTRON_SITE}"
  430. info_msg "testing public url .."
  431. if ! service_is_available "${PUBLIC_URL}"; then
  432. err_msg "Public service at ${PUBLIC_URL} is not available!"
  433. fi
  434. }
  435. remove_nginx_site() {
  436. rst_title "Remove nginx site $NGINX_FILTRON_SITE"
  437. rst_para "\
  438. This removes nginx site ${NGINX_FILTRON_SITE}."
  439. ! nginx_is_installed && err_msg "nginx is not installed."
  440. if ! ask_yn "Do you really want to continue?" Yn; then
  441. return
  442. fi
  443. nginx_remove_site "$FILTRON_FILTRON_SITE"
  444. }
  445. rst-doc() {
  446. eval "echo \"$(< "${REPO_ROOT}/docs/build-templates/filtron.rst")\""
  447. echo -e "\n.. START install systemd unit"
  448. cat <<EOF
  449. .. tabs::
  450. .. group-tab:: systemd
  451. .. code:: bash
  452. EOF
  453. eval "echo \"$(< "${TEMPLATES}/${SERVICE_SYSTEMD_UNIT}")\"" | prefix_stdout " "
  454. echo -e "\n.. END install systemd unit"
  455. # for DIST_NAME in ubuntu-20.04 arch fedora centos; do
  456. # (
  457. # DIST_ID=${DIST_NAME%-*}
  458. # DIST_VERS=${DIST_NAME#*-}
  459. # [[ $DIST_VERS =~ $DIST_ID ]] && DIST_VERS=
  460. # # ...
  461. # )
  462. # done
  463. }
  464. # ----------------------------------------------------------------------------
  465. main "$@"
  466. # ----------------------------------------------------------------------------