Searx/utils/makefile.python

# -*- coding: utf-8; mode: makefile-gmake -*-

# list of python packages (folders) or modules (files) of this build
PYOBJECTS ?=

SITE_PYTHON ?=$(dir $(abspath $(lastword $(MAKEFILE_LIST))))site-python
export PYTHONPATH := $(SITE_PYTHON):$$PYTHONPATH
export PY_ENV PYDIST PYBUILD

# folder where the python distribution takes place
PYDIST   = ./$(LXC_ENV_FOLDER)dist
# folder where the python intermediate build files take place
PYBUILD  = ./$(LXC_ENV_FOLDER)build
# python version to use
PY       ?=3
# $(PYTHON) points to the python interpreter from the OS!  The python from the
# OS is needed e.g. to create a virtualenv.  For tasks inside the virtualenv the
# interpeter from '$(PY_ENV_BIN)/python' is used.
PYTHON   ?= python$(PY)
PIP      ?= pip$(PY)
PIP_INST ?= --user

# https://www.python.org/dev/peps/pep-0508/#extras
#PY_SETUP_EXTRAS ?= \[develop,test\]
PY_SETUP_EXTRAS ?=

PYDEBUG  ?= --pdb
PYLINT_RC ?= .pylintrc

TEST_FOLDER  ?= ./tests
TEST         ?= .

PY_ENV       = ./$(LXC_ENV_FOLDER)local/py$(PY)
PY_ENV_BIN   = $(PY_ENV)/bin
PY_ENV_ACT   = . $(PY_ENV_BIN)/activate

ifeq ($(OS),Windows_NT)
  PYTHON     = python
  PY_ENV_BIN = $(PY_ENV)/Scripts
  PY_ENV_ACT = $(PY_ENV_BIN)/activate
endif

VTENV_OPTS ?=

python-help::
	@echo  'makefile.python:'
	@echo  '  pyenv | pyenv[un]install'
	@echo  '     build $(PY_ENV) & [un]install python objects'
	@echo  '  targts using pyenv $(PY_ENV):'
	@echo  '    pylint    - run pylint *linting*'
	@echo  '    pytest    - run *tox* test on python objects'
	@echo  '    pydebug   - run tests within a PDB debug session'
	@echo  '    pybuild   - build python packages ($(PYDIST) $(PYBUILD))'
	@echo  '    pyclean   - clean intermediate python objects'
	@echo  '  targets using system users environment:'
	@echo  '    py[un]install - [un]install python objects in editable mode'
	@echo  '    upload-pypi   - upload $(PYDIST)/* files to PyPi'
	@echo  'options:'
	@echo  '  make PY=3.7 [targets] => to eval targets with python 3.7 ($(PY))'
	@echo  '  make PIP_INST=        => to set/unset pip install options ($(PIP_INST))'
	@echo  '  make TEST=.           => choose test from $(TEST_FOLDER) (default "." runs all)'
	@echo  '  make DEBUG=           => target "debug": do not invoke PDB on errors'
	@echo  '  make PY_SETUP_EXTRAS  => also install extras_require from setup.py \[develop,test\]'
	@echo  '  when using target "pydebug", set breakpoints within py-source by adding::'
	@echo  '    DEBUG()'

# ------------------------------------------------------------------------------
# OS requirements
# ------------------------------------------------------------------------------

PHONY += msg-python-exe python-exe
msg-python-exe:
	@echo "\n  $(PYTHON) is required\n\n\
  Make sure you have $(PYTHON) installed, grab it from\n\
  https://www.python.org or install it from your package\n\
  manager. On debian based OS these requirements are\n\
  installed by::\n\n\
    sudo -H add-apt-repository ppa:deadsnakes/ppa\n\
    sudo -H apt update\n\
    sudo -H apt-get install $(PYTHON) $(PYTHON)-venv\n"

ifeq ($(shell which $(PYTHON) >/dev/null 2>&1; echo $$?), 1)
python-exe: msg-python-exe
	$(error The '$(PYTHON)' command was not found)
else
python-exe:
	@:
endif

msg-pip-exe:
	@echo "\n  $(PIP) is required\n\n\
  Make sure you have updated pip installed, grab it from\n\
  https://pip.pypa.io or install it from your package\n\
  manager. On debian based OS these requirements are\n\
  installed by::\n\n\
    sudo -H apt-get install python$(PY)-pip\n" | $(FMT)

ifeq ($(shell which $(PIP) >/dev/null 2>&1; echo $$?), 1)
pip-exe: msg-pip-exe
	$(error The '$(PIP)' command was not found)
else
pip-exe:
	@:
endif

# ------------------------------------------------------------------------------
# commands
# ------------------------------------------------------------------------------

# $2 path to folder with setup.py, this uses pip from the OS
quiet_cmd_pyinstall = INSTALL   $2
      cmd_pyinstall = $(PIP) $(PIP_VERBOSE) install $(PIP_INST) -e $2$(PY_SETUP_EXTRAS)

# $2 path to folder with setup.py, this uses pip from pyenv (not OS!)
quiet_cmd_pyenvinstall = PYENV     install $2
      cmd_pyenvinstall = \
	if ! cat $(PY_ENV)/requirements.sha256 2>/dev/null | sha256sum --check --status 2>/dev/null; then \
		rm -f $(PY_ENV)/requirements.sha256; \
		$(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) install -e $2$(PY_SETUP_EXTRAS) &&\
		sha256sum requirements*.txt > $(PY_ENV)/requirements.sha256 ;\
	else \
		echo "PYENV     $2 already installed"; \
	fi

# Uninstall the package.  Since pip does not uninstall the no longer needed
# depencies (something like autoremove) the depencies remain.

# $2 package name to uninstall, this uses pip from the OS.
quiet_cmd_pyuninstall = UNINSTALL $2
      cmd_pyuninstall = $(PIP) $(PIP_VERBOSE) uninstall --yes $2

# $2 path to folder with setup.py, this uses pip from pyenv (not OS!)
quiet_cmd_pyenvuninstall = PYENV     uninstall   $2
      cmd_pyenvuninstall = $(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) uninstall --yes $2

# $2 path to folder where virtualenv take place
quiet_cmd_virtualenv  = PYENV     usage: $ source ./$@/bin/activate
      cmd_virtualenv  = \
	if [ -d "./$(PY_ENV)" -a -x "./$(PY_ENV_BIN)/python" ]; then \
		echo "PYENV     using virtualenv from $2"; \
	else \
		$(PYTHON) -m venv $(VTENV_OPTS) $2; \
		$(PY_ENV_BIN)/python -m pip install $(PIP_VERBOSE) -U pip wheel setuptools; \
		$(PY_ENV_BIN)/python -m pip install $(PIP_VERBOSE) -r requirements.txt; \
	fi

# $2 path to lint
quiet_cmd_pylint      = LINT      $@
      cmd_pylint      = $(PY_ENV_BIN)/python -m pylint -j 0 --rcfile $(PYLINT_RC) $2

quiet_cmd_pytest      = TEST      $@
      cmd_pytest      = $(PY_ENV_BIN)/python -m tox -vv

# setuptools, pip, easy_install its a mess full of cracks, a documentation hell
# and broken by design ... all sucks, I really, really hate all this ... aaargh!
#
# About python packaging see `Python Packaging Authority`_.  Most of the names
# here are mapped to ``setup(<name1>=..., <name2>=...)`` arguments in
# ``setup.py``.  See `Packaging and distributing projects`_ about ``setup(...)``
# arguments. If this is all new for you, start with `PyPI Quick and Dirty`_.
#
# Further read:
#
# - pythonwheels_
# - setuptools_
# - packaging_
# - sdist_
# - installing_
#
# .. _`Python Packaging Authority`: https://www.pypa.io
# .. _`Packaging and distributing projects`: https://packaging.python.org/guides/distributing-packages-using-setuptools/
# .. _`PyPI Quick and Dirty`: https://hynek.me/articles/sharing-your-labor-of-love-pypi-quick-and-dirty/
# .. _pythonwheels: https://pythonwheels.com/
# .. _setuptools: https://setuptools.readthedocs.io/en/latest/setuptools.html
# .. _packaging: https://packaging.python.org/guides/distributing-packages-using-setuptools/#packaging-and-distributing-projects
# .. _sdist: https://packaging.python.org/guides/distributing-packages-using-setuptools/#source-distributions
# .. _bdist_wheel: https://packaging.python.org/guides/distributing-packages-using-setuptools/#pure-python-wheels
# .. _installing: https://packaging.python.org/tutorials/installing-packages/
#
quiet_cmd_pybuild     = BUILD     $@
      cmd_pybuild     = $(PY_ENV_BIN)/python setup.py \
			sdist -d $(PYDIST)  \
			bdist_wheel --bdist-dir $(PYBUILD) -d $(PYDIST)

quiet_cmd_pyclean     = CLEAN     $@
# remove 'build' folder since bdist_wheel does not care the --bdist-dir
      cmd_pyclean     = \
	rm -rf $(PYDIST) $(PYBUILD) $(PY_ENV) ./.tox *.egg-info     ;\
	find . -name '*.pyc' -exec rm -f {} +      ;\
	find . -name '*.pyo' -exec rm -f {} +      ;\
	find . -name __pycache__ -exec rm -rf {} +

# ------------------------------------------------------------------------------
# targets
# ------------------------------------------------------------------------------

# for installation use the pip from the OS!
PHONY += pyinstall
pyinstall: pip-exe
	$(call cmd,pyinstall,.)

PHONY += pyuninstall
pyuninstall: pip-exe
	$(call cmd,pyuninstall,$(PYOBJECTS))

# for installation use the pip from PY_ENV (not the OS)!
PHONY += pyenvinstall
pyenvinstall: $(PY_ENV)
	$(call cmd,pyenvinstall,.)

PHONY += pyenvuninstall
pyenvuninstall: $(PY_ENV)
	$(call cmd,pyenvuninstall,$(PYOBJECTS))

PHONY += pyclean
pyclean:
	$(call cmd,pyclean)

# to build *local* environment, python from the OS is needed!
pyenv: $(PY_ENV)
$(PY_ENV): python-exe
	$(call cmd,virtualenv,$(PY_ENV))

PHONY += pylint-exe
pylint-exe: $(PY_ENV)
	@$(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) install pylint

PHONY += pylint
pylint: pylint-exe
	$(call cmd,pylint,$(PYOBJECTS))

PHONY += pybuild
pybuild: $(PY_ENV)
	$(call cmd,pybuild)

PHONY += pytest
pytest: $(PY_ENV)
	$(call cmd,pytest)

PHONY += pydebug
# set breakpoint with:
#    DEBUG()
# e.g. to run tests in debug mode in emacs use:
#   'M-x pdb' ... 'make pydebug'
pydebug: $(PY_ENV)
	DEBUG=$(DEBUG) $(PY_ENV_BIN)/pytest $(DEBUG) -v $(TEST_FOLDER)/$(TEST)

# runs python interpreter from ./local/py<N>/bin/python
pyenv-python: pyenvinstall
	$(PY_ENV_BIN)/python -i

# With 'dependency_links=' setuptools supports dependencies on packages hosted
# on other reposetories then PyPi, see "Packages Not On PyPI" [1].  The big
# drawback is, due to security reasons (I don't know where the security gate on
# PyPi is), this feature is not supported by pip [2]. Thats why an upload to
# PyPi is required and since uploads via setuptools is not recommended, we have
# to imstall / use twine ... its really a mess.
#
# [1] https://python-packaging.readthedocs.io/en/latest/dependencies.html#packages-not-on-pypi
# [2] https://github.com/pypa/pip/pull/1519

# https://github.com/pypa/twine
PHONY += upload-pypi upload-pypi-test
upload-pypi: pyclean pyenvinstall pybuild
	@$(PY_ENV_BIN)/twine upload $(PYDIST)/*

upload-pypi-test: pyclean pyenvinstall pybuild
	@$(PY_ENV_BIN)/twine upload -r testpypi $(PYDIST)/*
.PHONY: $(PHONY)