forgefed/deprecated/0. rdflib/app.py

# -*- coding: utf-8 -*-

"""
    (c) 2020 - Copyright ...
    
    Authors:
        zPlus <zplus@peers.community>
"""

import blinker
import celery
import flask
import functools
import json
import os
import logging
import pagure
import pagure.config
import rdflib
import requests
import requests_http_signature
import urllib

from . import activitypub
from . import graph
from . import model
from . import settings
from . import tasks

log = logging.getLogger(__name__)
log.info('Initializing forgefed plugin...')

# Pagure uses several ways to send out notification about events that have
# occurred within the instance. We use "blinker" and subscribe to the pagure
# signal. See pagure.lib.notify for more info.
blinker.signal('pagure').connect(
    lambda sender, topic, message: tasks.notification.handle_pagure_signal.delay(topic, message),
    weak=False)

# This Flask Blueprint will be imported by Pagure
APP = flask.Blueprint('forgefed_ns', __name__, url_prefix='/',
                      template_folder='templates')

# TODO load Blueprint configuration from file
APP.config = {}

# The app URL defined in the Pagure configuration, eg. "https://example.org/"
# We need this for generating IDs.
APP_URL = pagure.config.config['APP_URL'].rstrip('/')
assert APP_URL and len(APP_URL) > 0, 'APP_URL missing from Pagure configuration.'

def get_actor_from_route(path, method='GET'):
    """
    This is a helper function to return the model.Actor that matches a
    given @route path. The motivation for this is that we can avoid writing
    the same routes for every Actor. For example we can have a single view
    that matches INBOXes instead of having an /inbox view for every Actor.
    
    :param path: The path to match Pagure routes against.
    :param method: The HTTP method to match.
    :return: An instance of model.Actor.
    
    TODO Replace this function with model.from_uri which is a more general function
         that can be used without flask's create_url_adapter().
    """
    
    # This works as a reverse Flask url_for function. It matches a string
    # against the Werkzeug Routes (defined by the Pagure app), and returns
    # the name of the matched endpoint with its arguments. This is more
    # convoluted than I like, but Flask doesn't have any function like
    # url_for so we need to bind/match using Werkzeug directly.
    # https://github.com/pallets/flask/issues/3619
    # https://flask.palletsprojects.com/en/1.1.x/api/#flask.Flask.url_map
    # https://flask.palletsprojects.com/en/1.1.x/api/#flask.Flask.create_url_adapter
    # https://werkzeug.palletsprojects.com/en/1.0.x/routing/#quickstart
    try:
        # https://flask.palletsprojects.com/en/1.1.x/api/#flask.current_app
        #(endpoint, arguments) = flask.current_app.url_map.bind('').match(path, method)
        (endpoint, arguments) = flask.current_app.create_url_adapter(flask.request).match(path, method)
    except Exception:
        return None
    
    if endpoint == 'ui_ns.view_user':
        
        return flask.g.session \
                    .query(model.Person) \
                    .filter(model.Person.user == arguments['username']) \
                    .one_or_none()
    
    if endpoint == 'ui_ns.view_repo_git':
        actor = flask.g.session \
                       .query(model.Repository) \
                       .filter(model.Repository.name == arguments['repo'])
        
        if 'username' in arguments:
            actor = actor.filter(model.Repository.is_fork == True) \
                         .filter(model.Repository.user.has(model.Person.user == arguments['username']))
        
        else:
            actor = actor.filter(model.Repository.is_fork == False)
        
        if 'namespace' in arguments:
            actor = actor.filter(model.Repository.namespace == arguments['namespace'])
        
        return actor.one_or_none()
    
    if endpoint in [ 'ui_ns.view_issue', 'ui_ns.view_repo' ]:
        actor = flask.g.session \
                        .query(model.Projects) \
                        .filter(model.Projects.name == arguments['repo'])
        
        if 'username' in arguments:
            actor = actor.filter(model.Projects.is_fork == True) \
                            .filter(model.Projects.user.has(model.Person.user == arguments['username']))
        
        else:
            actor = actor.filter(model.Projects.is_fork == False)
        
        if 'namespace' in arguments:
            actor = actor.filter(model.Projects.namespace == arguments['namespace'])
        
        return actor.one_or_none()

def requires_login(func):
    """
    A decorator for routes to check user login.
    """
    
    @functools.wraps(func)
    def decorator(*args, **kwargs):
        if not flask.g.authenticated:
            return ("", 401) # Unauthorized
        
        return func(*args, **kwargs)
    
    return decorator

@APP.after_request
def add_header(response):
    """
    Automatically set Content-Type header to all the Blueprint responses.
    # TODO Untangle this!
    """
    
    # Return default headers
    if flask.request.path.startswith('/federation'):
        return response
    
    if flask.request.path.startswith('/.well-known/host-meta'):
        response.headers['Content-Type'] = 'application/xrd+xml; charset=utf-8'
    elif flask.request.path.startswith('/.well-known/webfinger'):
        response.headers['Content-Type'] = 'application/jrd+json; charset=utf-8'
    else:
        response.headers['Content-Type'] = activitypub.default_header
    
    return response

@APP.record
def override_pagure_routes(setup_state):
    """
    We reuse Pagure routes in order to return ActivityPub objects in response
    to "Accept: application/ld+json" request headers. The "record" decorator
    registers a callback function that is called during initialization of the
    Blueprint by Flask. While Flask offers the app context during requests
    handling, the same context is not available during initialization.
    Therefore we need this callback which is called during Flask initialization
    in order to get the app context that we need to replace the Pagure views.
    
    See https://flask.palletsprojects.com/en/1.1.x/api/#flask.Blueprint.record
    for more info about the record() function/decorator.
    
    NOTE - If Flask accepted route dispatching based on headers value, we could
           just use something like @APP.route(accept='application/activity+json').
           But it does not, so we need to override the Pagure views.
         - ActivityPub requires some routes to exist for Actors, for example
           "inbox" and "followers". However, Pagure does not have suitable
           routes that we can reuse for this purpose. For this reason, those
           routes are defined directly on the Blueprint.
    """
    
    # Reference to the main Flask app created by Pagure and to which Blueprints
    # will be attached.
    pagure_app = setup_state.app
    
    # We add our templates folder to the app's jinja path, such that we can
    # override pagure templates.
    pagure_app.jinja_loader \
              .searchpath \
              .insert(0, os.path.join(os.path.abspath(os.path.dirname(__file__)), 'templates'))
    
    # Create a symlink to pagure templates folder. This is only used when
    # "extending" templates using {% extends "master.html" %}, because extending
    # a template with the same name will trigger an infinite recursion.
    pagure_path_symlink = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'templates/__pagure__')
    if not os.path.islink(pagure_path_symlink):
        os.symlink(os.path.join(settings.PAGURE_PATH, 'pagure/templates/'), pagure_path_symlink)
    
    @pagure_app.before_request
    def start_database_session():
        """
        At the beginning of every request we get a new connection to the
        forgefed graph store. Please note that this function is executed
        before *every* request, for every route, including the ones
        defined in the Blueprint.
        """
        
        flask.g.forgefed_graph = graph.Graph()
    
    """
    @pagure_app.after_request
    def do_something(response):
        return response
    """
    
    @pagure_app.teardown_request
    def free_database_session(exception=None):
        """
        Close and remove the database session that was initiated in
        @before_requrest. This instruction should be optional since the object
        should be automatically garbage-collected when the request is destroyed.
        """
        
        flask.g.forgefed_graph.commit()
        flask.g.forgefed_graph.disconnect()
    
    def pagure_route(endpoint):
        """
        This function returns a decorator whose job is to replace a Pagure
        view with another one that will check the HTTP "Accept" header. If the
        HTTP request is a ActivityPub one, we respond appropriately. Otherwise
        we just return control to the Pagure view.
        
        Additional documentation useful for this decorator: https://flask.palletsprojects.com/en/1.1.x/api/#flask.Flask.view_functions
        """
        
        def decorator(forgefed_view):
            # The Flask object "pagure_app.view_functions" contains all the
            # views defined by Pagure.
            pagure_view = pagure_app.view_functions[endpoint]
            
            # https://docs.python.org/3.9/library/functools.html#functools.wraps
            @functools.wraps(forgefed_view)
            def wrapper(*args, **kwargs):
                if 'Accept' in flask.request.headers:
                    # HTTP headers can contain multiple values separated by a comma
                    request_headers = [
                        value.strip(' ')
                        for value
                        in  flask.request.headers.get('Accept').split(',') ]
                    
                    if any(header in activitypub.headers for header in request_headers):
                        response = flask.make_response(forgefed_view(*args, **kwargs))
                        response.headers['Content-Type'] = activitypub.default_header
                        return response
                
                # If it's not an ActivityPub request, just fall through to the
                # Pagure default view.
                return pagure_view(*args, **kwargs)

            # Replace the pagure view with our own
            pagure_app.view_functions[endpoint] = wrapper
            return wrapper
        
        return decorator
    
    ###########################################################################
    # Person
    ###########################################################################
    
    @pagure_route('ui_ns.view_user')
    def person(*args, **kwargs):
        """
        Return a Person object from the Pagure user page.
        """
        
        actor = get_actor_from_route(flask.request.path, 'GET')
        
        if not actor:
            return ({}, 404)
        
        return actor.jsonld
    
    ###########################################################################
    # Repository
    # 
    # - Pagure uses 4 kind of URLs for repositories.
    # - The Pagure app defines a Flask before_request() function that does a
    #   lot of things, among which to check if there is a "repo" variable in
    #   the URL and in turn setup some context for the request. This happens
    #   for *every* request, regardless if it's about a repository or not. The
    #   rationale is that they rather do it this way than using a separate
    #   decorator for all the repositories views, because almost all requests
    #   are about repositories anyway. before_request() will then automatically
    #   return 404 if a repository does not exist, so we don't need to check
    #   that here, unlike what we did in person(*args, **kwargs), because these
    #   views will never be executed.
    # - The Pagure before_request() retrieves the repository using the function
    #   pagure.lib.query.get_authorized_project() which checks for
    #   authorization and, if the repo is private, it returns 404. So there
    #   should be no need to check for authorization here.
    ###########################################################################
    
    @pagure_route('ui_ns.view_repo_git')
    def repository(*args, **kwargs):
        """
        Return a Repository object from the project page.
        """
        
        actor = get_actor_from_route(flask.request.path, 'GET')
        
        if not actor:
            return ({}, 404)
        
        return actor.jsonld
    
    ###########################################################################
    # Project
    ###########################################################################
    
    @pagure_route('ui_ns.view_repo')
    def project(*args, **kwargs):
        """
        Return a Project object from the project page.
        """
        
        repository = get_actor_from_route(flask.request.path, 'GET')
        
        if not repository:
            return ({}, 404)
        
        project = flask.g.session \
                    .query(model.Projects) \
                    .filter(model.Projects.id == repository.id) \
                    .one_or_none()
        
        return project.jsonld
    
    ###########################################################################
    # Tickets
    ###########################################################################
    
    @pagure_route('ui_ns.view_issue')
    def ticket(*args, **kwargs):
        username   = kwargs.get('username')
        namespace  = kwargs.get('namespace')
        name       = kwargs.get('repo')
        issue_id   = kwargs.get('issueid')
        
        repository = get_actor_from_route(flask.request.path, 'GET')
        
        if not repository:
            return ({}, 404)
        
        ticket = flask.g.session \
            .query(model.Ticket) \
            .filter(model.Ticket.id == issue_id,
                    model.Ticket.project_id == repository.id) \
            .one_or_none()
        
        if not ticket:
            return ({}, 404)
        
        return ticket.jsonld
    
    log.info('forgefed plugin registered by Flask.')


###############################################################################
# WebFinger
# 
# This is primarily used to support other ActivityPub software such as
# Mastodon that relies on webfinger as a discovery protocol because users
# use @username@domain when mentioning other users.
# https://docs.joinmastodon.org/spec/webfinger/
###############################################################################

@APP.route('/.well-known/host-meta', methods=['GET'])
def host_meta():
    return flask.render_template('host-meta.xml', APP_URL=APP_URL)

@APP.route('/.well-known/webfinger/<path:uri>', methods=['GET'])
def webfinger_resource(uri):
    """
    Return the webfinger info for account "uri".
    
    :param uri: The "acct:userpard@host" to search.
    """
    
    # Only support acct: resources
    # Do we need to support other schemes? WebFinger is neutral regarding the
    # scheme of URI: it could be "acct", "http", "https", "mailto", or some
    # other scheme, but other AcitivityPub instances such as Mastodon only
    # use "acct".
    if not uri.startswith('acct:'):
        return ({}, 404)
    
    # The "acct" scheme is defined in the spec as
    #    "acct" ":" userpart "@" host
    # "host" is the domain where the account is hosted
    # "userpart" contains the localinfo used by the host to retrieve the account
    userpart, host = uri[5:].rsplit('@', 1)
    
    # Now we find the actual actor's URI.
    # The "userpart" is basically the "preferredUsername" property defined in
    # model.py
    if userpart.startswith('project/'):
        actor_uri = '{}/{}'.format(APP_URL, userpart[8:])
    else:
        actor_uri = '{}/user/{}'.format(APP_URL, userpart)
    
    return flask.render_template('webfinger.json', subject=uri, actor_uri=actor_uri)


###############################################################################
# Routes used to interact with remote objects of the federation, when we
# cannot reuse another pagure route.
###############################################################################

@APP.route('/federation', methods=['GET'])
@requires_login
def federation():
    user = flask.g.fas_user
    
    return flask.render_template(
        'federation/feed.html'
    )

@APP.route('/federation/followers', methods=['GET'])
@requires_login
def federation_followers():
    user = flask.g.fas_user
    
    return flask.render_template(
        'federation/followers.html'
    )

@APP.route('/federation/following', methods=['GET'])
@requires_login
def federation_following():
    person = flask.g.session \
                    .query(model.Person) \
                    .filter(model.Person.id == flask.g.fas_user.id) \
                    .one_or_none()
    
    if not person:
        return flask.redirect(url_for('forgefed_ns.federation'))
    
    # Retrieve "following" list from the local graph
    collection = flask.g.forgefed_graph.get_collection(person.following_uri)
    collection_page = flask.g.forgefed_graph.get_collection(collection['current'])
    following = []
    for actor in collection_page['orderedItems']:
        following.append(flask.g.forgefed_graph.get_json_node(actor))
    
    return flask.render_template('federation/following.html', following=following)

@APP.route('/federation/search', methods=['GET', 'POST'])
@requires_login
def federation_search():
    uri = flask.request.args.get('uri')
    search_result = None
    
    # Search for an object
    if uri:
        search_result = activitypub.fetch(uri)
    
    return flask.render_template(
        'federation/search.html',
        uri=uri,
        search_result=search_result)

@APP.route('/federation/follow', methods=['GET'])
@requires_login
def federation_follow():
    remote_actor_uri = flask.request.args.get('actor_uri')
    remote_actor = activitypub.fetch(remote_actor_uri)
    
    if not remote_actor:
        return 'Could not fetch remote actor.'
    
    # The user that clicked the "Follow" button
    person = flask.g.session \
                    .query(model.Person) \
                    .filter(model.Person.id == flask.g.fas_user.id) \
                    .one_or_none()
    
    if not remote_actor_uri or not person:
        return flask.redirect(url_for('forgefed_ns.federation'))
    
    # Send the Activity
    person.follow(object_uri=remote_actor_uri, to=remote_actor_uri)
    
    # We also save a copy of the remote actor, for easy listing in the
    # "following/followers" collections
    flask.g.forgefed_graph.parse(data=json.dumps(remote_actor), format='json-ld')
    
    # Add the remote actor to the "following" collection
    if not flask.g.forgefed_graph.collection_contains(person.following_uri, remote_actor_uri):
        flask.g.forgefed_graph.add_collection_item(person.following_uri, remote_actor_uri)
    
    return flask.redirect(flask.url_for('forgefed_ns.federation_following'))

@APP.route('/federation/submit_ticket', methods=['GET'])
@requires_login
def federation_submit_ticket():
    # The URL of the remote tracker (actor)
    actor_uri = flask.request.args.get('actor_uri')
    
    if not actor_uri:
        return flask.redirect(flask.url_for('ui_ns.index'))
    
    # Retrieve the remote actor
    actor = activitypub.fetch(actor_uri)
    
    # Create a local user for the project because in Pagure a project must
    # have a user ID
    user = model.test_or_set_remote_actor(
        pagure_db=flask.g.session,
        forgefed_graph=flask.g.forgefed_graph,
        uri=actor_uri)
    
    project = flask.g.session \
                   .query(model.Projects) \
                   .filter(model.Projects.name == user.username) \
                   .one_or_none()
    
    # Check if we already have a local tracker for contributing to the remote
    # tracker. If not, created it.
    if not project:
        # Create the project in the Pagure database.
        # This function will create the item in the database, and then will
        # start a new async task to create the actual .git folder.
        pagure.lib.query.new_project(
            flask.g.session,
            user=user.username,
            name=user.username,
            blacklist=[],
            allowed_prefix=[],
            repospanner_region=None,
            description="Remote",
            url=actor_uri,
            avatar_email=None,
            parent_id=None,
            add_readme=False,
            mirrored_from=None,
            userobj=user,
            prevent_40_chars=False,
            namespace=None,
            user_ns=False,
            ignore_existing_repo=False,
            private=False,
        )
        
        # The tracker that we've just created is only used to interact with a
        # remote one. It's not a "real" local tracker by a local user, so we
        # create a owl:sameAs relation in the graph.
        tracker_url = '{}/{}/issues'.format(APP_URL, user.username)
        flask.g.forgefed_graph.set((rdflib.URIRef(tracker_url),
                                    rdflib.OWL.sameAs,
                                    rdflib.URIRef(actor_uri)))
    
    # Redirect to the "new issue" page of the local tracker, where the user can
    # create a new issue for the remote tracker.
    return flask.redirect(flask.url_for('ui_ns.new_issue', repo=user.username))

@APP.route('/federation/ticket/<issue_uid>/comments', methods=['GET'])
def federation_ticket_comments(issue_uid):
    """
    Return the Collection containing a ticket's comments.
    This route exists because pagure does not have any route defined for
    comments. The path of a comment in pagure is "/<repo>/issue/<issueid>#comment-<commend_id>"
    but the trailing part of the URL after the # symbol is not sent to the server.
    
    :param issue_uid: The unique ID defined by pagure during ticket creation. This
        is used instead of the default key which is (project_id, issue_id).
    
    :param page: The page of the collection to show.
    """
    
    collection_uri = APP_URL + flask.request.path
    flask.g.forgefed_graph.test_or_set_ordered_collection(collection_uri)
    return flask.g.forgefed_graph.get_collection(collection_uri)

@APP.route('/federation/ticket/<issue_uid>/comments/<int:page>', methods=['GET'])
def federation_ticket_comments_page(issue_uid):
    """
    Return the CollectionPage containing a ticket's comments.
    
    :param issue_uid: The unique ID defined by pagure during ticket creation. This
        is used instead of the default key which is (project_id, issue_id).
    
    :param page: The page of the collection to show.
    """
    
    collection_uri = APP_URL + flask.request.path
    return flask.g.forgefed_graph.get_collection(collection_uri)

@APP.route('/federation/ticket_comment/<comment_id>', methods=['GET'])
def federation_ticket_comment(comment_id):
    """
    Return the JSONLD of a ticket comment.
    
    :param comment_id: The comment ID defined by pagure during comment creation.
        This is the default primary key of the comment and is unique across all
        local issues.
    """
    
    return flask.g.session \
                  .query(model.TicketComment) \
                  .filter(model.TicketComment.id == comment_id) \
                  .one_or_none() \
                  .jsonld



###############################################################################
# Actor
###############################################################################

@APP.route('/<path:actor>/key.pub', methods=['GET'])
def actor_key(actor):
    """
    This object represents the public GPG key used to sign HTTP requests.
    """
    
    actor_path = '/' + actor
    actor_uri  = APP_URL + '/' + actor
    actor      = get_actor_from_route(actor_path, 'GET')
    
    if not actor:
        return ({}, 404)
    
    # Test if this key exists, otherwise create a new one
    flask.g.forgefed_graph.test_or_set_key(actor.local_uri, actor.publickey_uri)
    
    # Retrieve the graph node of the key
    node = flask.g.forgefed_graph \
                    .subgraph((rdflib.URIRef(actor.publickey_uri), None, None))
    
    # Remove the private key before displaying
    node.remove((None, graph.SEC.privateKeyPem, None))
    
    # TODO do not replace CryptographicKey with Key. This is only here because
    #      vervis is giving an error
    key = json.loads(node.serialize(context=activitypub.cached_jsonld_context,
                                     format='json-ld'))
    key['type'] = 'Key'
    return key

@APP.route('/<path:actor>/inbox', methods=['GET'])
def actor_inbox(actor):
    """
    Returns an Actor's INBOX.
    This should only be called by the Actors who want to read their own INBOX.
    """
    
    return ({}, 501) # 501 Not Implemented. TODO implement C2S.

@APP.route('/<path:actor>/outbox', methods=['GET'])
def actor_outbox(actor, page=None):
    """
    Returns an Actor's OUTBOX.
    This should only be called by the Actors who want to read their own OUTBOX.
    """
    
    # TODO Show only "Public" OUTBOX
    # https://www.w3.org/TR/activitypub/#public-addressing
    
    return ({}, 501) # 501 Not Implemented. TODO implement C2S.

@APP.route('/<path:actor>/inbox', methods=['POST'])
def actor_receive(actor):
    """
    Somebody is sending a message to an actor's INBOX.
    """
    
    # TODO
    # Verify incoming request signature. Check if the HTTP POST request is
    # signed correctly.
    """
    def key_resolver(key_id, algorithm):
        return remote actor public key
    
    try: 
        requests_http_signature.HTTPSignatureAuth.verify(
            flask.request,
            key_resolver=key_resolver)
    except Exception:
        return ({}, 401)
    """
    
    actor_path = '/' + actor
    actor_uri  = APP_URL + '/' + actor
    actor      = get_actor_from_route(actor_path, 'GET')
    
    if not actor:
        return ({}, 404)
    
    # Retrieve the ActivityPub Activity from the HTTP request body. The
    # Activity is expected to be a JSON object.
    # https://flask.palletsprojects.com/en/1.1.x/api/#flask.Request.get_json
    activity = flask.request.get_json()
    
    # Schedule a task to process the incoming activity asynchronously
    tasks.activity.validate.delay(actor.__repr__(), activity)
    
    return ({}, 202) # 202 Accepted

@APP.route('/<path:actor>/outbox', methods=['POST'])
def actor_send(actor):
    """
    An Actor is trying to POST an Activity to its OUTBOX.
    This should only be called by an Actor's client that want to send out a new
    Activity.
    """
    
    return ({}, 501) # 501 Not Implemented. TODO implement C2S.

@APP.route('/<path:actor>/followers', methods=['GET'])
def actor_followers(actor):
    """
    Show the followers of an actor.
    """
    
    actor_path     = '/' + actor
    actor_uri      = APP_URL + '/' + actor
    collection_uri = APP_URL + flask.request.path
    actor          = get_actor_from_route(actor_path, 'GET')
    
    if not actor:
        return ({}, 404)
    
    flask.g.forgefed_graph.test_or_set_ordered_collection(collection_uri)
    
    return flask.g.forgefed_graph.get_collection(collection_uri)

@APP.route('/<path:actor>/followers/<int:page>', methods=['GET'])
def actor_followers_page(actor, page):
    """
    Show the followers of an actor.
    """
    
    collection_uri = APP_URL + flask.request.path
    return flask.g.forgefed_graph.get_collection(collection_uri)

@APP.route('/<path:actor>/following', methods=['GET'])
def actor_following(actor):
    """
    Show the actors that an actor is following.
    """
    
    actor_path     = '/' + actor
    actor_uri      = APP_URL + '/' + actor
    collection_uri = APP_URL + flask.request.path
    actor          = get_actor_from_route(actor_path, 'GET')
    
    if not actor:
        return ({}, 404)
    
    flask.g.forgefed_graph.test_or_set_ordered_collection(collection_uri)
    
    if page:
        collection_uri += '/' + str(page)
    
    return flask.g.forgefed_graph.get_collection(collection_uri)

@APP.route('/<path:actor>/following/<int:page>', methods=['GET'])
def actor_following_page(actor, page):
    """
    Show the actors that an actor is following.
    """
    
    collection_uri = APP_URL + flask.request.path
    return flask.g.forgefed_graph.get_collection(collection_uri)

log.info('forgefed plugin initialized.')