forgefed/deprecated/5. extending pagure database model/model.py

# -*- coding: utf-8 -*-

"""
    (c) 2020 - Copyright ...
    
    Authors:
        zPlus <zplus@peers.community>
    
    Notes:
        Useful documentation for SQLAlchemy ORM:
          https://docs.sqlalchemy.org/en/13/orm/tutorial.html
          https://docs.sqlalchemy.org/en/13/orm/basic_relationships.html
        
        When defining new classes it's important to prefix them in order to avoid
        conflicts with Pagure classes.
        Motivation: Pagure (see pagure.lib.model) uses simple class names when
        it creates relationships for the SQLAlchemy ORM. For example the class Issue
        has the property:
            project = relation(
                "Project",
                foreign_keys=[project_id],
                remote_side=[Project.id],
                backref=backref("issues", cascade="delete, delete-orphan"),
                single_parent=True)
        If we register a new class Project with SQLAlchemy, the single string
        "Project" in the relation is not enough to disambiguate which class
        SQLAlchemy should use. So, in order to preserve the Pagure source code,
        we add a prefix to our plugin classes. Other info available at:
        https://docs.sqlalchemy.org/en/13/orm/extensions/declarative/relationships.html#evaluation-of-relationship-arguments
"""

import copy
import datetime
import logging
import pagure.config
import pagure.lib.model
import pagure.lib.query
import urllib

from Crypto.PublicKey import RSA
from pagure.lib.model_base  import BASE
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, \
                       LargeBinary, UnicodeText, func
from sqlalchemy import PrimaryKeyConstraint
from sqlalchemy import create_engine
from sqlalchemy.orm import relationship
from sqlalchemy.orm.session import Session

from . import APP_URL
from . import activitypub
from . import settings
from . import tasks

log = logging.getLogger(__name__)

def from_uri(pagure_db, uri):
    """
    In ActivityPub, objects' ID are URI and occasionally we need to retrieve an
    object from the (pagure) database given only its URI. If pagure were using
    a graph as a database, this would be trivial (just query "DESCRIBE <uri>").
    Also, the plugin does not mirror the entire pagure database into a graph;
    objects' documents (the JSON-LD that is returned when GETting) are created
    "on the fly" for every request. We could add the pagure ID to the JSON-LD
    object, but it would be very inelegant and hacky. We could add URIs to the
    pagure database, but it would be very brittle and would also require to
    change the pagure database schema. So we need a way, given only a URI, to
    reverse it and get the object in the database. This function does that.
    
    NOTES: This function is for *local* users only. It return an SQLAlchemy
        object given an ActivityPub URI.
    
    :param pagure_db: A session context of the pagure database.
    
    :param uri: The URI to reverse.
    """
    
    log.debug('from_uri: {}'.format(uri))
    
    # Fetch the JSON-LD document of the object.
    # Remember: we do not store the pagure objects in the graph, so we must
    # HTTP-GET the URI.
    object = activitypub.fetch(uri)
    
    uri = urllib.parse.urlsplit(uri)
    
    if object['type'] == 'Person':
        return pagure_db.query(Person) \
                        .filter(Person.user == object['preferredUsername']) \
                        .one_or_none()
    
    if object['type'] == 'Project':
        components = uri.path.lstrip('/').split('/')
        
        repo_username  = None
        repo_namespace = None
        repo_name      = None
        
        if len(components) == 1:
            repo_name      = components[0]
        if len(components) == 2:
            repo_namespace = components[0]
            repo_name      = components[1]
        if len(components) == 3:
            repo_username  = components[1]
            repo_name      = components[2]
        if len(components) == 4:
            repo_username  = components[1]
            repo_namespace = components[2]
            repo_name      = components[3]
        
        project = pagure_db.query(Project).filter(Project.name == repo_name)
        
        if repo_username:
            project = project.filter(Project.is_fork == True) \
                             .filter(Project.user.has(Person.user == repo_username))
        else:
            project = project.filter(Project.is_fork == False)
        
        if repo_namespace:
            project = project.filter(Project.namespace == repo_namespace)
        
        return project.one_or_none()
    
    if object['type'] == 'Ticket':
        # Ticket "context" contains a link to the Project URI
        project = from_uri(pagure_db, object['context'])
        
        # Ticket ID from the URI
        ticket_id = uri.path.rsplit('/', 1)[-1]
        
        return pagure_db.query(Ticket) \
                        .filter(Ticket.id == ticket_id,
                                Ticket.project_id == project.id) \
                        .one_or_none()
    
    if object['type'] == 'Note':
        if uri.path.startswith('/federation/ticket_comment/'):
            comment_id = uri.path.rsplit('/', 1)[-1]
            return pagure_db.query(TicketComment) \
                            .filter(TicketComment.id == comment_id) \
                            .one_or_none()
    
    # No object found
    return None

def from_local_uri(pagure_db, uri):
    # Check if the URI is an actual local object
    if not uri.startswith(APP_URL):
        return None
    
    return from_uri(pagure_db, uri)

def from_path(pagure_db, path):
    return from_local_uri(pagure_db, '{}{}'.format(APP_URL, path))

def from_remote_uri(pagure_db, uri):
    # Check if the URI is an actual remote object
    if uri.startswith(APP_URL):
        return None
    
    # Get the URI of the local object that maps the remote object
    same_as = pagure_db.query(SameAs) \
                      .filter(SameAs.remote_uri == uri) \
                      .one_or_none()
    
    if not same_as:
        return None
    
    return from_uri(pagure_db, same_as.local_uri)

def test_or_set_remote_actor(pagure_db, uri):
    """
    We want to use the existing pagure UI to collaborate with remote users, for
    example to hold a conversation on a Ticket. Problem is, the pagure database
    model is built around *local* users that have IDs and relations with other
    schemas. So, we create a local mock user that represents a remote one, and
    call it user@domain.
    
    :param pagure_db: A session to the pagure database.
    :param uri: The URI of the remote actor.
    """
    
    # Fetch JSONLD of the remote actor
    actor = activitypub.fetch(uri)
    webfinger = '{}@{}'.format(actor['preferredUsername'], urllib.parse.urlparse(uri).netloc)
    
    # This will create a new user in the Pagure database if it doesn't exist
    user = pagure.lib.query.set_up_user(
        session       = pagure_db,
        username      = webfinger.replace('/', '+'),
        fullname      = actor['name'],
        default_email = webfinger)
    
    # Return a Person object instead of the pagure User class
    person = pagure_db.query(Person) \
                      .filter(Person.id == user.id) \
                      .one_or_none()
    
    # Set a owl:sameAs link in the forgefed database, so that we know that this
    # person is only used to represent a remote user.
    pagure_db.merge(SameAs(
        local_uri = person.local_uri,
        remote_uri = uri))
    
    return person

def test_or_set_remote_comment(pagure_db, uri):
    """
    This is the same as test_or_set_remote_actor() but for comments.
    
    :param pagure_db: A session to the pagure database.
    :param uri: The URI of the remote Note.
    """
    
    # If the URI of the comment is a URI to the local instance, we don't
    # need to do anything because the pagure database already contains the
    # comment.
    if uri.startswith(APP_URL):
        return
    
    # Fetch JSONLD of the remote Note
    note = activitypub.fetch(uri)
    
    # Check if we already have a local object for this remote comment
    if pagure_db.query(
           pagure_db.query(SameAs).filter(SameAs.local_uri == uri).exists()
       ).scalar():
        
        log.debug('The note {} is already stored in the database. Will not create a new one.'.format(uri))
        return
    
    # Otherwise we create a local object in the pagure database for the remote Note...
    
    # The "context" property is used to specify if the Note is for a Ticket or a
    # MergeRequest
    if 'context' not in note:
        return
    
    # If the "context" of the Note (which can be a Ticker or a MergeRequest) is
    # a local URL, it means somebody has created a Note for an object in our
    # database. This is the case for example when a remote user is contributing
    # a comment to a local project.
    if note['context'].startswith(APP_URL):
        
        # Get the database object
        context = from_local_uri(pagure_db, note['context'])
        
        # This is a new Note for a local Ticket
        if isinstance(context, Ticket):
            author = test_or_set_remote_actor(pagure_db, note['attributedTo'])
            
            # Create the new comment to the ticket
            pagure.lib.query.add_issue_comment(
                session = pagure_db,
                issue = context,
                comment = note['content'],
                user = author.username)
        
        #if isinstance(context, MergeRequest):
            # TODO
        #    pass
    
    # If the "context" of the Note is not a local ticket or a local MR, this
    # note was created for a remote object. This is the case for example when a
    # user has sent a comment to a remote ticket, and we have received the
    # Activity because we are following that ticket.
    else:
        
        # Check if there is any local ticket or MR in the pagure database that
        # is used to track a remote object
        context = from_remote_uri(pagure_db, note['context'])
        
        # This is a new Note for a remote Ticket
        if isinstance(context, Ticket):
            author = test_or_set_remote_actor(pagure_db, note['attributedTo'])
            
            # Create the new comment to the ticket
            pagure.lib.query.add_issue_comment(
                session = pagure_db,
                issue = context,
                comment = note['content'],
                user = author.username)


################################################################################
# Extension of the Pagure model
################################################################################

class ActivityStreamObject:
    """
    An ActivityStrem Object. 
    """
    
    def _get_database_session(self):
        """
        Return the SQLAlchemy session associated with this instance.
        """
        
        return Session.object_session(self)
    
    @property
    def uri(self):
        """
        The URI of this object.
        """
        
        remote_uri = self.remote_uri
        local_uri  = self.local_uri
        
        if remote_uri: return remote_uri
        if local_uri:  return local_uri
        return None
    
    @property
    def remote_uri(self):
        """
        The URI of the remote object if this object is only a local placeholder
        for a remote object.
        """
        
        db = self._get_database_session()
        same_as = db.query(SameAs) \
                    .filter(SameAs.local_uri == self.local_uri) \
                    .one_or_none()
        
        return same_as.remote_uri if same_as else None
    
    @property
    def is_remote(self):
        """
        Return True if this object is a local copy of a remote object. An example
        of such object would be a comment. A local copy is created such that users
        on the local instance can use the Pagure UI to interact with the remotes.
        """
        
        return self.remote_uri != None

class Actor(ActivityStreamObject):
    """
    An ActivityStream Actor.
    """
    
    def __new__(cls, *args, **kwargs):
        if cls is Actor:
            raise TypeError('Cannot instantiate model.Actor class directly.')
        
        return ActivityStreamObject.__new__(cls, *args, **kwargs)
    
    @property
    def inbox_uri(self):
        return self.local_uri + '/inbox'
    
    @property
    def outbox_uri(self):
        return self.local_uri + '/outbox'
    
    @property
    def followers_uri(self):
        return self.local_uri + '/followers'
    
    @property
    def following_uri(self):
        return self.local_uri + '/following'
    
    @property
    def publickey_uri(self):
        return self.local_uri + '/key.pub'
    
    @property
    def jsonld(self):
        return {
            '@context':  activitypub.jsonld_context,
            'id':        self.local_uri,
            'inbox':     self.inbox_uri,
            'outbox':    self.outbox_uri,
            'followers': self.followers_uri,
            'following': self.following_uri,
            'publicKey': self.publickey_uri }

class Person(pagure.lib.model.User, Actor):
    """
    An ActivityStream Person.
    """
    
    @property
    def local_uri(self):
        return '{}/{}'.format(APP_URL, self.url_path)
    
    @property
    def jsonld(self):
        return {
            **super().jsonld,
            'type': 'Person',
            'name': self.fullname,
            'preferredUsername': self.username }

class ForgefedProject(pagure.lib.model.Project, Actor):
    """
    A ForgeFed Project.
    """
    
    @property
    def local_uri(self):
        return APP_URL + '/' + self.url_path
    
    @property
    def jsonld(self):
        return {
            **super().jsonld,
            'type': 'Project',
            'name': self.name,
            'preferredUsername': 'project/{}'.format(self.url_path) }

class Repository(pagure.lib.model.Project, Actor):
    """
    A ForgeFed Repository.
    """
    
    @property
    def local_uri(self):
        return APP_URL + '/' + self.url_path + '.git'
    
    @property
    def jsonld(self):
        return {
            **super().jsonld,
            'type': 'Repository',
            'name': self.name,
            'team': None }

class BugTracker(pagure.lib.model.Project, Actor):
    """
    A ForgeFed BugTracker.
    """
    
    @property
    def local_uri(self):
        return APP_URL + '/' + self.url_path + '/issues'
    
    @property
    def jsonld(self):
        return {
            **super().jsonld,
            'type': 'BugTracker' }

class Ticket(pagure.lib.model.Issue, ActivityStreamObject):
    @property
    def local_uri(self):
        return '{}/{}/issue/{}'.format(APP_URL, self.project.url_path, self.id)
    
    @property
    def jsonld(self):
        return {
          '@context': activitypub.jsonld_context,
          'id': self.local_uri,
          'type': 'Ticket',
          'context': '{}/{}'.format(APP_URL, self.project.url_path),
          'attributedTo': '{}/{}'.format(APP_URL, self.user.url_path),
          'summary': self.title,
          'content': self.content,
          'mediaType': 'text/plain',
          'source': {
              'content': self.content,
              'mediaType': 'text/markdown; variant=CommonMark'
          },
          'assignedTo': None,
          'isResolved': False
        }

class TicketComment(pagure.lib.model.IssueComment, ActivityStreamObject):
    @property
    def local_uri(self):
        return '{}/federation/ticket_comment/{}'.format(APP_URL, self.id)
    
    @property
    def jsonld(self):
        # Find the "context" of the comment (ie. the URI of the Ticket it belongs to)
        ticket = self.issue #copy.deepcopy(self.issue)
        ticket.__class__ = Ticket
        context = ticket.uri
        
        return {
            '@context': activitypub.jsonld_context,
            'id': self.local_uri,
            'type': 'Note',
            'context': context,
            'attributedTo': APP_URL + '/' + self.user.url_path,
            'inReplyTo': None, # Pagure does not use nested comments
            'mediaType': 'text/plain',
            'content': self.comment,
            'source': {
                'mediaType': 'text/markdown; variant=Commonmark',
                'content': self.comment
            },
            'published': activitypub.format_datetime(self.date_created)
        }

class MRComment(pagure.lib.model.IssueComment, ActivityStreamObject):
    """
    Merge Request Comment.
    """
    
    @property
    def local_uri(self):
        return '{}/{}/issue/{}'.format(APP_URL, self.project.url_path, self.id)
    
    @property
    def jsonld(self):
        return {
          '@context': activitypub.jsonld_context,
        }


################################################################################
# ForgeFed relations.
# These tables are created/added to the Pagure database. They are used for
# various tasks required by federation.
################################################################################

class GpgKey(BASE, ActivityStreamObject):
    """
    This class represents an Actor GPG key that is used to sign HTTP POST
    requests.
    """
    
    __tablename__ = 'forgefed_gpg_key'
    
    # The URI of the key
    uri = Column(UnicodeText, primary_key=True, nullable=False)
    
    # The actor who owns the key
    actor_uri = Column(UnicodeText, nullable=False)
    
    # The private part of the key
    private = Column(LargeBinary, nullable=False)
    
    # The public part of the key
    public = Column(LargeBinary, nullable=False)
    
    # When was this key created
    created = Column(DateTime, nullable=False,
                     default=lambda: datetime.datetime.now(datetime.timezone.utc))
    
    @property
    def jsonld(self):
        return {
            '@context':      activitypub.jsonld_context,
            'id':            self.uri,
            'type':          'CryptographicKey',
            'owner':         self.actor_uri,
            #'created':      None,
            #'expires':      None,
            #'revoked':      None,
            #'privateKeyPem': self.private.decode('UTF-8'), DO NOT DISPLAY PRIVATE KEY
            'publicKeyPem':  self.public.decode('UTF-8')
        }
    
    @staticmethod
    def new_key():
        """
        Returns a new private/public key pair.
        """
        
        key = RSA.generate(settings.HTTP_SIGNATURE_KEY_BITS)
        
        return {
            'private': key.export_key('PEM'),
            'public':  key.publickey().export_key('PEM')
        }
    
    @staticmethod
    def test_or_set(database, actor_uri, key_uri):
        """
        Test if an Actor already has a GPG key, otherwise automatically generate
        a new one.
        
        :param database: Since this function is static, we need a working session
        :param actor_uri: URI of the ActivityPub Actor
        :param key_uri: URI of the ActivityPub Key for the Actor
        """
        
        key = database.query(GpgKey) \
                      .filter(GpgKey.uri == key_uri) \
                      .one_or_none()
        
        if not key:
            key = GpgKey.new_key()
            
            database.add(GpgKey(uri       = key_uri,
                          actor_uri = actor_uri,
                          private   = key['private'],
                          public    = key['public']))
            
            database.commit()

class Collection(BASE, ActivityStreamObject):
    """
    This class represents a list of items for a Collection.
    """
    
    __tablename__ = 'forgefed_collection'
    __table_args__ = (
        PrimaryKeyConstraint('uri', 'item'),
    )
    
    # The URI of the Collection.
    uri = Column(UnicodeText, nullable=False)
    
    # The URI of an item in the Collection.
    item = Column(UnicodeText, nullable=False)
    
    # When was this item added to the collection
    added = Column(DateTime, nullable=False,
                   default=lambda: datetime.datetime.now(datetime.timezone.utc))

    @property
    def local_uri(self):
        return self.uri

class SameAs(BASE):
    """
    This class is used to map links between local URIs and remote URIs.
    The reason is that we need to create local objects in Pagure to
    represents remote objects such as comments, so we keep track of which
    local objects are only a local representation of remote ones.
    """
    
    __tablename__ = 'forgefed_sameas'
    
    local_uri = Column(UnicodeText, primary_key=True, nullable=False)
    remote_uri = Column(UnicodeText, nullable=False)

class Resource(BASE):
    """
    This is a relation used for saving copies of JSONLD documents. It stores a URI
    and its corresponding JSON document.
    It's used for storing Activities, as well as caching remote objects that we can
    query quickly without making a GET request (for example when displaying the list
    of Followers, we store the JSONLD document of the remote user).
    """
    
    __tablename__ = 'forgefed_resource'
    
    # The URI of the object.
    uri = Column(UnicodeText, primary_key=True, default=None)
    
    # The JSON-LD document of the object.
    document = Column(UnicodeText, nullable=False)
    
    @property
    def jsonld(self):
        return activitypub.Document(self.document)

class Feed(BASE):
    """
    This class is used to store feeds about federation events. These feeds are
    simply displayed to the user in their federation page.
    
    NOTE: This only exists because Pagure feed is strictly linked (with a foreign key)
          to existing objects of the database, for example comments ID. Using the
          Pagure feed would require messing with the Pagure schema, therefore I
          decided not to use it. This means there's a separate feed for federation
          events only.
    """
    
    __tablename__ = 'forgefed_feed'
    
    # This only exists because SQLAlchemy requires a primary key, but this
    # primary key is not used anywhere.
    id = Column(Integer, primary_key=True)
    
    # The URI of the local Actor this feed item belongs to.
    actor_uri = Column(UnicodeText, nullable=False)
    
    # The content of the feed.
    content = Column(UnicodeText, nullable=False)
    
    # When the feed item was created.
    created = Column(DateTime, nullable=False,
                     default=lambda: datetime.datetime.now(datetime.timezone.utc))




# Automatically create the database tables on startup. The tables are first
# checked for existence before any CREATE TABLE command is issued.
# https://docs.sqlalchemy.org/en/13/orm/tutorial.html#create-a-schema

db_url = pagure.config.config.get('DB_URL')
if db_url.startswith('postgres'):
    engine = create_engine(db_url, echo=True, client_encoding="utf8", pool_recycle=3600)
else:
    engine = create_engine(db_url, echo=True, pool_recycle=3600)

BASE.metadata.create_all(engine)