mcfi/mcfi.py

#!/usr/bin/env python3

__authors__ = [ 'zPlus <zplus@peers.community>' ]
__version__ = '0.0.0'
__license__ = 'AGPL-3.0-or-later'
###############################################################################

import bottle, json, requests
import configparser
from bottle import abort, get, post, request, response, route
import datetime
from string import Template
from Crypto.PublicKey import RSA
#from requests_http_signature import HTTPSignatureAuth, HTTPSignatureHeaderAuth
import base64, hashlib
import httpsig
from httpsig.requests_auth import HTTPSignatureAuth
from httpsig.verify import HeaderVerifier
from time import mktime
from wsgiref.handlers import format_date_time
from email.utils import formatdate
from urllib.parse import urlparse

# Load settings
import settings

# Load actors (from plain text file)
actors = configparser.ConfigParser()
actors.read('/var/lib/gitolite3/.gitolite/forgefed/actors')

# Fire up database
import database

# This is used to export the bottle object for the WSGI server
application = bottle.app()

# HTTP headers to use when making requests
"""
request_headers={ 'Accept':       'application/activity+json; charset=utf-8',
                  'Content-Type': 'application/activity+json; charset=utf-8'}
"""
request_headers={ 'Accept':       'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
                  'Content-Type': 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'}

jsonld_context = [
    'https://www.w3.org/ns/activitystreams',
    'https://w3id.org/security/v1',
    'https://forgefed.peers.community/ns' ]

class McfiSignatureAuth(HTTPSignatureAuth):
    """
    Extend HTTPSignatureAuth to replace HeaderSigner initialization, in
    order to include sign_header.
    
    See also: https://requests.kennethreitz.org//en/master/user/authentication/#new-forms-of-authentication
    """
    
    def __init__(self, key_id='', secret='', algorithm=None,
                    headers=None, sign_header='Authorization'):
        
        # Init parent class
        super().__init__(key_id, secret, algorithm, headers)
        
        self.required_headers = headers or []
        
        # Replace HeaderSigner initialization
        self.header_signer = httpsig.HeaderSigner(
            key_id=key_id, secret=secret, algorithm=algorithm,
            headers=headers, sign_header=sign_header)
    
    def __call__(self, r):
        # Python Requests does not add a Date header by default?
        # Date format: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Date
        r.headers['Date'] = formatdate(
            timeval=mktime(datetime.datetime.now().timetuple()),
            localtime=False,
            usegmt=True)
        
        # Add a "Digest:" header
        if 'digest' in self.required_headers and 'Digest' not in r.headers and r.body is not None:
            digest = hashlib.sha256(r.body).digest()
            r.headers['Digest'] = 'SHA-256=' + base64.b64encode(digest).decode()
        
        return super().__call__(r)

def _url(name, *anons, **parameters):
    """
    Return a URL string corresponding to a Bottle route, given its arguments.
    We use this function because it's shorter than calling application.router.build.
    """
    
    return application.router.build(name, *anons, **parameters)

def _get_base_url():
    return  (request.headers['X-Forwarded-Proto']
             if 'X-Forwarded-Proto' in request.headers
             else request.urlparts[0]) + '://' + request.headers['Host']

def _get_private_key(actor):
    # Get the local actor's private key used to sign the Activity
    key = database.get_private_key(actor)
    
    # Key doesn't exist?
    if not key:
        # Create one
        key = RSA.generate(2048).exportKey('PEM')
        
        # Save copy in the database
        database.set_private_key(actor, key)
    
    return key    

def _get_http_auth(actor):
    """
    return HTTPSignatureAuth(
        headers=[ '(request-target)', 'host', 'date', 'digest' ],
        algorithm='rsa-sha256',
        key=_get_private_key(actor),
        key_id=_get_base_url() + '/' + actor + '#publicKey')
    """
    
    return McfiSignatureAuth(
        key_id=_get_base_url() + '/' + actor + '#publicKey',
        secret=_get_private_key(actor),
        algorithm='rsa-sha256',
        headers=[ '(request-target)', 'host', 'date', 'digest' ],
        sign_header='Signature')

def _send(actor, activity):
    """
    Send an activity to a actor's INBOX.
    """
    
    if 'actor' not in activity:
        print('This activity does not have an actor.')
        print(activity)
        return None
    
    if 'to' not in activity:
        if activity['type'] == 'Follow':
            activity['to'] = activity['object']
        else:
            abort(400, 'Error: No recipient defined.')
    
    # Normalize "to:" to a list
    if isinstance(activity['to'], str):
        activity['to'] = [ activity['to'] ]
    
    if isinstance(activity['to'], dict):
        print('Dict "to:" not implemented.')
        response.status = 500
        return
    
    # Find the INBOX of all recipients
    recipients = list(activity['to'])
    inboxes = []
    
    if 'cc' in activity:
        recipients.extend(activity['cc'])
    
    # Remove properties according to spec.
    # https://www.w3.org/TR/activitypub/#client-to-server-interactions
    if 'bto' in activity:
        recipients.extend(activity['bto'])
        del activity['bto']
    
    if 'bcc' in activity:
        recipients.extend(activity['bcc'])
        del activity['bcc']
    
    while len(recipients) > 0:
        recipient = recipients[0]
        del recipients[0]
        
        # Retrieve remote actor
        try:
            remote_actor = requests.get(recipient, headers=request_headers)
            assert remote_actor.status_code == 200
            remote_actor = remote_actor.json()
        except Exception as e:
            print('Actor error: ' + recipient)
            continue
        
        if remote_actor['type'] == 'Collection':
            recipients.extend(remote_actor['items'])
        elif remote_actor['type'] == 'OrderedCollection':
            recipients.extend(remote_actor['orderedItems'])
        else:
            if 'inbox' in remote_actor:
                inboxes.append(remote_actor['inbox'])
            else:
                print('Actor does not have an inbox: ' + remote_actor)
    
    # Remove duplicate inboxes
    inboxes = list(set(inboxes))
    
    # Deliver messages
    for inbox in inboxes:
        # Now send the activity to the actor's INBOX
        #result = requests.post(inbox,
        #                       headers=request_headers,
        #                       data=json.dumps(activity).encode(),
        #                       auth=_get_http_auth(actor))
        
        req = requests.Request('POST',
                               inbox,
                               headers=request_headers,
                               data=json.dumps(activity).encode(),
                               auth=_get_http_auth(actor))
        prepared = req.prepare()

        def pretty_print_POST(req):
            with open('/opt/mcfi/TEST', 'w') as f:
                f.write('{}\n{}\r\n{}\r\n\r\n{}\n{}\n'.format(
                    '-----------START-----------',
                    req.method + ' ' + req.url,
                    '\r\n'.join('{}: {}'.format(k, v) for k, v in req.headers.items()),
                    req.body,
                    '------------END------------',
                ))

        pretty_print_POST(prepared)
        requests.Session().send(prepared)

################################################################################
# Controllers for federation.
# These should respond to "Accept: application/activity+json" requests.
################################################################################

@get('/inbox/<actor:path>')
def read_inbox(actor):
    """
    Read an actor's INBOX.
    """
    
    if actor not in actors:
        abort(401)
    
    if 'Authorization' not in request.headers:
        # Should the public be able to read actors' INBOX?
        abort(403) # Not implemented
    
    if request.get_header('Authorization') != 'Bearer {}'.format(actors[actor]['authorization_token']):
        abort(401)
    
    messages = database.get_inbox_messages(actor)
    
    return {
        '@context': jsonld_context,
        'type': 'OrderedCollection',
        'totalItems': len(messages),
        'orderedItems': messages
    }

@post('/inbox/<actor:path>')
def write_inbox(actor):
    """
    Somebody wants to write to a local actor's INBOX.
    
    TODO Verify Activity signatures instead of accepting everything.
    """
    
    """
    print("HEADER:")
    for h in request.headers:
        print(h + ':[' + request.headers[h] + ']')
    print('\n')
    
    v = HeaderVerifier(
        request.headers,
        RSA.import_key(_get_private_key("zplus")).publickey().exportKey('PEM'),
        required_headers=[ '(request-target)', 'host', 'date' ],
        method='post', path='/inbox/'+actor,
        sign_header='Signature')
    print(">> ", v.verify())
    
    return
    """
    
    if actor not in actors:
        abort(401, 'Not a valid actor.')
    
    if 'Authorization' in request.headers:
        # Should a user write to his own inbox?
        abort(403)
    
    # We have received a messaged from a remote actor to our INBOX.
    
    # Parse the JSON payload
    activity = json.loads(request.body.getvalue().decode('UTF-8'))
    
    if not activity:
        abort(400)
    
    # Store a copy of the activity
    database.store_inbox(actor, activity)
    
    # Make sure post_activity is an absolute URLs
    if not bool(urlparse(settings.post_activity).netloc):
        settings.post_activity = _get_base_url() + settings.post_activity
    
    # Relay activity to the forge (avoid forge polling)
    answer = requests.post(settings.post_activity + '/' + actor, json=activity)
    
    if answer.status_code != 200:
        return abort(500, 'Could not contact forge.')
    
    answer = answer.json()
    
    # No Activity was returned
    if 'type' not in answer:
        return
    
    # Automatically add properties to the given Activity (which contains partial data)
    answer['@context'] = jsonld_context,
    answer['actor']    = _get_base_url() + '/' + actor
    
    if answer['type'] == 'Accept':
        if answer['object']['type'] == 'Follow':
            database.add_follower(actor, activity['actor'])
        
    # Send Activity
    _send(actor, answer)

@get('/outbox/<actor:path>')
def read_outbox(actor):
    """
    Read an actor's OUTBOX.
    """
    
    if actor not in actors:
        abort(400)
    
    if 'Authorization' in request.headers:
        abort(403) # Not implemented
    else:
        abort(403) # Not implemented

@post('/outbox/<actor:path>')
def write_outbox(actor):
    """
    A local actor is writing to his OUTBOX.
    """
    
    if actor not in actors:
        abort(400, 'Actor doesn\'t exist.')
    
    # Must be authenticated to write OUTBOX
    if 'Authorization' not in request.headers:
        abort(401, 'Missing authorization token.')
    
    # Validate authentication token
    if request.get_header('Authorization') != 'Bearer ' + actors[actor]['authorization_token']:
        abort(401, 'Bad authorization token.')
    
    # The Activity payload, a JSON object that was sent with the request
    message = json.loads(request.body.getvalue().decode('UTF-8'))
    
    if not message:
        abort(400, 'Bad message body.')
    
    # Server must ignore any given ID and generate a new one, as per spec
    if 'id' in message:
        del message['id']
    
    # Use an hash of the Activity as ID
    message['id'] = _get_base_url() + _url('activity', id=hashlib.sha512(json.dumps(message, sort_keys=True).encode()).hexdigest())
    
    # Add publishing date
    message['published'] = datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc).isoformat()
    
    # Keep a copy of the message in the database
    database.store_outbox(actor, message)
    
    # Send message
    # TODO make this async
    _send(actor, message)
    
    if message['type'] == 'Follow':
        # Add remote actor to local actor's list of "following"
        database.add_following(actor, message['object'])
    
    response.status = 201 # Created
    response.set_header('Location', message['id'])

@get('/followers/<actor:path>', name='followers')
def read_followers(actor):
    """
    Read an actor's collection of followers.
    """
    
    followers = database.get_followers(actor)
    
    return {
        '@context': jsonld_context,
        'type': 'Collection',
        'totalItems': len(followers),
        'items': followers
    }

@get('/following/<actor:path>', name='following')
def read_following(actor):
    """
    Read an actor's collection of following.
    """

@get('/activity/<id>', name='activity')
def get_activity(id):
    """
    Retrieve an Activity.
    """
    
    response.set_header('Accept',       request_headers['Accept'])
    response.set_header('Content-Type', request_headers['Content-Type'])
    
    return json.dumps(database.get_activity(_get_base_url() + '/activity/' + id))

################################################################################
# Controllers for a demo forge
################################################################################

@get('/.well-known/host-meta')
def webfinger_host_meta():
    response.set_header('Content-Type', 'application/xrd+xml; charset=utf-8')
    
    return Template("""
    <?xml version="1.0" encoding="UTF-8"?>
    <XRD xmlns="http://docs.oasis-open.org/ns/xri/xrd-1.0">
        <Link rel="lrdd" type="application/xrd+xml" template="$baseurl/.well-known/webfinger?resource={uri}"/>
    </XRD>
    """).substitute({ 'baseurl': _get_base_url() })

@get('/.well-known/webfinger')
def webfinger_resource():
    if 'resource' not in request.GET:
        return abort(404)
    
    resource = request.GET.getunicode('resource')
    
    # Remove prefix "acct:" and split hostname
    actor, host = resource[5:].rsplit('@', 1)
    
    if host != request.headers['Host'] or actor not in actors:
        abort(404)
    
    response.set_header('Content-Type', 'application/jrd+json; charset=utf-8')
    
    return Template("""
    {
        "subject": "acct:$actor@$host",
        "aliases": [
            "$baseurl/$actor"
        ],
        "links": [
            {
                "rel": "http://webfinger.net/rel/profile-page",
                "type": "text/html",
                "href": "$baseurl/$actor"
            },
            {
                "rel": "self",
                "type": "application/activity+json",
                "href": "$baseurl/$actor"
            }
        ]
    }
    """).substitute({
        'actor': actor,
        'host': host,
        'baseurl': _get_base_url()
    })

@get('/federation/actor/<actor:path>')
def get_actor(actor):
    if actor not in actors:
       abort(404)
    
    base_url = _get_base_url()
    
    if actors[actor]['type'] == 'Person':
        return {
            'type': 'Person',
            'name': actor,
        }
    
    if actors[actor]['type'] == 'Repository':
        return {
            'type': 'Repository',
            'name': actor,
            'clone_url': '{}/{}.git'.format(base_url, actor),
        }

@post('/federation/activity/<actor:path>')
def post_activity(actor):
    if actor not in actors:
       abort(404)
    
    # Ignore users. This forge does not automatically respond to any incoming
    # Activity for a Person. It only responds automatically for Repository actors.
    # Use CLIFF instead.
    if actors[actor]['type'] == 'Person':
        return {}
    
    # Bottle: If the Content-Type header is application/json or
    # application/json-rpc, the "json" property holds the parsed content of the
    # request body.
    activity = request.json
    
    # Automatically accept Follow requests
    if activity['type'] == 'Follow':
        return {
            'type':   'Accept',
            'to':     _get_base_url() + _url('followers', actor=actor),
            'object': activity }
    
    if activity['type'] == 'Offer':
        
        # Automatically accept new tickets
        if activity['object']['type'] == 'Ticket':
            ticket_number = database.add_ticket(actor)
            
            # Assign a new Ticket number
            activity['object']['number'] = ticket_number
            
            # Assign a new Ticket ID
            activity['object']['id'] = '{}/{}/ticket/{}'.format(_get_base_url(),
                                                                actor, ticket_number)
            
            return {
                'type':   'Accept',
                'to':     _get_base_url() + _url('followers', actor=actor),
                'object': activity }
    
    if activity['type'] == 'Create':
        
        if activity['object']['type'] == 'Note':
            # Return the same Activity. Will be sent to followers
            
            activity['to'] = _get_base_url() + _url('followers', actor=actor)
            
            return activity

@post('/git-hooks-post-receive')
def post_receive_hook():
    """
    This is called by the post-receive hook after every push. The JSON request
    body contains a list of the pushed commits.
    """
    
    # Allow requests from this machine only.
    if request.environ.get('REMOTE_ADDR') != '127.0.0.1':
        abort(403, 'Who are you?')
    
    # Parse "git push" data
    push = request.json
    
    commits_object = []
    
    for commit in push['commits']:
        commits_object.append({
            '@context': jsonld_context,
            'type': 'Commit',
            'id': 'todo: assign an id here',
            'attributedTo': commit['author']['name'],
            'committedBy': commit['committer']['name'],
            'hash': commit['id'],
            'description': {
                'mediaType': 'text/plain',
                'content': commit['message']
            },
            'created': commit['author']['time'],
            'committed': commit['commit_time']
        })
    
    _send(
        push['REMOTE_USER'],
        {
            '@context': jsonld_context,
            'type': 'Push',
            'actor': '{}/{}'.format(_get_base_url(), push['repository_name']),
            'to': _get_base_url() + _url('followers', actor=push['repository_name']),
            'object': commits_object
        }
    )

@get('/<actor:path>')
def actor_document(actor):
    """
    Return profile of an actor.
    """
    
    if actor not in actors:
       abort(404)
    
    base_url = _get_base_url()
    
    # Make sure get_actor is an absolute URLs
    if not bool(urlparse(settings.get_actor).netloc):
        settings.get_actor = _get_base_url() + settings.get_actor
    
    # Retrieve the actor from the forge
    actor_document = requests.get(settings.get_actor + '/' + actor)
    
    if actor_document.status_code != 200:
        abort(404)
    
    # Parse forge data
    actor_document = actor_document.json()
    
    # Add federation properties to user document
    actor_document = {
        **actor_document,
        '@context':   jsonld_context,
        'id':        '{}/{}'.format(base_url, actor),
        'inbox':     '{}/inbox/{}'.format(base_url, actor),
        'outbox':    '{}/outbox/{}'.format(base_url, actor),
        'followers': '{}/followers/{}'.format(base_url, actor),
        'following': '{}/following/{}'.format(base_url, actor),
        'publicKey' : {
            'id': '{}/{}#publicKey'.format(base_url, actor),
            'owner': '{}/{}'.format(base_url, actor),
            'publicKeyPem': RSA.import_key(_get_private_key(actor)).publickey().exportKey('PEM').decode(),
            'https://forgefed.angeley.es/ns#isShared': False
        }}
    
    # This is text/html request
    if 'application/activity+json' not in request.headers.get('Accept', '*/*') and \
       'application/activity+json' not in request.headers.get('Content-Type', '*/*') and \
       'application/ld+json' not in request.headers.get('Accept', '*/*') and \
       'application/ld+json' not in request.headers.get('Content-Type', '*/*'):
        
        if actor_document['type'] == 'Person':
            return Template("""
                <html>
                    <head>
                    </head>
                    <body>
                        <p><strong>User:</strong> $user</p>
                        <p><strong>Federation URL:</strong> $fed</p>
                    </body>
                </html>
                """).substitute({
                    'user': actor_document['name'],
                    'fed': actor_document['id'] })
        
        if actor_document['type'] == 'Repository':
            return Template("""
                <html>
                    <head>
                    </head>
                    <body>
                        <p><strong>Repository:</strong> $name</p>
                        <p><strong>Clone URL:</strong> git clone $url</p>
                        <p><strong>Federation URL:</strong> $fed</p>
                    </body>
                </html>
                """).substitute({
                    'name': actor_document['name'],
                    'url': actor_document['clone_url'],
                    'fed': actor_document['id'] })
        
        return
    
    # ActivityPub clients accept 'Accept: application/ld+json'
    
    response.set_header('Accept',       request_headers['Accept'])
    response.set_header('Content-Type', request_headers['Content-Type'])
    
    return json.dumps(actor_document)

@route('/')
def index():
    return '<p><em>This is a demo forge.</em></p>' + \
           'MCFI version ' + __version__