123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289 |
- #!/usr/bin/env python3
- # -*- coding: utf8 -*-
- import os
- import sys
- import tarfile
- import sqlite3
- import requests
- from lxml import etree # What the fuck, Python: https://docs.python.org/3.9/library/xml.html#xml-vulnerabilities
- from http import HTTPStatus
- from datetime import datetime
- if __name__ == '__main__':
- versions = [
- '10.14.5',
- '10.14.4',
- '10.14.3',
- '10.14.2',
- '10.14.1',
- '10.14',
- '10.13.6',
- '10.13.5',
- '10.13.4',
- '10.13.3',
- '10.13.2',
- '10.13.1',
- '10.13',
- '10.12.6',
- '10.12.5',
- '10.12.4',
- '10.12.3',
- '10.12.2',
- '10.12.1',
- '10.12',
- '10.11.6',
- '10.11.5',
- '10.11.4',
- '10.11.3',
- '10.11.2',
- '10.11.1',
- '10.11',
- '10.10.5',
- '10.10.4',
- '10.10.3',
- '10.10.2',
- '10.10.1',
- '10.10',
- '10.9.5',
- '10.9.4',
- '10.9.3',
- '10.9.2',
- '10.9.1',
- '10.9',
- '10.8.5',
- '10.8.4',
- '10.8.3',
- '10.8.2',
- '10.8.1',
- '10.8',
- '10.7.5',
- '10.7.4',
- '10.7.3',
- '10.7.2',
- '10.7.1',
- '10.7',
- '10.6.8',
- '10.6.7',
- '10.6.6',
- '10.6.5',
- '10.6.4',
- '10.6.3',
- '10.6.2',
- '10.6.1',
- '10.6',
- '10.5.8',
- '10.5.7',
- '10.5.6',
- '10.5.5',
- '10.5.4',
- '10.5.3',
- '10.5.2',
- '10.5.1',
- '10.5',
- '10.4.11.x86',
- '10.4.11.ppc',
- '10.4.10.x86',
- '10.4.10.ppc',
- '10.4.9.x86',
- '10.4.9.ppc',
- '10.4.8.ppc',
- '10.4.8.x86',
- '10.4.7.ppc',
- '10.4.7.x86',
- '10.4.6.ppc',
- '10.4.6.x86',
- '10.4.5.x86',
- '10.4.5.ppc',
- '10.4.4.x86',
- '10.4.4.ppc',
- '10.4.3',
- '10.4.2',
- '10.4.1',
- '10.4',
- '10.3.9',
- '10.3.8',
- '10.3.7',
- '10.3.6',
- '10.3.5',
- '10.3.4',
- '10.3.3',
- '10.3.2',
- '10.3.1',
- '10.3',
- '10.2.8',
- '10.2.8.G5',
- '10.2.7',
- '10.2.6',
- '10.2.5',
- '10.2.4',
- '10.2.3',
- '10.2.2',
- '10.2.1',
- '10.2',
- '10.1.5',
- '10.1.4',
- '10.1.3',
- '10.1.2',
- '10.1.1',
- '10.1',
- '10.0.4',
- '10.0.3',
- '10.0.2',
- '10.0.1',
- '10.0',
- ]
- attributes = {
- 'OpenSourceImportDate': 'imported',
- 'OpenSourceLicense': 'license',
- 'OpenSourceLicenseFile': 'license_file',
- 'OpenSourceModifications': 'modifications',
- 'OpenSourceProject': 'name',
- 'OpenSourceSHA1': 'sha1',
- 'OpenSourceURL': 'url',
- 'OpenSourceVersion': 'version',
- 'OpenSourceWebsite': 'website',
- }
- db = sqlite3.connect('openapple.db')
- c = db.cursor()
- c.execute("""
- CREATE TABLE IF NOT EXISTS `projects` (
- `id` INTEGER PRIMARY KEY AUTOINCREMENT,
- `macos` TEXT NOT NULL,
- `name` TEXT NOT NULL,
- `build` TEXT NOT NULL,
- `modified` TEXT NOT NULL,
- `done` INT NOT NULL DEFAULT 0,
- UNIQUE(`macos`, `name`)
- )
- """)
- c.execute("""
- CREATE TABLE IF NOT EXISTS `opensource` (
- `id` INTEGER PRIMARY KEY AUTOINCREMENT,
- `project_id` INTEGER NOT NULL,
- `macos` TEXT NOT NULL,
- `name` TEXT,
- `imported` TEXT,
- `license` TEXT,
- `license_file` TEXT,
- `modifications` TEXT,
- `sha1` TEXT,
- `url` TEXT,
- `version` TEXT,
- `website` TEXT,
- UNIQUE(`macos`, `name`)
- )
- """)
- db.commit()
- for version in versions:
- main = requests.get('https://opensource.apple.com/plist/macos-%s.plist' % version.replace('.', ''))
- if main.status_code == HTTPStatus.OK:
- projectList = etree.fromstring(main.content)[0].find('dict')
- for i, child in enumerate(projectList):
- if child.tag != 'key':
- continue
- name = child.text
- # Skip project if done in db
- c.execute('SELECT COUNT(*) FROM `projects` WHERE `name`=(?) AND `done`=TRUE', (name, ))
- if c.fetchone()[0]:
- continue
- buildVersion = projectList[i+1].find('string').text
- tar = requests.get('https://opensource.apple.com/tarballs/%s/%s-%s.tar.gz' % (name, name, buildVersion))
- if tar.status_code == HTTPStatus.OK:
- with open('temp.tar.gz', 'wb') as f:
- f.write(tar.content)
- else:
- print('Error! %s' % name)
- continue
-
- tar = tarfile.open('temp.tar.gz')
- members = tar.getmembers()
- lastModified = datetime.utcfromtimestamp(int(members[0].get_info()['mtime'])).strftime('%Y-%m-%d %H:%M:%S')
- try:
- c.execute('INSERT INTO `projects` (`macos`, `name`, `build`, `modified`) VALUES (?, ?, ?, ?) ', (version, name, buildVersion, lastModified))
- db.commit()
- except sqlite3.IntegrityError:
- pass
- finally:
- c.execute('SELECT `id` FROM `projects` WHERE `name`=(?)', (name, ))
- projectId = c.fetchone()[0]
-
- plists = [member for member in members if '.plist' in member.name or '.partial' in member.name]
- for member in plists:
- try:
- plist = etree.fromstring(tar.extractfile(member).read())
- if plist.tag != 'dict':
- opensource = plist.find('dict')
- if opensource == None:
- opensource = plist.find('array')
- if opensource == None:
- continue
- opensource = opensource.find('dict')
-
- if opensource == None:
- continue
- else:
- opensource = plist
- except etree.XMLSyntaxError:
- # Skip invalid xml files
- continue
- relevant = {}
-
- for index, attribute in enumerate(opensource):
- if attribute.tag != 'key':
- continue
- if attribute.text in attributes.keys():
- relevant[attributes[attribute.text]] = opensource[index+1].text
- if not relevant:
- continue
- relevant['macos'] = version
- relevant['project_id'] = projectId
- for attribute in attributes.values():
- if not attribute in relevant:
- relevant[attribute] = ''
- try:
- c.execute('INSERT INTO opensource VALUES (NULL, :macos, :project_id, :name, :imported, :license, :license_file, :modifications, :sha1, :url, :version, :website)', relevant)
- db.commit()
- except sqlite3.IntegrityError:
- pass
-
-
- c.execute('UPDATE `projects` SET `done`=TRUE WHERE `id`=(?)', (projectId, ))
- db.commit()
- os.remove('temp.tar.gz')
|