|
@@ -15,8 +15,10 @@ import hashlib
|
|
import gzip
|
|
import gzip
|
|
import shutil
|
|
import shutil
|
|
import dateutil.parser
|
|
import dateutil.parser
|
|
|
|
+import msgpack
|
|
|
|
+import base64
|
|
|
|
|
|
-import uploader
|
|
|
|
|
|
+import config
|
|
|
|
|
|
|
|
|
|
class TimetableDownloader:
|
|
class TimetableDownloader:
|
|
@@ -58,14 +60,19 @@ class TimetableDownloader:
|
|
|
|
|
|
for file in to_download:
|
|
for file in to_download:
|
|
print('getting {}.zip'.format(file))
|
|
print('getting {}.zip'.format(file))
|
|
- self.__get_timetable(file)
|
|
|
|
- checksum = self.__converter()
|
|
|
|
- for p in Path('.').glob('*.txt'):
|
|
|
|
- p.unlink()
|
|
|
|
- size_u = os.path.getsize('timetable.db')
|
|
|
|
- self.__compress(checksum)
|
|
|
|
- self.__archive(file, checksum, size_u, os.path
|
|
|
|
- .getsize('{}.db.gz'.format(checksum)))
|
|
|
|
|
|
+ try:
|
|
|
|
+ self.__get_timetable(file)
|
|
|
|
+ except zipfile.BadZipFile:
|
|
|
|
+ print('ERROR: file {} is not a zip'.format(file))
|
|
|
|
+ else:
|
|
|
|
+ checksum = self.__converter()
|
|
|
|
+ for p in Path('.').glob('*.txt'):
|
|
|
|
+ p.unlink()
|
|
|
|
+ size_u = os.path.getsize('timetable.db')
|
|
|
|
+ self.__compress(checksum)
|
|
|
|
+ meta = self.__archive(file, checksum, size_u, os.path
|
|
|
|
+ .getsize('{}.db.gz'.format(checksum)))
|
|
|
|
+ self.__upload(checksum, meta)
|
|
|
|
|
|
def __is_valid(self, name):
|
|
def __is_valid(self, name):
|
|
today = date.today().strftime('%Y%m%d')
|
|
today = date.today().strftime('%Y%m%d')
|
|
@@ -136,6 +143,7 @@ dla-deweloperow/getGTFSFile?file={}.zip'
|
|
metadata['start'] = start_date
|
|
metadata['start'] = start_date
|
|
metadata['end'] = end_date
|
|
metadata['end'] = end_date
|
|
self.__metadata.append(metadata)
|
|
self.__metadata.append(metadata)
|
|
|
|
+ return metadata
|
|
|
|
|
|
def __tidy_up(self):
|
|
def __tidy_up(self):
|
|
names = ['_'.join((row['start'], row['end'])) for row in
|
|
names = ['_'.join((row['start'], row['end'])) for row in
|
|
@@ -159,6 +167,29 @@ dla-deweloperow/getGTFSFile?file={}.zip'
|
|
|
|
|
|
self.__metadata = new_metadata
|
|
self.__metadata = new_metadata
|
|
|
|
|
|
|
|
+ def __upload(self, id, meta):
|
|
|
|
+ with open('{}.db.gz'.format(id), 'rb') as f:
|
|
|
|
+ t = f.read()
|
|
|
|
+ sha = hashlib.sha256(t).hexdigest()
|
|
|
|
+ print('uploading {}'.format(id))
|
|
|
|
+ signature = config.key.sign(bytes(sha, 'utf-8'))
|
|
|
|
+ data = msgpack.packb({'meta': meta, 'signature': signature.signature})
|
|
|
|
+ length = len(data)
|
|
|
|
+ data = bytes(str(length), 'utf-8')+b'\n'+data+t
|
|
|
|
+ session = requests.Session()
|
|
|
|
+ response = session.put(config.receiver, data)
|
|
|
|
+ print(response)
|
|
|
|
+ print(response.text)
|
|
|
|
+
|
|
|
|
+ def __upload_del(self, id):
|
|
|
|
+ print('uploading del {}'.format(id))
|
|
|
|
+ signature = config.key.sign(bytes(id, 'utf-8'))
|
|
|
|
+ session = requests.Session()
|
|
|
|
+ s = str(base64.b64encode(signature.signature), 'utf-8')
|
|
|
|
+ response = session.delete('{}/{}:{}'.format(config.receiver, id, s))
|
|
|
|
+ print(response)
|
|
|
|
+ print(response.text)
|
|
|
|
+
|
|
|
|
|
|
class TimetableConverter:
|
|
class TimetableConverter:
|
|
__BUF_SIZE = 65536
|
|
__BUF_SIZE = 65536
|
|
@@ -377,5 +408,4 @@ class TimetableConverter:
|
|
if __name__ == '__main__':
|
|
if __name__ == '__main__':
|
|
downloader = TimetableDownloader()
|
|
downloader = TimetableDownloader()
|
|
downloader()
|
|
downloader()
|
|
- uploader.upload()
|
|
|
|
print('done')
|
|
print('done')
|