oscapi.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386
  1. #!/usr/bin/python -tt
  2. # vim: ai ts=4 sts=4 et sw=4
  3. #
  4. # Copyright (c) 2012 Intel, Inc.
  5. #
  6. # This program is free software; you can redistribute it and/or modify it
  7. # under the terms of the GNU General Public License as published by the Free
  8. # Software Foundation; version 2 of the License
  9. #
  10. # This program is distributed in the hope that it will be useful, but
  11. # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  12. # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
  13. # for more details.
  14. #
  15. # You should have received a copy of the GNU General Public License along
  16. # with this program; if not, write to the Free Software Foundation, Inc., 59
  17. # Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18. """
  19. This module provides wrapper class around OSC API.
  20. Only APIs which are required by cmd_remotebuild present here.
  21. """
  22. import os
  23. import re
  24. import urllib2
  25. import M2Crypto
  26. from M2Crypto.SSL.Checker import SSLVerificationError
  27. import ssl
  28. from collections import defaultdict
  29. from urllib import quote_plus, pathname2url
  30. from xml.etree import cElementTree as ET
  31. from gitbuildsys.utils import hexdigest
  32. from gitbuildsys.errors import ObsError
  33. from gitbuildsys.log import waiting
  34. from gitbuildsys.log import LOGGER as logger
  35. from osc import conf, core
  36. class OSCError(Exception):
  37. """Local exception class."""
  38. pass
  39. class OSC(object):
  40. """Interface to OSC API"""
  41. def __init__(self, apiurl=None, oscrc=None):
  42. if oscrc:
  43. try:
  44. conf.get_config(override_conffile = oscrc)
  45. except OSError, err:
  46. if err.errno == 1:
  47. # permission problem, should be the chmod(0600) issue
  48. raise ObsError('Current user has no write permission '\
  49. 'for specified oscrc: %s' % oscrc)
  50. raise # else
  51. except urllib2.URLError:
  52. raise ObsError("invalid service apiurl: %s" % apiurl)
  53. else:
  54. conf.get_config()
  55. if apiurl:
  56. self.apiurl = apiurl
  57. else:
  58. self.apiurl = conf.config['apiurl']
  59. @staticmethod
  60. def core_http(method, url, data=None, filep=None):
  61. """Wrapper above core.<http_METHOD> to catch exceptions."""
  62. # Workarounded osc bug. http_GET sometimes returns empty response
  63. # Usually next try succeeds, so let's try 3 times
  64. for count in (1, 2, 3):
  65. try:
  66. return method(url, data=data, file=filep)
  67. except (urllib2.URLError, M2Crypto.m2urllib2.URLError,
  68. M2Crypto.SSL.SSLError, ssl.SSLError), err:
  69. if count == 3:
  70. raise OSCError(str(err))
  71. raise OSCError('Got empty response from %s %s' % \
  72. (method.func_name.split('_')[-1], url))
  73. def get_repos_of_project(self, project):
  74. """Get dictionary name: list of archs for project repos"""
  75. repos = defaultdict(list)
  76. for repo in core.get_repos_of_project(self.apiurl, project):
  77. repos[repo.name].append(repo.arch)
  78. return repos
  79. def get_tags(self, project, tags):
  80. """Get tags content from meta."""
  81. meta_xml = self.get_meta(project)
  82. xml_root = ET.fromstring(meta_xml)
  83. result = ''
  84. for tag in tags:
  85. element = xml_root.find(tag)
  86. if element is not None:
  87. result += ET.tostring(element)
  88. return result
  89. def create_project(self, target, src=None, rewrite=False,
  90. description='', linkto='', linkedbuild=''):
  91. """
  92. Create new OBS project based on existing project.
  93. Copy config and repositories from src project to target
  94. if src exists.
  95. """
  96. if src and not self.exists(src):
  97. raise ObsError('base project: %s not exists' % src)
  98. if self.exists(target):
  99. logger.warning('target project: %s exists' % target)
  100. if rewrite:
  101. logger.warning('rewriting target project %s' % target)
  102. else:
  103. return
  104. # Create target meta
  105. meta = '<project name="%s"><title></title>'\
  106. '<description>%s</description>'\
  107. '<person role="maintainer" userid="%s"/>' % \
  108. (target, description, conf.get_apiurl_usr(self.apiurl))
  109. if linkto:
  110. meta += '<link project="%s"/>' % linkto
  111. # Collect source repos if src project exist
  112. if src:
  113. # Copy debuginfo, build, useforbuild and publish meta
  114. meta += self.get_tags(src, ['debuginfo', 'build',
  115. 'useforbuild', 'publish'])
  116. # Copy repos to target
  117. repos = self.get_repos_of_project(src)
  118. for name in repos:
  119. if linkedbuild:
  120. meta += '<repository name="%s" linkedbuild="%s">' % \
  121. (name, linkedbuild)
  122. else:
  123. meta += '<repository name="%s">' % name
  124. meta += '<path project="%s" repository="%s" />' % (src, name)
  125. for arch in repos[name]:
  126. meta += "<arch>%s</arch>\n" % arch
  127. meta += "</repository>\n"
  128. else:
  129. logger.warning('no project repos in target project, please add '
  130. 'repos from OBS webUI manually, or specify base project '
  131. 'with -B <base_prj>, then gbs can help to set repos '
  132. 'using the settings of the specified base project.')
  133. meta += "</project>\n"
  134. try:
  135. # Create project and set its meta
  136. core.edit_meta('prj', path_args=quote_plus(target), data=meta)
  137. except (urllib2.URLError, M2Crypto.m2urllib2.URLError,
  138. M2Crypto.SSL.SSLError), err:
  139. raise ObsError("Can't set meta for %s: %s" % (target, str(err)))
  140. # don't need set project config if no src project
  141. if not src:
  142. return
  143. # copy project config
  144. try:
  145. config = core.show_project_conf(self.apiurl, src)
  146. except (urllib2.URLError, M2Crypto.m2urllib2.URLError,
  147. M2Crypto.SSL.SSLError), err:
  148. raise ObsError("Can't get config from project %s: %s" \
  149. % (src, str(err)))
  150. url = core.make_meta_url("prjconf", quote_plus(target),
  151. self.apiurl, False)
  152. try:
  153. self.core_http(core.http_PUT, url, data=''.join(config))
  154. except OSCError, err:
  155. raise ObsError("can't copy config from %s to %s: %s" \
  156. % (src, target, err))
  157. def delete_project(self, prj, force=False, msg=None):
  158. """Delete OBS project."""
  159. query = {}
  160. if force:
  161. query['force'] = "1"
  162. if msg:
  163. query['comment'] = msg
  164. url = core.makeurl(self.apiurl, ['source', prj], query)
  165. try:
  166. self.core_http(core.http_DELETE, url)
  167. except OSCError, err:
  168. raise ObsError("can't delete project %s: %s" % (prj, err))
  169. def exists(self, prj, pkg=''):
  170. """Check if project or package exists."""
  171. metatype, path_args = self.get_path(prj, pkg)
  172. err = None
  173. try:
  174. core.meta_exists(metatype = metatype, path_args = path_args,
  175. create_new = False, apiurl = self.apiurl)
  176. except urllib2.HTTPError, err:
  177. if err.code == 404:
  178. return False
  179. except (urllib2.URLError, M2Crypto.m2urllib2.URLError, \
  180. M2Crypto.SSL.SSLError), err:
  181. pass
  182. except SSLVerificationError:
  183. raise ObsError("SSL verification error.")
  184. if err:
  185. raise ObsError("can't check if %s/%s exists: %s" % (prj, pkg, err))
  186. return True
  187. def rebuild(self, prj, pkg, arch):
  188. """Rebuild package."""
  189. try:
  190. return core.rebuild(self.apiurl, prj, pkg, repo=None, arch=arch)
  191. except (urllib2.URLError, M2Crypto.m2urllib2.URLError, \
  192. M2Crypto.SSL.SSLError), err:
  193. raise ObsError("Can't trigger rebuild for %s/%s: %s" % \
  194. (prj, pkg, str(err)))
  195. except SSLVerificationError:
  196. raise ObsError("SSL verification error.")
  197. def diff_files(self, prj, pkg, paths):
  198. """
  199. Find difference between local and remote filelists
  200. Return 4 lists: (old, not changed, changed, new)
  201. where:
  202. old - present only remotely
  203. changed - present remotely and locally and differ
  204. not changed - present remotely and locally and does not not differ
  205. new - present only locally
  206. old is a list of remote filenames
  207. changed, not changed and new are lists of local filepaths
  208. """
  209. # Get list of files from the OBS
  210. rfiles = core.meta_get_filelist(self.apiurl, prj, pkg, verbose=True,
  211. expand=True)
  212. old, not_changed, changed, new = [], [], [], []
  213. if not rfiles:
  214. # no remote files - all local files are new
  215. return old, not_changed, changed, paths[:]
  216. # Helper dictionary helps to avoid looping over remote files
  217. rdict = dict((fobj.name, (fobj.size, fobj.md5)) for fobj in rfiles)
  218. for lpath in paths:
  219. lname = os.path.basename(lpath)
  220. if lname in rdict:
  221. lsize = os.path.getsize(lpath)
  222. rsize, rmd5 = rdict[lname]
  223. if rsize == lsize and rmd5 == core.dgst(lpath):
  224. not_changed.append(lpath)
  225. else:
  226. changed.append(lpath)
  227. # remove processed files from the remote dict
  228. # after processing only old files will be letf there
  229. rdict.pop(lname)
  230. else:
  231. new.append(lpath)
  232. return rdict.keys(), not_changed, changed, new
  233. @waiting
  234. def commit_files(self, prj, pkg, files, message):
  235. """Commits files to OBS."""
  236. query = {'cmd' : 'commitfilelist',
  237. 'user' : conf.get_apiurl_usr(self.apiurl),
  238. 'comment': message,
  239. 'keeplink': 1}
  240. url = core.makeurl(self.apiurl, ['source', prj, pkg], query=query)
  241. xml = "<directory>"
  242. for fpath, _ in files:
  243. with open(fpath) as fhandle:
  244. xml += '<entry name="%s" md5="%s"/>' % \
  245. (os.path.basename(fpath), hexdigest(fhandle))
  246. xml += "</directory>"
  247. try:
  248. self.core_http(core.http_POST, url, data=xml)
  249. for fpath, commit_flag in files:
  250. if commit_flag:
  251. put_url = core.makeurl(
  252. self.apiurl, ['source', prj, pkg,
  253. pathname2url(os.path.basename(fpath))],
  254. query="rev=repository")
  255. self.core_http(core.http_PUT, put_url, filep=fpath)
  256. self.core_http(core.http_POST, url, data=xml)
  257. except OSCError, err:
  258. raise ObsError("can't commit files to %s/%s: %s" % (prj, pkg, err))
  259. def create_package(self, prj, pkg):
  260. """Create package in the project."""
  261. meta = '<package project="%s" name="%s">'\
  262. '<title/><description/></package>' % (prj, pkg)
  263. url = core.make_meta_url("pkg", (quote_plus(prj), quote_plus(pkg)),
  264. self.apiurl, False)
  265. try:
  266. self.core_http(core.http_PUT, url, data=meta)
  267. except OSCError, err:
  268. raise ObsError("can't create %s/%s: %s" % (prj, pkg, err))
  269. def get_results(self, prj, pkg):
  270. """Get package build results."""
  271. results = defaultdict(dict)
  272. try:
  273. build_status = core.get_results(self.apiurl, prj, pkg)
  274. except (urllib2.URLError, M2Crypto.m2urllib2.URLError,
  275. M2Crypto.SSL.SSLError), err:
  276. raise ObsError("can't get %s/%s build results: %s" \
  277. % (prj, pkg, str(err)))
  278. # This regular expression is created for parsing the
  279. # results of of core.get_results()
  280. stat_re = re.compile(r'^(?P<repo>\S+)\s+(?P<arch>\S+)\s+'
  281. '(?P<status>\S*)$')
  282. for res in build_status:
  283. match = stat_re.match(res)
  284. if match:
  285. results[match.group('repo')][match.group('arch')] = \
  286. match.group('status')
  287. else:
  288. logger.warning('not valid build status received: %s' % res)
  289. return results
  290. def get_buildlog(self, prj, pkg, repo, arch):
  291. """Get package build log from OBS."""
  292. url = core.makeurl(self.apiurl, ['build', prj, repo, arch, pkg,
  293. '_log?nostream=1&start=0'])
  294. try:
  295. log = self.core_http(core.http_GET, url).read()
  296. except OSCError, err:
  297. raise ObsError("can't get %s/%s build log: %s" % (prj, pkg, err))
  298. return log.translate(None, "".join([chr(i) \
  299. for i in range(10) + range(11,32)]))
  300. @staticmethod
  301. def get_path(prj, pkg=None):
  302. """Helper to get path_args out of prj and pkg."""
  303. metatype = 'prj'
  304. path_args = [quote_plus(prj)]
  305. if pkg:
  306. metatype = 'pkg'
  307. path_args.append(quote_plus(pkg))
  308. return metatype, tuple(path_args)
  309. def get_meta(self, prj, pkg=None):
  310. """Get project/package meta."""
  311. metatype, path_args = self.get_path(prj, pkg)
  312. url = core.make_meta_url(metatype, path_args, self.apiurl)
  313. return self.core_http(core.http_GET, url).read()
  314. def set_meta(self, meta, prj, pkg=None):
  315. """Set project/package meta."""
  316. metatype, path_args = self.get_path(prj, pkg)
  317. url = core.make_meta_url(metatype, path_args, self.apiurl)
  318. return self.core_http(core.http_PUT, url, data=meta)
  319. def get_description(self, prj, pkg=None):
  320. """Get project/package description."""
  321. meta = self.get_meta(prj, pkg)
  322. result = ET.fromstring(meta).find('description')
  323. return result or result.text
  324. def set_description(self, description, prj, pkg=None):
  325. """Set project/package description."""
  326. meta = ET.fromstring(self.get_meta(prj, pkg))
  327. dsc = meta.find('description')
  328. dsc.text = description
  329. self.set_meta(ET.tostring(meta), prj, pkg)