/[soft]/mga-gnome/trunk/mga-gnome
ViewVC logotype

Contents of /mga-gnome/trunk/mga-gnome

Parent Directory Parent Directory | Revision Log Revision Log


Revision 3115 - (show annotations) (download)
Tue Feb 28 10:23:35 2012 UTC (8 years, 1 month ago) by ovitters
File size: 21837 byte(s)
add ability to check hash of original tarball
1 #!/usr/bin/python
2
3 # A lot of the code comes from ftpadmin, see
4 # http://git.gnome.org/browse/sysadmin-bin/tree/ftpadmin
5 # Written by Olav Vitters
6
7 # basic modules:
8 import os
9 import os.path
10 import sys
11 import re
12 import subprocess
13
14 # command line parsing, error handling:
15 import argparse
16 import errno
17
18 # overwriting files by moving them (safer):
19 import tempfile
20 import shutil
21
22 # version comparison:
23 import rpm
24
25 # opening tarballs:
26 import tarfile
27 import gzip
28 import bz2
29 import lzma # pyliblzma
30
31 # getting links from HTML document:
32 from sgmllib import SGMLParser
33 import urllib2
34 import urlparse
35
36 # for checking hashes
37 import hashlib
38
39 MEDIA="Core Release Source"
40 URL="http://download.gnome.org/sources/"
41 PKGROOT='~/pkgs'
42
43 re_majmin = re.compile(r'^([0-9]+\.[0-9]+).*')
44 re_version = re.compile(r'([-.]|\d+|[^-.\d]+)')
45
46 def version_cmp(a, b):
47 """Compares two versions
48
49 Returns
50 -1 if a < b
51 0 if a == b
52 1 if a > b
53 """
54
55 return rpm.labelCompare(('1', a, '1'), ('1', b, '1'))
56
57 def get_latest_version(versions, max_version=None):
58 """Gets the latest version number
59
60 if max_version is specified, gets the latest version number before
61 max_version"""
62 latest = None
63 for version in versions:
64 if ( latest is None or version_cmp(version, latest) > 0 ) \
65 and ( max_version is None or version_cmp(version, max_version) < 0 ):
66 latest = version
67 return latest
68
69 def judge_version_increase(version_old, version_new):
70 """Judge quality of version increase:
71
72 Returns a tuple containing judgement and message
73
74 Judgement:
75 Less than 0: Error
76 0 to 4: Better not
77 5+: Ok"""
78 versions = (version_old, version_new)
79
80 # First do a basic version comparison to ensure version_new is actually newer
81 compare = version_cmp(version_new, version_old)
82
83 if compare == 0:
84 return (-2, "Already at version %s!" % (version_old))
85
86 if compare != 1:
87 return (-3, "Version %s is older than current version %s!" % (version_new, version_old))
88
89 # Version is newer, but we don't want to see if it follows the GNOME versioning scheme
90 majmins = [re_majmin.sub(r'\1', ver) for ver in versions if re_majmin.match(ver) is not None]
91
92 if len(majmins) == 1:
93 return (-1, "Version number scheme changes: %s" % (", ".join(versions)))
94
95 if len(majmins) == 0:
96 return (0, "Unsupported version numbers: %s" % (", ".join(versions)))
97
98 # Follows GNOME versioning scheme
99 # Meaning: x.y.z
100 # x = major
101 # y = minor : even if stable
102 # z = micro
103
104 # Major+minor the same? Then go ahead and upgrade!
105 if majmins[0] == majmins[1]:
106 # Majmin of both versions are the same, looks good!
107 return (10, None)
108
109 # More detailed analysis needed, so figure out the numbers
110 majmin_nrs = [map(long, ver.split('.')) for ver in majmins]
111
112 # Check/ensure major version number is the same
113 if majmin_nrs[0][0] != majmin_nrs[1][0]:
114 return (1, "Major version number increase")
115
116 # Minor indicates stable/unstable
117 devstate = (majmin_nrs[0][1] % 2 == 0, majmin_nrs[1][1] % 2 == 0)
118
119 # Upgrading to unstable is weird
120 if not devstate[1]:
121 if devstate[0]:
122 return (1, "Stable to unstable increase")
123
124 return (4, "Unstable to unstable version increase")
125
126 # Unstable => stable is always ok
127 if not devstate[0]:
128 return (5, "Unstable to stable")
129
130 # Can only be increase of minors from one stable to the next
131 return (6, "Stable version increase")
132
133 def line_input (file):
134 for line in file:
135 if line[-1] == '\n':
136 yield line[:-1]
137 else:
138 yield line
139
140 def call_editor(filename):
141 """Return a sequence of possible editor binaries for the current platform"""
142
143 editors = []
144
145 for varname in 'VISUAL', 'EDITOR':
146 if varname in os.environ:
147 editors.append(os.environ[varname])
148
149 editors.extend(('/usr/bin/editor', 'vi', 'pico', 'nano', 'joe'))
150
151 for editor in editors:
152 try:
153 ret = subprocess.call([editor, filename])
154 except OSError, e:
155 if e.errno == 2:
156 continue
157 raise
158
159 if ret == 127:
160 continue
161
162 return True
163
164 class urllister(SGMLParser):
165 def reset(self):
166 SGMLParser.reset(self)
167 self.urls = []
168
169 def start_a(self, attrs):
170 href = [v for k, v in attrs if k=='href']
171 if href:
172 self.urls.extend(href)
173
174 class XzTarFile(tarfile.TarFile):
175
176 OPEN_METH = tarfile.TarFile.OPEN_METH.copy()
177 OPEN_METH["xz"] = "xzopen"
178
179 @classmethod
180 def xzopen(cls, name, mode="r", fileobj=None, **kwargs):
181 """Open gzip compressed tar archive name for reading or writing.
182 Appending is not allowed.
183 """
184 if len(mode) > 1 or mode not in "rw":
185 raise ValueError("mode must be 'r' or 'w'")
186
187 if fileobj is not None:
188 fileobj = _LMZAProxy(fileobj, mode)
189 else:
190 fileobj = lzma.LZMAFile(name, mode)
191
192 try:
193 # lzma doesn't immediately return an error
194 # try and read a bit of data to determine if it is a valid xz file
195 fileobj.read(_LZMAProxy.blocksize)
196 fileobj.seek(0)
197 t = cls.taropen(name, mode, fileobj, **kwargs)
198 except IOError:
199 raise tarfile.ReadError("not a xz file")
200 except lzma.error:
201 raise tarfile.ReadError("not a xz file")
202 t._extfileobj = False
203 return t
204
205 if not hasattr(tarfile.TarFile, 'xzopen'):
206 tarfile.open = XzTarFile.open
207
208 def is_valid_hash(path, algo, hexdigest):
209 if algo not in hashlib.algorithms:
210 raise ValueError("Unknown hash algorithm: %s" % algo)
211
212 local_hash = getattr(hashlib, algo)()
213
214 with open(path, 'rb') as fp:
215 data = fp.read(32768)
216 while data:
217 local_hash.update(data)
218 data = fp.read(32768)
219
220 return local_hash.hexdigest() == hexdigest
221
222 class SpecFile(object):
223 re_update_version = re.compile(r'^(?P<pre>Version:\s*)(?P<version>.+)(?P<post>\s*)$', re.MULTILINE + re.IGNORECASE)
224 re_update_release = re.compile(r'^(?P<pre>Release:\s*)(?P<release>%mkrel \d+)(?P<post>\s*)$', re.MULTILINE + re.IGNORECASE)
225
226 def __init__(self, path):
227 self.path = path
228 self.cwd = os.path.dirname(path)
229
230 @property
231 def version(self):
232 return subprocess.check_output(["rpm", "--specfile", self.path, "--queryformat", "%{VERSION}\n"]).splitlines()[0]
233 @property
234 def sources(self):
235 ts = rpm.ts()
236 spec = ts.parseSpec(self.path)
237 srclist = spec.sources if isinstance(spec.sources, (list, tuple)) \
238 else spec.sources()
239 return dict((os.path.basename(name), name) for name, no, flags in srclist)
240
241 def update(self, version):
242 """Update specfile (increase version)"""
243 cur_version = self.version
244
245 (judgement, msg) = judge_version_increase(cur_version, version)
246
247 if judgement < 0:
248 print >>sys.stderr, "ERROR: %s!" % (msg)
249 return False
250
251 if judgement < 5:
252 print "WARNING: %s!" % (msg)
253 return False
254
255 # XXX - os.path.join is hackish
256 if subprocess.check_output(["svn", "diff", os.path.join(self.path, '..')]) != '':
257 print >>sys.stderr, "ERROR: Package has uncommitted changes!"
258 return False
259
260 with open(self.path, "rw") as f:
261 data = f.read()
262
263 if data.count("%mkrel") != 1:
264 print >>sys.stderr, "ERROR: Multiple %mkrel found; don't know what to do!"
265 return False
266
267 data, nr = self.re_update_version.subn(r'\g<pre>%s\g<post>' % version, data, 1)
268 if nr != 1:
269 print >>sys.stderr, "ERROR: Could not increase version!"
270 return False
271
272 data, nr = self.re_update_release.subn(r'\g<pre>%mkrel 1\g<post>', data, 1)
273 if nr != 1:
274 print >>sys.stderr, "ERROR: Could not reset release!"
275 return False
276
277 # Overwrite file with new version number
278 write_file(self.path, data)
279
280
281 # Verify that RPM also agrees that version number has changed
282 if self.version != version:
283 print "ERROR: Increased version to %s, but RPM doesn't agree!?!" % version
284 return False
285
286 try:
287 # Download new tarball
288 subprocess.check_call(['mgarepo', 'sync', '-d'], cwd=self.cwd)
289 # Check patches still apply
290 subprocess.check_call(['bm', '-p', '--nodeps'], cwd=self.cwd)
291 except subprocess.CalledProcessError:
292 return False
293
294 return True
295
296 class Patch(object):
297 """Do things with patches"""
298
299 re_dep3 = re.compile(r'^(?:#\s*)?(?P<header>[-A-Za-z0-9]+?):\s*(?P<data>.*)$')
300 re_dep3_cont = re.compile(r'^#?\s+(?P<data>.*)$')
301
302 def __init__(self, path, show_path=False):
303 """Path: path to patch (might not exist)"""
304 self.path = path
305 self.show_path = show_path
306
307 def __str__(self):
308 return self.path if self.show_path else os.path.basename(self.path)
309
310 def add_dep3(self):
311 """Add DEP-3 headers to a patch file"""
312 if self.dep3['valid']:
313 return False
314
315 new_headers = (
316 ('Author', self.svn_author),
317 ('Subject', ''),
318 ('Applied-Upstream', ''),
319 ('Forwarded', ''),
320 ('Bug', ''),
321 )
322
323 with tempfile.NamedTemporaryFile(dir=os.path.dirname(self.path), delete=False) as fdst:
324 with open(self.path, "r") as fsrc:
325 # Start with any existing DEP3 headers
326 for i in range(self.dep3['last_nr']):
327 fdst.write(fsrc.read())
328
329 # After that add the DEP3 headers
330 add_line = False
331 for header, data in new_headers:
332 if header in self.dep3['headers']:
333 continue
334
335 # XXX - wrap this at 80 chars
336 add_line = True
337 print >>fdst, "%s: %s" % (header, "" if data is None else data)
338
339 if add_line: print >>fdst, ""
340 # Now copy any other data and the patch
341 shutil.copyfileobj(fsrc, fdst)
342
343 fdst.flush()
344 os.rename(fdst.name, self.path)
345
346 call_editor(self.path)
347
348 #Author: fwang
349 #Subject: Build fix: Fix glib header inclusion
350 #Applied-Upstream: commit:30602
351 #Forwarded: yes
352 #Bug: http://bugzilla.abisource.com/show_bug.cgi?id=13247
353
354 def _read_dep3(self):
355 """Read DEP-3 headers from an existing patch file
356
357 This will also parse git headers"""
358 dep3 = {}
359 headers = {}
360
361 last_header = None
362 last_nr = 0
363 nr = 0
364 try:
365 with open(self.path, "r") as f:
366 for line in line_input(f):
367 nr += 1
368 # stop trying to parse when real patch begins
369 if line == '---':
370 break
371
372 r = self.re_dep3.match(line)
373 if r:
374 info = r.groupdict()
375
376 # Avoid matching URLS
377 if info['data'].startswith('//') and info['header'].lower () == info['header']:
378 continue
379
380 headers[info['header']] = info['data']
381 last_header = info['header']
382 last_nr = nr
383 continue
384
385 r = self.re_dep3_cont.match(line)
386 if r:
387 info = r.groupdict()
388 if last_header:
389 headers[last_header] = " ".join((headers[last_header], info['data']))
390 last_nr = nr
391 continue
392
393 last_header = None
394 except IOError:
395 pass
396
397 dep3['valid'] = \
398 (('Description' in headers and headers['Description'].strip() != '')
399 or ('Subject' in headers and headers['Subject'].strip() != '')) \
400 and (('Origin' in headers and headers['Origin'].strip() != '') \
401 or ('Author' in headers and headers['Author'].strip() != '') \
402 or ('From' in headers and headers['From'].strip() != ''))
403 dep3['last_nr'] = last_nr
404 dep3['headers'] = headers
405
406 self._dep3 = dep3
407
408 @property
409 def dep3(self):
410 if not hasattr(self, '_dep3'):
411 self._read_dep3()
412
413 return self._dep3
414
415 @property
416 def svn_author(self):
417 if not hasattr(self, '_svn_author'):
418 try:
419 contents = subprocess.check_output(['svn', 'log', '-q', "--", self.path], close_fds=True).strip("\n").splitlines()
420
421 for line in contents:
422 if ' | ' not in line:
423 continue
424
425 fields = line.split(' | ')
426 if len(fields) >= 3:
427 self._svn_author = fields[1]
428 except subprocess.CalledProcessError:
429 pass
430
431 if not hasattr(self, '_svn_author'):
432 return None
433
434 return self._svn_author
435
436 def get_upstream_names():
437 urlopen = urllib2.build_opener()
438
439 good_dir = re.compile('^[-A-Za-z0-9_+.]+/$')
440
441 # Get the files
442 usock = urlopen.open(URL)
443 parser = urllister()
444 parser.feed(usock.read())
445 usock.close()
446 parser.close()
447 files = parser.urls
448
449 tarballs = set([filename.replace('/', '') for filename in files if good_dir.search(filename)])
450
451 return tarballs
452
453 def get_downstream_names():
454 re_file = re.compile(r'^(?P<module>.*?)[_-](?:(?P<oldversion>([0-9]+[\.])*[0-9]+)-)?(?P<version>([0-9]+[\.\-])*[0-9]+)\.(?P<format>(?:tar\.|diff\.)?[a-z][a-z0-9]*)$')
455
456 contents = subprocess.check_output(['urpmf', '--files', '.', "--media", MEDIA], close_fds=True).strip("\n").splitlines()
457
458 FILES = {}
459 TARBALLS = {}
460
461 for line in contents:
462 try:
463 srpm, filename = line.split(":")
464 except ValueError:
465 print >>sys.stderr, line
466 continue
467
468 if '.tar' in filename:
469 r = re_file.match(filename)
470 if r:
471 fileinfo = r.groupdict()
472 module = fileinfo['module']
473
474 if module not in TARBALLS:
475 TARBALLS[module] = set()
476 TARBALLS[module].add(srpm)
477
478 if srpm not in FILES:
479 FILES[srpm] = set()
480 FILES[srpm].add(filename)
481
482 return TARBALLS, FILES
483
484
485 def write_file(path, data):
486 with tempfile.NamedTemporaryFile(dir=os.path.dirname(path), delete=False) as fdst:
487 fdst.write(data)
488 fdst.flush()
489 os.rename(fdst.name, path)
490
491 def cmd_co(options, parser):
492 upstream = get_upstream_names()
493 downstream, downstream_files = get_downstream_names()
494
495 cwd = os.path.expanduser(PKGROOT)
496
497 matches = upstream & set(downstream.keys())
498 for module in matches:
499 print module, "\t".join(downstream[module])
500 for package in downstream[module]:
501 subprocess.call(['mgarepo', 'co', package], cwd=cwd)
502
503 def join_streams():
504 upstream = get_upstream_names()
505 downstream, downstream_files = get_downstream_names()
506
507 matches = upstream & set(downstream.keys())
508 for module in matches:
509 for package in downstream[module]:
510 yield (package, module)
511
512 def cmd_ls(options, parser):
513 for package, module in sorted(join_streams()):
514 print "\t".join((package, module)) if options.upstream else package
515
516 def cmd_patches(options, parser):
517 upstream = get_upstream_names()
518 downstream, downstream_files = get_downstream_names()
519
520 path = os.path.expanduser(PKGROOT)
521
522 import pprint
523
524 matches = upstream & set(downstream.keys())
525 for module in sorted(matches):
526 for srpm in downstream[module]:
527 for filename in downstream_files[srpm]:
528 if '.patch' in filename or '.diff' in filename:
529
530 p = Patch(os.path.join(path, srpm, "SOURCES", filename), show_path=options.path)
531 valid = ""
532 forwarded = ""
533 if p.dep3['headers']:
534 forwarded = p.dep3['headers'].get('Forwarded', "no")
535 if p.dep3['valid']:
536 valid="VALID"
537 print "\t".join((module, srpm, str(p), forwarded, valid))
538
539 def cmd_dep3(options, parser):
540 p = Patch(options.patch)
541 p.add_dep3()
542
543 def cmd_package_new_version(options, parser):
544 # Determine the package name
545 if options.upstream:
546 downstream, downstream_files = get_downstream_names()
547
548 if options.package not in downstream:
549 print >>sys.stderr, "ERROR: No packages for upstream name: %s" % options.package
550 sys.exit(1)
551
552 if len(downstream[options.package]) != 1:
553 # XXX - Make it more intelligent
554 print >>sys.stderr, "ERROR: Multiple packages found for %s: %s" % (options.package, ", ".join(downstream[options.package]))
555 sys.exit(1)
556
557 package = list(downstream[options.package])[0]
558 else:
559 package = options.package
560
561 # Directories packages are located in
562 root = os.path.expanduser(PKGROOT)
563 cwd = os.path.join(root, package)
564
565 # Checkout package to ensure the checkout reflects the latest changes
566 try:
567 subprocess.check_call(['mgarepo', 'co', package], cwd=root)
568 except subprocess.CalledProcessError:
569 sys.exit(1)
570
571 # SpecFile class handles the actual version+release change
572 s = SpecFile(os.path.join(cwd, "SPECS", "%s.spec" % package))
573 print "%s => %s" % (s.version, options.version)
574 if not s.update(options.version):
575 sys.exit(1)
576
577 # Check hash, if given
578 if options.hexdigest is not None:
579 sources = [name for name, origname in s.sources.iteritems() if '://' in origname]
580 if not len(sources):
581 print >>sys.stderr, "ERROR: Cannot determine source file (for hash check)!"
582 sys.stderr(1)
583
584 for filename in sources:
585 if not is_valid_hash(os.path.join(cwd, "SOURCES", filename), options.algo, options.hexdigest):
586 print >>sys.stderr, "ERROR: Hash file failed check for %s!" % path
587 print >>sys.stderr, "ERROR: Reverting changes!"
588 subprocess.call(['svn', 'revert', '-R', cwd], cwd=cwd)
589 sys.exit(1)
590
591 # We can even checkin and submit :-)
592 if options.submit:
593 try:
594 # checkin changes
595 subprocess.check_call(['mgarepo', 'ci', '-m', 'new version %s' % options.version], cwd=cwd)
596 # and submit
597 subprocess.check_call(['mgarepo', 'submit'], cwd=cwd)
598 except subprocess.CalledProcessError:
599 sys.exit(1)
600
601
602 def main():
603 description = """Mageia GNOME commands."""
604 epilog="""Report bugs to Olav Vitters"""
605 parser = argparse.ArgumentParser(description=description,epilog=epilog)
606
607 # SUBPARSERS
608 subparsers = parser.add_subparsers(title='subcommands')
609 # install
610 subparser = subparsers.add_parser('co', help='checkout all GNOME modules')
611 subparser.set_defaults(
612 func=cmd_co
613 )
614
615 subparser = subparsers.add_parser('packages', help='list all GNOME packages')
616 subparser.add_argument("-m", "--m", action="store_true", dest="upstream",
617 help="Show upstream module")
618 subparser.set_defaults(
619 func=cmd_ls, upstream=False
620 )
621
622 subparser = subparsers.add_parser('patches', help='list all GNOME patches')
623 subparser.add_argument("-p", "--path", action="store_true", dest="path",
624 help="Show full path to patch")
625 subparser.set_defaults(
626 func=cmd_patches, path=False
627 )
628
629 subparser = subparsers.add_parser('dep3', help='Add dep3 headers')
630 subparser.add_argument("patch", help="Patch")
631 subparser.set_defaults(
632 func=cmd_dep3, path=False
633 )
634
635 subparser = subparsers.add_parser('increase', help='Increase version number')
636 subparser.add_argument("package", help="Package name")
637 subparser.add_argument("version", help="Version number")
638 subparser.add_argument("-u", "--upstream", action="store_true", dest="upstream",
639 help="Package name reflects the upstream name")
640 subparser.add_argument("-s", "--submit", action="store_true", dest="submit",
641 help="Commit changes and submit")
642 subparser.add_argument("-a", "--algorithm", choices=hashlib.algorithms, dest="algo",
643 help="Hash algorithm")
644 subparser.add_argument("--hash", dest="hexdigest",
645 help="Hexdigest of the hash")
646 subparser.set_defaults(
647 func=cmd_package_new_version, submit=False, upstream=False, hexdigest=None, algo="sha256"
648 )
649
650 if len(sys.argv) == 1:
651 parser.print_help()
652 sys.exit(2)
653
654 options = parser.parse_args()
655
656 try:
657 options.func(options, parser)
658 except KeyboardInterrupt:
659 print('Interrupted')
660 sys.exit(1)
661 except EOFError:
662 print('EOF')
663 sys.exit(1)
664 except IOError, e:
665 if e.errno != errno.EPIPE:
666 raise
667 sys.exit(0)
668
669 if __name__ == "__main__":
670 main()

Properties

Name Value
svn:executable *

  ViewVC Help
Powered by ViewVC 1.1.26