/[soft]/mga-gnome/trunk/mga-gnome
ViewVC logotype

Annotation of /mga-gnome/trunk/mga-gnome

Parent Directory Parent Directory | Revision Log Revision Log


Revision 3090 - (hide annotations) (download)
Mon Feb 27 14:32:00 2012 UTC (12 years, 1 month ago) by ovitters
File size: 19817 byte(s)
add forgotten regexp
1 ovitters 2932 #!/usr/bin/python
2    
3 ovitters 3057 # A lot of the code comes from ftpadmin, see
4     # http://git.gnome.org/browse/sysadmin-bin/tree/ftpadmin
5     # Written by Olav Vitters
6    
7     # basic modules:
8 ovitters 2932 import os
9     import os.path
10     import sys
11     import re
12     import subprocess
13 ovitters 3057
14     # command line parsing, error handling:
15 ovitters 2932 import argparse
16 ovitters 2936 import errno
17 ovitters 3057
18     # overwriting files by moving them (safer):
19 ovitters 2944 import tempfile
20     import shutil
21 ovitters 3057
22     # version comparison:
23 ovitters 3045 import rpm
24 ovitters 3057
25     # opening tarballs:
26     import tarfile
27     import gzip
28     import bz2
29     import lzma # pyliblzma
30    
31     # getting links from HTML document:
32 ovitters 2932 from sgmllib import SGMLParser
33 ovitters 3057 import urllib2
34     import urlparse
35 ovitters 2932
36     MEDIA="Core Release Source"
37     URL="http://download.gnome.org/sources/"
38     PKGROOT='~/pkgs'
39    
40 ovitters 3090 re_majmin = re.compile(r'^([0-9]+\.[0-9]+).*')
41 ovitters 3012 re_version = re.compile(r'([-.]|\d+|[^-.\d]+)')
42    
43     def version_cmp(a, b):
44     """Compares two versions
45    
46     Returns
47     -1 if a < b
48     0 if a == b
49     1 if a > b
50 ovitters 3045 """
51 ovitters 3012
52 ovitters 3045 return rpm.labelCompare(('1', a, '1'), ('1', b, '1'))
53 ovitters 3012
54     def get_latest_version(versions, max_version=None):
55     """Gets the latest version number
56    
57     if max_version is specified, gets the latest version number before
58     max_version"""
59     latest = None
60     for version in versions:
61     if ( latest is None or version_cmp(version, latest) > 0 ) \
62     and ( max_version is None or version_cmp(version, max_version) < 0 ):
63     latest = version
64     return latest
65    
66 ovitters 3088 def judge_version_increase(version_old, version_new):
67     """Judge quality of version increase:
68    
69     Returns a tuple containing judgement and message
70    
71     Judgement:
72     Less than 0: Error
73     0 to 4: Better not
74     5+: Ok"""
75     versions = (version_old, version_new)
76    
77     print " => ".join(versions)
78    
79     # First do a basic version comparison to ensure version_new is actually newer
80     compare = version_cmp(version_new, version_old)
81    
82     if compare == 0:
83     return (-2, "Already at version %s!" % (version_old))
84    
85     if compare != 1:
86     return (-3, "Version %s is older than current version %s!" % (version_new, version_old))
87    
88     # Version is newer, but we don't want to see if it follows the GNOME versioning scheme
89     majmins = [re_majmin.sub(r'\1', ver) for ver in versions if re_majmin.match(ver) is not None]
90    
91     if len(majmins) == 1:
92     return (-1, "Version number scheme changes: %s" % (", ".join(versions)))
93    
94     if len(majmins) == 0:
95     return (0, "Unsupported version numbers: %s" % (", ".join(versions)))
96    
97     # Follows GNOME versioning scheme
98     # Meaning: x.y.z
99     # x = major
100     # y = minor : even if stable
101     # z = micro
102    
103     # Major+minor the same? Then go ahead and upgrade!
104     if majmins[0] == majmins[1]:
105     # Majmin of both versions are the same, looks good!
106     return (10, None)
107    
108     # More detailed analysis needed, so figure out the numbers
109     majmin_nrs = [map(long, ver.split('.')) for ver in majmins]
110    
111     # Check/ensure major version number is the same
112     if majmin_nrs[0][0] != majmin_nrs[1][0]:
113     return (1, "Major version number increase")
114    
115     # Minor indicates stable/unstable
116     devstate = (majmin_nrs[0][1] % 2 == 0, majmin_nrs[1][1] % 2 == 0)
117    
118     # Upgrading to unstable is weird
119     if not devstate[1]:
120     if devstate[0]:
121     return (1, "Stable to unstable increase")
122    
123     return (4, "Unstable to unstable version increase")
124    
125     # Unstable => stable is always ok
126     if not devstate[0]:
127     return (5, "Unstable to stable")
128    
129     # Can only be increase of minors from one stable to the next
130     return (6, "Stable version increase")
131    
132 ovitters 2936 def line_input (file):
133     for line in file:
134     if line[-1] == '\n':
135     yield line[:-1]
136     else:
137     yield line
138    
139 ovitters 2955 def call_editor(filename):
140     """Return a sequence of possible editor binaries for the current platform"""
141    
142     editors = []
143    
144     for varname in 'VISUAL', 'EDITOR':
145     if varname in os.environ:
146     editors.append(os.environ[varname])
147    
148     editors.extend(('/usr/bin/editor', 'vi', 'pico', 'nano', 'joe'))
149    
150     for editor in editors:
151     try:
152     ret = subprocess.call([editor, filename])
153     except OSError, e:
154     if e.errno == 2:
155     continue
156     raise
157    
158     if ret == 127:
159     continue
160    
161     return True
162    
163 ovitters 2932 class urllister(SGMLParser):
164     def reset(self):
165     SGMLParser.reset(self)
166     self.urls = []
167    
168     def start_a(self, attrs):
169     href = [v for k, v in attrs if k=='href']
170     if href:
171     self.urls.extend(href)
172    
173 ovitters 3057 class XzTarFile(tarfile.TarFile):
174    
175     OPEN_METH = tarfile.TarFile.OPEN_METH.copy()
176     OPEN_METH["xz"] = "xzopen"
177    
178     @classmethod
179     def xzopen(cls, name, mode="r", fileobj=None, **kwargs):
180     """Open gzip compressed tar archive name for reading or writing.
181     Appending is not allowed.
182     """
183     if len(mode) > 1 or mode not in "rw":
184     raise ValueError("mode must be 'r' or 'w'")
185    
186     if fileobj is not None:
187     fileobj = _LMZAProxy(fileobj, mode)
188     else:
189     fileobj = lzma.LZMAFile(name, mode)
190    
191     try:
192     # lzma doesn't immediately return an error
193     # try and read a bit of data to determine if it is a valid xz file
194     fileobj.read(_LZMAProxy.blocksize)
195     fileobj.seek(0)
196     t = cls.taropen(name, mode, fileobj, **kwargs)
197     except IOError:
198     raise tarfile.ReadError("not a xz file")
199     except lzma.error:
200     raise tarfile.ReadError("not a xz file")
201     t._extfileobj = False
202     return t
203    
204 ovitters 3082 if not hasattr(tarfile.TarFile, 'xzopen'):
205 ovitters 3057 tarfile.open = XzTarFile.open
206    
207 ovitters 3012 class SpecFile(object):
208     re_update_version = re.compile(r'^(?P<pre>Version:\s*)(?P<version>.+)(?P<post>\s*)$', re.MULTILINE + re.IGNORECASE)
209     re_update_release = re.compile(r'^(?P<pre>Release:\s*)(?P<release>%mkrel \d+)(?P<post>\s*)$', re.MULTILINE + re.IGNORECASE)
210    
211     def __init__(self, path):
212     self.path = path
213     self.cwd = os.path.dirname(path)
214    
215     @property
216     def version(self):
217     return subprocess.check_output(["rpm", "--specfile", self.path, "--queryformat", "%{VERSION}\n"]).splitlines()[0]
218 ovitters 3037
219 ovitters 3012 def update(self, version):
220     """Update specfile (increase version)"""
221     cur_version = self.version
222    
223 ovitters 3088 (judgement, msg) = judge_version_increase(cur_version, version)
224 ovitters 3037
225 ovitters 3088 if judgement < 0:
226     print >>sys.stderr, "ERROR: %s!" % (msg)
227 ovitters 3037 return False
228    
229 ovitters 3088 if judgement < 5:
230 ovitters 3089 print "WARNING: %s!" % (msg)
231 ovitters 3012 return False
232    
233 ovitters 3039 # XXX - os.path.join is hackish
234     if subprocess.check_output(["svn", "diff", os.path.join(self.path, '..')]) != '':
235     print >>sys.stderr, "ERROR: Package has uncommitted changes!"
236     return False
237    
238 ovitters 3012 with open(self.path, "rw") as f:
239     data = f.read()
240    
241     if data.count("%mkrel") != 1:
242 ovitters 3037 print >>sys.stderr, "ERROR: Multiple %mkrel found; don't know what to do!"
243 ovitters 3012 return False
244    
245     data, nr = self.re_update_version.subn(r'\g<pre>%s\g<post>' % version, data, 1)
246     if nr != 1:
247 ovitters 3037 print >>sys.stderr, "ERROR: Could not increase version!"
248 ovitters 3012 return False
249    
250     data, nr = self.re_update_release.subn(r'\g<pre>%mkrel 1\g<post>', data, 1)
251     if nr != 1:
252 ovitters 3037 print >>sys.stderr, "ERROR: Could not reset release!"
253 ovitters 3012 return False
254    
255     # Overwrite file with new version number
256     write_file(self.path, data)
257    
258    
259 ovitters 3045 # Verify that RPM also agrees that version number has changed
260 ovitters 3012 if self.version != version:
261     print "ERROR: Increased version to %s, but RPM doesn't agree!?!" % version
262     return False
263    
264 ovitters 3034 try:
265     # Download new tarball
266     subprocess.check_call(['mgarepo', 'sync', '-d'], cwd=self.cwd)
267     # Check patches still apply
268     subprocess.check_call(['bm', '-p', '--nodeps'], cwd=self.cwd)
269     except subprocess.CalledProcessError:
270     return False
271 ovitters 3012
272     return True
273    
274 ovitters 2936 class Patch(object):
275     """Do things with patches"""
276    
277     re_dep3 = re.compile(r'^(?:#\s*)?(?P<header>[-A-Za-z0-9]+?):\s*(?P<data>.*)$')
278     re_dep3_cont = re.compile(r'^#?\s+(?P<data>.*)$')
279    
280     def __init__(self, path, show_path=False):
281     """Path: path to patch (might not exist)"""
282     self.path = path
283     self.show_path = show_path
284    
285     def __str__(self):
286     return self.path if self.show_path else os.path.basename(self.path)
287    
288     def add_dep3(self):
289 ovitters 2955 """Add DEP-3 headers to a patch file"""
290 ovitters 2944 if self.dep3['valid']:
291     return False
292    
293     new_headers = (
294     ('Author', self.svn_author),
295     ('Subject', ''),
296     ('Applied-Upstream', ''),
297     ('Forwarded', ''),
298     ('Bug', ''),
299     )
300    
301     with tempfile.NamedTemporaryFile(dir=os.path.dirname(self.path), delete=False) as fdst:
302     with open(self.path, "r") as fsrc:
303     # Start with any existing DEP3 headers
304     for i in range(self.dep3['last_nr']):
305     fdst.write(fsrc.read())
306    
307     # After that add the DEP3 headers
308     add_line = False
309     for header, data in new_headers:
310     if header in self.dep3['headers']:
311     continue
312    
313     # XXX - wrap this at 80 chars
314     add_line = True
315 ovitters 2955 print >>fdst, "%s: %s" % (header, "" if data is None else data)
316 ovitters 2944
317     if add_line: print >>fdst, ""
318     # Now copy any other data and the patch
319     shutil.copyfileobj(fsrc, fdst)
320    
321     fdst.flush()
322     os.rename(fdst.name, self.path)
323    
324 ovitters 2955 call_editor(self.path)
325    
326 ovitters 2936 #Author: fwang
327     #Subject: Build fix: Fix glib header inclusion
328     #Applied-Upstream: commit:30602
329     #Forwarded: yes
330     #Bug: http://bugzilla.abisource.com/show_bug.cgi?id=13247
331    
332     def _read_dep3(self):
333 ovitters 2955 """Read DEP-3 headers from an existing patch file
334    
335     This will also parse git headers"""
336 ovitters 2936 dep3 = {}
337 ovitters 2944 headers = {}
338 ovitters 2936
339     last_header = None
340 ovitters 2944 last_nr = 0
341     nr = 0
342 ovitters 2936 try:
343     with open(self.path, "r") as f:
344     for line in line_input(f):
345 ovitters 2944 nr += 1
346     # stop trying to parse when real patch begins
347 ovitters 2936 if line == '---':
348     break
349    
350     r = self.re_dep3.match(line)
351     if r:
352     info = r.groupdict()
353 ovitters 3012
354     # Avoid matching URLS
355     if info['data'].startswith('//') and info['header'].lower () == info['header']:
356     continue
357    
358 ovitters 2944 headers[info['header']] = info['data']
359 ovitters 2936 last_header = info['header']
360 ovitters 2944 last_nr = nr
361 ovitters 2936 continue
362    
363     r = self.re_dep3_cont.match(line)
364     if r:
365     info = r.groupdict()
366     if last_header:
367 ovitters 2944 headers[last_header] = " ".join((headers[last_header], info['data']))
368     last_nr = nr
369 ovitters 2936 continue
370    
371     last_header = None
372     except IOError:
373     pass
374 ovitters 2944
375     dep3['valid'] = \
376     (('Description' in headers and headers['Description'].strip() != '')
377     or ('Subject' in headers and headers['Subject'].strip() != '')) \
378     and (('Origin' in headers and headers['Origin'].strip() != '') \
379     or ('Author' in headers and headers['Author'].strip() != '') \
380     or ('From' in headers and headers['From'].strip() != ''))
381     dep3['last_nr'] = last_nr
382     dep3['headers'] = headers
383    
384 ovitters 2936 self._dep3 = dep3
385    
386     @property
387     def dep3(self):
388     if not hasattr(self, '_dep3'):
389     self._read_dep3()
390    
391     return self._dep3
392    
393 ovitters 2944 @property
394     def svn_author(self):
395     if not hasattr(self, '_svn_author'):
396 ovitters 3083 try:
397     contents = subprocess.check_output(['svn', 'log', '-q', "--", self.path], close_fds=True).strip("\n").splitlines()
398    
399 ovitters 2944 for line in contents:
400     if ' | ' not in line:
401     continue
402 ovitters 2936
403 ovitters 2944 fields = line.split(' | ')
404     if len(fields) >= 3:
405     self._svn_author = fields[1]
406 ovitters 3083 except subprocess.CalledProcessError:
407     pass
408 ovitters 2944
409 ovitters 2955 if not hasattr(self, '_svn_author'):
410     return None
411    
412 ovitters 2944 return self._svn_author
413    
414 ovitters 2932 def get_upstream_names():
415     urlopen = urllib2.build_opener()
416    
417     good_dir = re.compile('^[-A-Za-z0-9_+.]+/$')
418    
419     # Get the files
420     usock = urlopen.open(URL)
421     parser = urllister()
422     parser.feed(usock.read())
423     usock.close()
424     parser.close()
425     files = parser.urls
426    
427     tarballs = set([filename.replace('/', '') for filename in files if good_dir.search(filename)])
428    
429     return tarballs
430    
431     def get_downstream_names():
432     re_file = re.compile(r'^(?P<module>.*?)[_-](?:(?P<oldversion>([0-9]+[\.])*[0-9]+)-)?(?P<version>([0-9]+[\.\-])*[0-9]+)\.(?P<format>(?:tar\.|diff\.)?[a-z][a-z0-9]*)$')
433    
434 ovitters 3083 contents = subprocess.check_output(['urpmf', '--files', '.', "--media", MEDIA], close_fds=True).strip("\n").splitlines()
435 ovitters 2932
436     FILES = {}
437     TARBALLS = {}
438    
439     for line in contents:
440     try:
441     srpm, filename = line.split(":")
442     except ValueError:
443     print >>sys.stderr, line
444     continue
445    
446     if '.tar' in filename:
447     r = re_file.match(filename)
448     if r:
449     fileinfo = r.groupdict()
450     module = fileinfo['module']
451    
452     if module not in TARBALLS:
453     TARBALLS[module] = set()
454     TARBALLS[module].add(srpm)
455    
456     if srpm not in FILES:
457     FILES[srpm] = set()
458     FILES[srpm].add(filename)
459    
460     return TARBALLS, FILES
461    
462 ovitters 3012
463     def write_file(path, data):
464     with tempfile.NamedTemporaryFile(dir=os.path.dirname(path), delete=False) as fdst:
465     fdst.write(data)
466     fdst.flush()
467     os.rename(fdst.name, path)
468    
469 ovitters 2932 def cmd_co(options, parser):
470     upstream = get_upstream_names()
471     downstream, downstream_files = get_downstream_names()
472    
473     cwd = os.path.expanduser(PKGROOT)
474    
475     matches = upstream & set(downstream.keys())
476     for module in matches:
477     print module, "\t".join(downstream[module])
478     for package in downstream[module]:
479     subprocess.call(['mgarepo', 'co', package], cwd=cwd)
480    
481     def cmd_ls(options, parser):
482     upstream = get_upstream_names()
483     downstream, downstream_files = get_downstream_names()
484    
485     matches = upstream & set(downstream.keys())
486     for module in matches:
487 ovitters 3083 print "\n".join(sorted(downstream[module]))
488 ovitters 2932
489     def cmd_patches(options, parser):
490     upstream = get_upstream_names()
491     downstream, downstream_files = get_downstream_names()
492    
493     path = os.path.expanduser(PKGROOT)
494    
495 ovitters 2936 import pprint
496    
497 ovitters 2932 matches = upstream & set(downstream.keys())
498     for module in sorted(matches):
499     for srpm in downstream[module]:
500     for filename in downstream_files[srpm]:
501     if '.patch' in filename or '.diff' in filename:
502 ovitters 3012
503 ovitters 2936 p = Patch(os.path.join(path, srpm, "SOURCES", filename), show_path=options.path)
504 ovitters 3012 valid = ""
505     forwarded = ""
506 ovitters 2944 if p.dep3['headers']:
507 ovitters 3012 forwarded = p.dep3['headers'].get('Forwarded', "no")
508 ovitters 2944 if p.dep3['valid']:
509 ovitters 3012 valid="VALID"
510     print "\t".join((module, srpm, str(p), forwarded, valid))
511 ovitters 2932
512 ovitters 2944 def cmd_dep3(options, parser):
513     p = Patch(options.patch)
514     p.add_dep3()
515    
516 ovitters 3012 def cmd_package_new_version(options, parser):
517 ovitters 3087 # Determine the package name
518 ovitters 3086 if options.upstream:
519     downstream, downstream_files = get_downstream_names()
520 ovitters 3012
521 ovitters 3086 if options.package not in downstream:
522     print >>sys.stderr, "ERROR: No packages for upstream name: %s" % options.package
523     sys.exit(1)
524    
525     if len(downstream[options.package]) != 1:
526     # XXX - Make it more intelligent
527 ovitters 3087 print >>sys.stderr, "ERROR: Multiple packages found for %s: %s" % (options.package, ", ".join(downstream[options.package]))
528 ovitters 3086 sys.exit(1)
529    
530     package = list(downstream[options.package])[0]
531     else:
532     package = options.package
533    
534 ovitters 3087 # Directories packages are located in
535 ovitters 3044 root = os.path.expanduser(PKGROOT)
536     cwd = os.path.join(root, package)
537 ovitters 3038
538 ovitters 3087 # Checkout package to ensure the checkout reflects the latest changes
539 ovitters 3044 try:
540     subprocess.check_call(['mgarepo', 'co', package], cwd=root)
541     except subprocess.CalledProcessError:
542     sys.exit(1)
543 ovitters 3087
544     # SpecFile class handles the actual version+release change
545 ovitters 3038 s = SpecFile(os.path.join(cwd, "SPECS", "%s.spec" % package))
546 ovitters 3037 print "%s => %s" % (s.version, options.version)
547 ovitters 3012 if not s.update(options.version):
548     sys.exit(1)
549    
550 ovitters 3087 # We can even checkin and submit :-)
551 ovitters 3038 if options.submit:
552     try:
553     # checkin changes
554 ovitters 3087 subprocess.check_call(['mgarepo', 'ci', '-m', 'new version %s' % options.version], cwd=cwd)
555 ovitters 3038 # and submit
556     subprocess.check_call(['mgarepo', 'submit'], cwd=cwd)
557     except subprocess.CalledProcessError:
558     sys.exit(1)
559 ovitters 3012
560 ovitters 3038
561 ovitters 2932 def main():
562     description = """Mageia GNOME commands."""
563     epilog="""Report bugs to Olav Vitters"""
564     parser = argparse.ArgumentParser(description=description,epilog=epilog)
565    
566     # SUBPARSERS
567     subparsers = parser.add_subparsers(title='subcommands')
568     # install
569     subparser = subparsers.add_parser('co', help='checkout all GNOME modules')
570     subparser.set_defaults(
571     func=cmd_co
572     )
573    
574     subparser = subparsers.add_parser('packages', help='list all GNOME packages')
575     subparser.set_defaults(
576     func=cmd_ls
577     )
578    
579     subparser = subparsers.add_parser('patches', help='list all GNOME patches')
580     subparser.add_argument("-p", "--path", action="store_true", dest="path",
581 ovitters 2944 help="Show full path to patch")
582 ovitters 2932 subparser.set_defaults(
583     func=cmd_patches, path=False
584     )
585    
586 ovitters 2944 subparser = subparsers.add_parser('dep3', help='Add dep3 headers')
587     subparser.add_argument("patch", help="Patch")
588     subparser.set_defaults(
589     func=cmd_dep3, path=False
590     )
591 ovitters 2932
592 ovitters 3012 subparser = subparsers.add_parser('increase', help='Increase version number')
593     subparser.add_argument("package", help="Package name")
594     subparser.add_argument("version", help="Version number")
595 ovitters 3086 subparser.add_argument("-u", "--upstream", action="store_true", dest="upstream",
596     help="Package name reflects the upstream name")
597 ovitters 3038 subparser.add_argument("-s", "--submit", action="store_true", dest="submit",
598     help="Commit changes and submit")
599 ovitters 3012 subparser.set_defaults(
600 ovitters 3086 func=cmd_package_new_version, submit=False, upstream=False
601 ovitters 3012 )
602    
603 ovitters 2932 if len(sys.argv) == 1:
604     parser.print_help()
605     sys.exit(2)
606    
607     options = parser.parse_args()
608    
609     try:
610     options.func(options, parser)
611     except KeyboardInterrupt:
612     print('Interrupted')
613     sys.exit(1)
614     except EOFError:
615     print('EOF')
616     sys.exit(1)
617     except IOError, e:
618     if e.errno != errno.EPIPE:
619     raise
620     sys.exit(0)
621    
622     if __name__ == "__main__":
623     main()

Properties

Name Value
svn:executable *

  ViewVC Help
Powered by ViewVC 1.1.30