#!/usr/bin/python # A lot of the code comes from ftpadmin, see # http://git.gnome.org/browse/sysadmin-bin/tree/ftpadmin # Written by Olav Vitters # basic modules: import os import os.path import sys import re import subprocess # command line parsing, error handling: import argparse import errno # overwriting files by moving them (safer): import tempfile import shutil # version comparison: import rpm # opening tarballs: import tarfile import gzip import bz2 import lzma # pyliblzma # getting links from HTML document: from sgmllib import SGMLParser import urllib2 import urlparse MEDIA="Core Release Source" URL="http://download.gnome.org/sources/" PKGROOT='~/pkgs' re_version = re.compile(r'([-.]|\d+|[^-.\d]+)') def version_cmp(a, b): """Compares two versions Returns -1 if a < b 0 if a == b 1 if a > b """ return rpm.labelCompare(('1', a, '1'), ('1', b, '1')) def get_latest_version(versions, max_version=None): """Gets the latest version number if max_version is specified, gets the latest version number before max_version""" latest = None for version in versions: if ( latest is None or version_cmp(version, latest) > 0 ) \ and ( max_version is None or version_cmp(version, max_version) < 0 ): latest = version return latest def line_input (file): for line in file: if line[-1] == '\n': yield line[:-1] else: yield line def call_editor(filename): """Return a sequence of possible editor binaries for the current platform""" editors = [] for varname in 'VISUAL', 'EDITOR': if varname in os.environ: editors.append(os.environ[varname]) editors.extend(('/usr/bin/editor', 'vi', 'pico', 'nano', 'joe')) for editor in editors: try: ret = subprocess.call([editor, filename]) except OSError, e: if e.errno == 2: continue raise if ret == 127: continue return True class urllister(SGMLParser): def reset(self): SGMLParser.reset(self) self.urls = [] def start_a(self, attrs): href = [v for k, v in attrs if k=='href'] if href: self.urls.extend(href) class XzTarFile(tarfile.TarFile): OPEN_METH = tarfile.TarFile.OPEN_METH.copy() OPEN_METH["xz"] = "xzopen" @classmethod def xzopen(cls, name, mode="r", fileobj=None, **kwargs): """Open gzip compressed tar archive name for reading or writing. Appending is not allowed. """ if len(mode) > 1 or mode not in "rw": raise ValueError("mode must be 'r' or 'w'") if fileobj is not None: fileobj = _LMZAProxy(fileobj, mode) else: fileobj = lzma.LZMAFile(name, mode) try: # lzma doesn't immediately return an error # try and read a bit of data to determine if it is a valid xz file fileobj.read(_LZMAProxy.blocksize) fileobj.seek(0) t = cls.taropen(name, mode, fileobj, **kwargs) except IOError: raise tarfile.ReadError("not a xz file") except lzma.error: raise tarfile.ReadError("not a xz file") t._extfileobj = False return t if not hasattr(tarfile.TarFile, 'xvopen'): tarfile.open = XzTarFile.open class SpecFile(object): re_update_version = re.compile(r'^(?P
Version:\s*)(?P.+)(?P\s*)$', re.MULTILINE + re.IGNORECASE)
    re_update_release = re.compile(r'^(?P
Release:\s*)(?P%mkrel \d+)(?P\s*)$', re.MULTILINE + re.IGNORECASE)

    def __init__(self, path):
        self.path = path
        self.cwd = os.path.dirname(path)

    @property
    def version(self):
        return subprocess.check_output(["rpm", "--specfile", self.path, "--queryformat", "%{VERSION}\n"]).splitlines()[0]

    def update(self, version):
        """Update specfile (increase version)"""
        cur_version = self.version

        compare = version_cmp(version, cur_version)

        if compare == 0:
            print >>sys.stderr, "ERROR: Already at version %s!" % (cur_version)
            return False

        if compare != 1:
            print >>sys.stderr, "ERROR: Version %s is older than current version %s!" % (version, cur_version)
            return False

        # XXX - os.path.join is hackish
        if subprocess.check_output(["svn", "diff", os.path.join(self.path, '..')]) != '':
            print >>sys.stderr, "ERROR: Package has uncommitted changes!"
            return False

        with open(self.path, "rw") as f:
            data = f.read()

            if data.count("%mkrel") != 1:
                print >>sys.stderr, "ERROR: Multiple %mkrel found; don't know what to do!"
                return False

            data, nr = self.re_update_version.subn(r'\g
%s\g' % version, data, 1)
            if nr != 1:
                print >>sys.stderr, "ERROR: Could not increase version!"
                return False

            data, nr = self.re_update_release.subn(r'\g
%mkrel 1\g', data, 1)
            if nr != 1:
                print >>sys.stderr, "ERROR: Could not reset release!"
                return False

            # Overwrite file with new version number
            write_file(self.path, data)


        # Verify that RPM also agrees that version number has changed
        if self.version != version:
            print "ERROR: Increased version to %s, but RPM doesn't agree!?!" % version
            return False

        try:
            # Download new tarball
            subprocess.check_call(['mgarepo', 'sync', '-d'], cwd=self.cwd)
            # Check patches still apply
            subprocess.check_call(['bm', '-p', '--nodeps'], cwd=self.cwd)
        except subprocess.CalledProcessError:
            return False

        return True

class Patch(object):
    """Do things with patches"""

    re_dep3 = re.compile(r'^(?:#\s*)?(?P
[-A-Za-z0-9]+?):\s*(?P.*)$') re_dep3_cont = re.compile(r'^#?\s+(?P.*)$') def __init__(self, path, show_path=False): """Path: path to patch (might not exist)""" self.path = path self.show_path = show_path def __str__(self): return self.path if self.show_path else os.path.basename(self.path) def add_dep3(self): """Add DEP-3 headers to a patch file""" if self.dep3['valid']: return False new_headers = ( ('Author', self.svn_author), ('Subject', ''), ('Applied-Upstream', ''), ('Forwarded', ''), ('Bug', ''), ) with tempfile.NamedTemporaryFile(dir=os.path.dirname(self.path), delete=False) as fdst: with open(self.path, "r") as fsrc: # Start with any existing DEP3 headers for i in range(self.dep3['last_nr']): fdst.write(fsrc.read()) # After that add the DEP3 headers add_line = False for header, data in new_headers: if header in self.dep3['headers']: continue # XXX - wrap this at 80 chars add_line = True print >>fdst, "%s: %s" % (header, "" if data is None else data) if add_line: print >>fdst, "" # Now copy any other data and the patch shutil.copyfileobj(fsrc, fdst) fdst.flush() os.rename(fdst.name, self.path) call_editor(self.path) #Author: fwang #Subject: Build fix: Fix glib header inclusion #Applied-Upstream: commit:30602 #Forwarded: yes #Bug: http://bugzilla.abisource.com/show_bug.cgi?id=13247 def _read_dep3(self): """Read DEP-3 headers from an existing patch file This will also parse git headers""" dep3 = {} headers = {} last_header = None last_nr = 0 nr = 0 try: with open(self.path, "r") as f: for line in line_input(f): nr += 1 # stop trying to parse when real patch begins if line == '---': break r = self.re_dep3.match(line) if r: info = r.groupdict() # Avoid matching URLS if info['data'].startswith('//') and info['header'].lower () == info['header']: continue headers[info['header']] = info['data'] last_header = info['header'] last_nr = nr continue r = self.re_dep3_cont.match(line) if r: info = r.groupdict() if last_header: headers[last_header] = " ".join((headers[last_header], info['data'])) last_nr = nr continue last_header = None except IOError: pass dep3['valid'] = \ (('Description' in headers and headers['Description'].strip() != '') or ('Subject' in headers and headers['Subject'].strip() != '')) \ and (('Origin' in headers and headers['Origin'].strip() != '') \ or ('Author' in headers and headers['Author'].strip() != '') \ or ('From' in headers and headers['From'].strip() != '')) dep3['last_nr'] = last_nr dep3['headers'] = headers self._dep3 = dep3 @property def dep3(self): if not hasattr(self, '_dep3'): self._read_dep3() return self._dep3 @property def svn_author(self): if not hasattr(self, '_svn_author'): p = subprocess.Popen(['svn', 'log', '-q', "--", self.path], stdout=subprocess.PIPE, close_fds=True) contents = p.stdout.read().strip("\n").splitlines() ecode = p.wait() if ecode == 0: for line in contents: if ' | ' not in line: continue fields = line.split(' | ') if len(fields) >= 3: self._svn_author = fields[1] if not hasattr(self, '_svn_author'): return None return self._svn_author def get_upstream_names(): urlopen = urllib2.build_opener() good_dir = re.compile('^[-A-Za-z0-9_+.]+/$') # Get the files usock = urlopen.open(URL) parser = urllister() parser.feed(usock.read()) usock.close() parser.close() files = parser.urls tarballs = set([filename.replace('/', '') for filename in files if good_dir.search(filename)]) return tarballs def get_downstream_names(): re_file = re.compile(r'^(?P.*?)[_-](?:(?P([0-9]+[\.])*[0-9]+)-)?(?P([0-9]+[\.\-])*[0-9]+)\.(?P(?:tar\.|diff\.)?[a-z][a-z0-9]*)$') p = subprocess.Popen(['urpmf', '--files', '.', "--media", MEDIA], stdout=subprocess.PIPE, close_fds=True) contents = p.stdout.read().strip("\n").splitlines() ecode = p.wait() if ecode != 0: sys.exit(1) FILES = {} TARBALLS = {} for line in contents: try: srpm, filename = line.split(":") except ValueError: print >>sys.stderr, line continue if '.tar' in filename: r = re_file.match(filename) if r: fileinfo = r.groupdict() module = fileinfo['module'] if module not in TARBALLS: TARBALLS[module] = set() TARBALLS[module].add(srpm) if srpm not in FILES: FILES[srpm] = set() FILES[srpm].add(filename) return TARBALLS, FILES def write_file(path, data): with tempfile.NamedTemporaryFile(dir=os.path.dirname(path), delete=False) as fdst: fdst.write(data) fdst.flush() os.rename(fdst.name, path) def cmd_co(options, parser): upstream = get_upstream_names() downstream, downstream_files = get_downstream_names() cwd = os.path.expanduser(PKGROOT) matches = upstream & set(downstream.keys()) for module in matches: print module, "\t".join(downstream[module]) for package in downstream[module]: subprocess.call(['mgarepo', 'co', package], cwd=cwd) def cmd_ls(options, parser): upstream = get_upstream_names() downstream, downstream_files = get_downstream_names() matches = upstream & set(downstream.keys()) for module in matches: print "\n".join(downstream[module]) def cmd_patches(options, parser): upstream = get_upstream_names() downstream, downstream_files = get_downstream_names() path = os.path.expanduser(PKGROOT) import pprint matches = upstream & set(downstream.keys()) for module in sorted(matches): for srpm in downstream[module]: for filename in downstream_files[srpm]: if '.patch' in filename or '.diff' in filename: p = Patch(os.path.join(path, srpm, "SOURCES", filename), show_path=options.path) valid = "" forwarded = "" if p.dep3['headers']: forwarded = p.dep3['headers'].get('Forwarded', "no") if p.dep3['valid']: valid="VALID" print "\t".join((module, srpm, str(p), forwarded, valid)) def cmd_dep3(options, parser): p = Patch(options.patch) p.add_dep3() def cmd_package_new_version(options, parser): package = options.package root = os.path.expanduser(PKGROOT) cwd = os.path.join(root, package) try: subprocess.check_call(['mgarepo', 'co', package], cwd=root) except subprocess.CalledProcessError: sys.exit(1) s = SpecFile(os.path.join(cwd, "SPECS", "%s.spec" % package)) print "%s => %s" % (s.version, options.version) if not s.update(options.version): sys.exit(1) if options.submit: try: # checkin changes subprocess.check_call(['mgarepo', 'ci', '-m', 'new version'], cwd=cwd) # and submit subprocess.check_call(['mgarepo', 'submit'], cwd=cwd) except subprocess.CalledProcessError: sys.exit(1) def main(): description = """Mageia GNOME commands.""" epilog="""Report bugs to Olav Vitters""" parser = argparse.ArgumentParser(description=description,epilog=epilog) # SUBPARSERS subparsers = parser.add_subparsers(title='subcommands') # install subparser = subparsers.add_parser('co', help='checkout all GNOME modules') subparser.set_defaults( func=cmd_co ) subparser = subparsers.add_parser('packages', help='list all GNOME packages') subparser.set_defaults( func=cmd_ls ) subparser = subparsers.add_parser('patches', help='list all GNOME patches') subparser.add_argument("-p", "--path", action="store_true", dest="path", help="Show full path to patch") subparser.set_defaults( func=cmd_patches, path=False ) subparser = subparsers.add_parser('dep3', help='Add dep3 headers') subparser.add_argument("patch", help="Patch") subparser.set_defaults( func=cmd_dep3, path=False ) subparser = subparsers.add_parser('increase', help='Increase version number') subparser.add_argument("package", help="Package name") subparser.add_argument("version", help="Version number") subparser.add_argument("-s", "--submit", action="store_true", dest="submit", help="Commit changes and submit") subparser.set_defaults( func=cmd_package_new_version, submit=False ) if len(sys.argv) == 1: parser.print_help() sys.exit(2) options = parser.parse_args() try: options.func(options, parser) except KeyboardInterrupt: print('Interrupted') sys.exit(1) except EOFError: print('EOF') sys.exit(1) except IOError, e: if e.errno != errno.EPIPE: raise sys.exit(0) if __name__ == "__main__": main()