/[soft]/mga-gnome/trunk/mga-gnome
ViewVC logotype

Diff of /mga-gnome/trunk/mga-gnome

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 3648 by ovitters, Wed Mar 21 16:16:55 2012 UTC revision 5287 by ovitters, Fri Aug 3 05:18:24 2012 UTC
# Line 46  import time Line 46  import time
46  # version freeze  # version freeze
47  import datetime  import datetime
48    
49  MEDIA="Core Release Source"  # packages --sort
50  URL="http://download.gnome.org/sources/"  import itertools
51  PKGROOT='~/pkgs'  
52  SLEEP_INITIAL=180  SLEEP_INITIAL=180
53  SLEEP_REPEAT=30  SLEEP_REPEAT=30
54  SLEEP_TIMES=20  SLEEP_TIMES=20
# Line 484  class Patch(object): Line 484  class Patch(object):
484    
485          return self._svn_author          return self._svn_author
486    
 def get_upstream_names():  
     urlopen = urllib2.build_opener()  
487    
488      good_dir = re.compile('^[-A-Za-z0-9_+.]+/$')  class Upstream(object):
489    
490      # Get the files      URL="http://download.gnome.org/sources/"
491      usock = urlopen.open(URL)      limit = None
     parser = urllister()  
     parser.feed(usock.read())  
     usock.close()  
     parser.close()  
     files = parser.urls  
492    
493      tarballs = set([filename.replace('/', '') for filename in files if good_dir.search(filename)])      def __init__(self):
494            urlopen = urllib2.build_opener()
495    
496      return tarballs          good_dir = re.compile('^[-A-Za-z0-9_+.]+/$')
497    
498  def get_downstream_names():          # Get the files
499      re_file = re.compile(r'^(?P<module>.*?)[_-](?:(?P<oldversion>([0-9]+[\.])*[0-9]+)-)?(?P<version>([0-9]+[\.\-])*[0-9]+)\.(?P<format>(?:tar\.|diff\.)?[a-z][a-z0-9]*)$')          usock = urlopen.open(self.URL)
500            parser = urllister()
501            parser.feed(usock.read())
502            usock.close()
503            parser.close()
504            files = parser.urls
505    
506      contents = subprocess.check_output(['urpmf', '--qf', '%name|%version|%files', '.', "--media", MEDIA], close_fds=True).strip("\n").splitlines()          tarballs = set([filename.replace('/', '') for filename in files if good_dir.search(filename)])
507            if self.limit is not None:
508                tarballs.intersection_update(self.limit)
509    
510      FILES = {}          self.names = tarballs
     TARBALLS = {}  
511    
512      for line in  contents:  class Downstream(object):
513          try:      re_file = re.compile(r'^(?P<module>.*?)[_-](?:(?P<oldversion>([0-9]+[\.])*[0-9]+)-)?(?P<version>([0-9]+[\.\-])*[0-9]+)\.(?P<format>(?:tar\.|diff\.)?[a-z][a-z0-9]*)$')
             srpm, version, filename = line.split("|")  
         except ValueError:  
             print >>sys.stderr, line  
             continue  
514    
515          if '.tar' in filename:      MEDIA="Core Release Source"
516              r = re_file.match(filename)      PKGROOT='~/pkgs'
517              if r:      DISTRO=None
                 fileinfo = r.groupdict()  
                 module = fileinfo['module']  
   
                 if module not in TARBALLS:  
                     TARBALLS[module] = {}  
                 TARBALLS[module][srpm] = version  
   
         if srpm not in FILES:  
             FILES[srpm] = set()  
         FILES[srpm].add(filename)  
518    
519      return TARBALLS, FILES      def __init__(self):
520            contents = subprocess.check_output(['urpmf', '--qf', '%name|%version|%files', '.', "--media", self.MEDIA], close_fds=True).strip("\n").splitlines()
521    
522  def get_downstream_from_upstream(upstream, version):          FILES = {}
523      # Determine the package name          TARBALLS = {}
     downstream, downstream_files = get_downstream_names()  
524    
525      if upstream not in downstream:          for line in  contents:
526          raise ValueError("No packages for upstream name: %s" % upstream)              try:
527                    srpm, version, filename = line.split("|")
528                except ValueError:
529                    print >>sys.stderr, line
530                    continue
531    
532      if len(downstream[upstream]) == 1:              if '.tar' in filename:
533          return downstream[upstream].keys()                  r = self.re_file.match(filename)
534                    if r:
535                        fileinfo = r.groupdict()
536                        module = fileinfo['module']
537    
538                        if module not in TARBALLS:
539                            TARBALLS[module] = {}
540                        TARBALLS[module][srpm] = version
541    
542                if srpm not in FILES:
543                    FILES[srpm] = set()
544                FILES[srpm].add(filename)
545    
546      # Directories packages are located in          self.tarballs = TARBALLS
547      root = os.path.expanduser(PKGROOT)          self.files = FILES
548    
549      packages = {}      @classmethod
550      for package in downstream[upstream].keys():      def co(cls, package, cwd=None):
551          cwd = os.path.join(root, package)          if cwd is None:
552                cwd = os.path.expanduser(cls.PKGROOT)
553    
554            cmd = ['mgarepo', 'co']
555            if cls.DISTRO:
556                cmd.extend(('-d', cls.DISTRO))
557            cmd.append(package)
558            return subprocess.check_call(cmd, cwd=cwd)
559    
560        def get_downstream_from_upstream(self, upstream, version):
561            if upstream not in self.tarballs:
562                raise ValueError("No packages for upstream name: %s" % upstream)
563    
564            if len(self.tarballs[upstream]) == 1:
565                return self.tarballs[upstream].keys()
566    
567            # Directories packages are located in
568            root = os.path.expanduser(self.PKGROOT)
569    
570            packages = {}
571            for package in self.tarballs[upstream].keys():
572                cwd = os.path.join(root, package)
573    
574          # Checkout package to ensure the checkout reflects the latest changes              # Checkout package to ensure the checkout reflects the latest changes
575          try:              try:
576              subprocess.check_call(['mgarepo', 'co', package], cwd=root)                  self.co(package, cwd=root)
577          except subprocess.CalledProcessError:              except subprocess.CalledProcessError:
578              raise ValueError("Multiple packages found and cannot checkout %s" % package)                  raise ValueError("Multiple packages found and cannot checkout %s" % package)
579    
580          # Determine version from spec file              # Determine version from spec file
581          try:              try:
582              packages[package] = SpecFile(os.path.join(cwd, "SPECS", "%s.spec" % package)).version                  packages[package] = SpecFile(os.path.join(cwd, "SPECS", "%s.spec" % package)).version
583          except subprocess.CalledProcessError:              except subprocess.CalledProcessError:
584              raise ValueError("Multiple packages found and cannot determine version of %s" % package)                  raise ValueError("Multiple packages found and cannot determine version of %s" % package)
585    
586      # Return all packages reflecting the current version          # Return all packages reflecting the current version
587      matches = [package for package in packages if packages[package] == version]          matches = [package for package in packages if packages[package] == version]
588      if len(matches):          if len(matches):
589          return matches              return matches
590    
591      # Return all packages reflecting the version before the current version          # Return all packages reflecting the version before the current version
592      latest_version = get_latest_version(packages.values(), max_version=version)          latest_version = get_latest_version(packages.values(), max_version=version)
593      matches = [package for package in packages if packages[package] == latest_version]          matches = [package for package in packages if packages[package] == latest_version]
594      if len(matches):          if len(matches):
595          return matches              return matches
596    
597      # Give up          # Give up
598      raise ValueError("Multiple packages found and cannot determine package for version %s" % version)          raise ValueError("Multiple packages found and cannot determine package for version %s" % version)
599    
600  def write_file(path, data):  def write_file(path, data):
601      with tempfile.NamedTemporaryFile(dir=os.path.dirname(path), delete=False) as fdst:      with tempfile.NamedTemporaryFile(dir=os.path.dirname(path), delete=False) as fdst:
# Line 582  def write_file(path, data): Line 604  def write_file(path, data):
604          os.rename(fdst.name, path)          os.rename(fdst.name, path)
605    
606  def cmd_co(options, parser):  def cmd_co(options, parser):
     root = os.path.expanduser(PKGROOT)  
   
607      for package, module, package_version, spec_version, downstream_files in sorted(join_streams()):      for package, module, package_version, spec_version, downstream_files in sorted(join_streams()):
608          print "%s => %s" % (module, package)          print "%s => %s" % (module, package)
609          subprocess.call(['mgarepo', 'co', package], cwd=root)          try:
610                Downstream.co(package)
611            except subprocess.CalledProcessError:
612                pass
613    
614  def join_streams(show_version=False, only_diff_version=False):  def join_streams(show_version=False, only_diff_version=False):
615      root = os.path.expanduser(PKGROOT)      root = os.path.expanduser(Downstream.PKGROOT)
616    
617      upstream = get_upstream_names()      upstream = Upstream().names
618      downstream, downstream_files = get_downstream_names()      downstream = Downstream()
619    
620      matches = upstream & set(downstream.keys())      matches = upstream & set(downstream.tarballs.keys())
621      for module in matches:      for module in matches:
622          for package in downstream[module].keys():          for package in downstream.tarballs[module].keys():
623              package_version = downstream[module][package]              package_version = downstream.tarballs[module][package]
624              spec_version = None              spec_version = None
625              if show_version or only_diff_version:              if show_version or only_diff_version:
626                  cwd = os.path.join(root, package)                  cwd = os.path.join(root, package)
# Line 609  def join_streams(show_version=False, onl Line 632  def join_streams(show_version=False, onl
632              if only_diff_version and package_version == spec_version:              if only_diff_version and package_version == spec_version:
633                  continue                  continue
634    
635              yield (package, module, package_version, spec_version, downstream_files[package])              yield (package, module, package_version, spec_version, downstream.files[package])
636    
637  def cmd_ls(options, parser):  def cmd_ls(options, parser):
638      for package, module, package_version, spec_version, downstream_files in sorted(join_streams(show_version=options.show_version, only_diff_version=options.diff)):      streams = join_streams(show_version=options.show_version, only_diff_version=options.diff)
639        if options.sort:
640            SORT=dict(zip(options.sort.read().splitlines(), itertools.count()))
641    
642            streams = sorted(streams, key=lambda a: (SORT.get(a[1], 9999), a[0]))
643        else:
644            streams = sorted(streams)
645    
646        for package, module, package_version, spec_version, downstream_files in streams:
647          sys.stdout.write(package)          sys.stdout.write(package)
648          if options.upstream: sys.stdout.write("\t%s" % module)          if options.upstream: sys.stdout.write("\t%s" % module)
649          if options.show_version: sys.stdout.write("\t%s\t%s".join((spec_version, package_version)))          if options.show_version: sys.stdout.write("\t%s\t%s" % (spec_version, package_version))
650          print          print
651    
652  def cmd_patches(options, parser):  def cmd_patches(options, parser):
653      root = os.path.expanduser(PKGROOT)      root = os.path.expanduser(Downstream.PKGROOT)
654    
655      for package, module, package_version, spec_version, downstream_files in sorted(join_streams()):      for package, module, package_version, spec_version, downstream_files in sorted(join_streams()):
656          for filename in downstream_files:          for filename in downstream_files:
# Line 642  def cmd_package_new_version(options, par Line 673  def cmd_package_new_version(options, par
673      # Determine the package name      # Determine the package name
674      if options.upstream:      if options.upstream:
675          try:          try:
676              package = get_downstream_from_upstream(options.package, options.version)[0]              package = Downstream().get_downstream_from_upstream(options.package, options.version)[0]
677          except ValueError, e:          except ValueError, e:
678              print >>sys.stderr, "ERROR: %s" % e              print >>sys.stderr, "ERROR: %s" % e
679              sys.exit(1)              sys.exit(1)
# Line 650  def cmd_package_new_version(options, par Line 681  def cmd_package_new_version(options, par
681          package = options.package          package = options.package
682    
683      # Directories packages are located in      # Directories packages are located in
684      root = os.path.expanduser(PKGROOT)      root = os.path.expanduser(Downstream.PKGROOT)
685      cwd = os.path.join(root, package)      cwd = os.path.join(root, package)
686    
687      # Checkout package to ensure the checkout reflects the latest changes      # Checkout package to ensure the checkout reflects the latest changes
688      try:      try:
689          subprocess.check_call(['mgarepo', 'co', package], cwd=root)          Downstream.co(package, cwd=root)
690      except subprocess.CalledProcessError:      except subprocess.CalledProcessError:
691          sys.exit(1)          sys.exit(1)
692    
# Line 680  def cmd_package_new_version(options, par Line 711  def cmd_package_new_version(options, par
711                  subprocess.call(['svn', 'revert', '-R', cwd], cwd=cwd)                  subprocess.call(['svn', 'revert', '-R', cwd], cwd=cwd)
712                  sys.exit(1)                  sys.exit(1)
713    
714      # We can even checkin and submit :-)      try:
715      if options.submit:          # If we made it this far, checkin the changes
716          try:          subprocess.check_call(['mgarepo', 'ci', '-m', 'new version %s' % options.version], cwd=cwd)
717              # checkin changes  
718              subprocess.check_call(['mgarepo', 'ci', '-m', 'new version %s' % options.version], cwd=cwd)          # Submit is optional
719              # and submit          if options.submit:
720              # XXX HACK NOT TO AUTO SUBMIT ATM              subprocess.check_call(['mgarepo', 'submit'], cwd=cwd)
721              if options.hexdigest is None:      except subprocess.CalledProcessError:
722                  subprocess.check_call(['mgarepo', 'submit'], cwd=cwd)          sys.exit(1)
         except subprocess.CalledProcessError:  
             sys.exit(1)  
723    
724  def cmd_parse_ftp_release_list(options, parser):  def cmd_parse_ftp_release_list(options, parser):
725      def _send_reply_mail(contents, orig_msg, to, packages=[], error=False):      def _send_reply_mail(contents, orig_msg, to, packages=[], error=False):
# Line 749  def cmd_parse_ftp_release_list(options, Line 778  def cmd_parse_ftp_release_list(options,
778          sys.exit(1)          sys.exit(1)
779    
780      try:      try:
781          packages = get_downstream_from_upstream(module, version)          packages = Downstream().get_downstream_from_upstream(module, version)
782      except ValueError, e:      except ValueError, e:
783          print >>stderr, "ERROR: %s" % e          print >>stderr, "ERROR: %s" % e
784          if options.mail: _send_reply_mail(stdout, msg, options.mail, error=True)          if options.mail: _send_reply_mail(stdout, msg, options.mail, error=True)
# Line 770  def cmd_parse_ftp_release_list(options, Line 799  def cmd_parse_ftp_release_list(options,
799    
800      error = False      error = False
801      for package in packages:      for package in packages:
802          cmd = ['mga-gnome', 'increase', '--submit', '--hash', hexdigest]          cmd = ['mga-gnome', 'increase', '--hash', hexdigest]
803            if options.submit:
804                cmd.append('--submit')
805          if options.force:          if options.force:
806              cmd.append('--force')              cmd.append('--force')
807          cmd.extend((package, version))          cmd.extend((package, version))
# Line 783  def main(): Line 814  def main():
814      description = """Mageia GNOME commands."""      description = """Mageia GNOME commands."""
815      epilog="""Report bugs to Olav Vitters"""      epilog="""Report bugs to Olav Vitters"""
816      parser = argparse.ArgumentParser(description=description,epilog=epilog)      parser = argparse.ArgumentParser(description=description,epilog=epilog)
817        parser.add_argument("-l", "--limit", type=argparse.FileType('r', 0),
818                            dest="limit_upstream", metavar="FILE",
819                            help="File containing upstream names")
820        parser.add_argument("-d", "--distro", action="store", dest="distro",
821                                           help="Distribution release")
822    
823      # SUBPARSERS      # SUBPARSERS
824      subparsers = parser.add_subparsers(title='subcommands')      subparsers = parser.add_subparsers(title='subcommands')
# Line 799  def main(): Line 835  def main():
835                                         help="Show version numbers")                                         help="Show version numbers")
836      subparser.add_argument(      "--diff", action="store_true", dest="diff",      subparser.add_argument(      "--diff", action="store_true", dest="diff",
837                                         help="Only show packages with different version")                                         help="Only show packages with different version")
838        subparser.add_argument(      "--sort", type=argparse.FileType('r', 0),
839                            dest="sort", metavar="FILE",
840                            help="Sort packages according to order in given FILE")
841    
842      subparser.set_defaults(      subparser.set_defaults(
843          func=cmd_ls, upstream=False, show_version=False, diff=False          func=cmd_ls, upstream=False, show_version=False, diff=False
844      )      )
# Line 830  def main(): Line 870  def main():
870      subparser.add_argument("--hash", dest="hexdigest",      subparser.add_argument("--hash", dest="hexdigest",
871                                         help="Hexdigest of the hash")                                         help="Hexdigest of the hash")
872      subparser.set_defaults(      subparser.set_defaults(
873          func=cmd_package_new_version, submit=False, upstream=False, hexdigest=None, algo="sha256",          func=cmd_package_new_version, submit=True, upstream=False, hexdigest=None, algo="sha256",
874          force=False          force=False
875      )      )
876    
# Line 838  def main(): Line 878  def main():
878      subparser.add_argument("-m", "--mail", help="Email address to send the progress to")      subparser.add_argument("-m", "--mail", help="Email address to send the progress to")
879      subparser.add_argument("-w", "--wait", action="store_true",      subparser.add_argument("-w", "--wait", action="store_true",
880                                   help="Wait before trying to retrieve the new version")                                   help="Wait before trying to retrieve the new version")
881        subparser.add_argument("-s", "--submit", action="store_true", dest="submit",
882                                           help="Commit changes and submit")
883      subparser.add_argument("-f", "--force", action="store_true",      subparser.add_argument("-f", "--force", action="store_true",
884                                   help="Force submission")                                   help="Force submission")
885      subparser.set_defaults(      subparser.set_defaults(
# Line 849  def main(): Line 891  def main():
891          sys.exit(2)          sys.exit(2)
892    
893      options = parser.parse_args()      options = parser.parse_args()
894        if options.limit_upstream:
895            Upstream.limit = set(options.limit_upstream.read().strip("\n").splitlines())
896    
897        if options.distro:
898            Downstream.PKGROOT = os.path.join('~/pkgs', options.distro)
899            Downstream.MEDIA = "Core Release %s Source" % options.distro
900            Downstream.DISTRO = options.distro
901    
902      try:      try:
903          options.func(options, parser)          options.func(options, parser)
Legend:
Removed from v.3648  
changed lines
  Added in v.5287
  ViewVC Help
Powered by ViewVC 1.1.30