/[soft]/mga-gnome/trunk/mga-gnome
ViewVC logotype

Diff of /mga-gnome/trunk/mga-gnome

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 3555 by ovitters, Sun Mar 18 11:08:07 2012 UTC revision 3650 by ovitters, Wed Mar 21 19:34:00 2012 UTC
# Line 49  import datetime Line 49  import datetime
49  MEDIA="Core Release Source"  MEDIA="Core Release Source"
50  URL="http://download.gnome.org/sources/"  URL="http://download.gnome.org/sources/"
51  PKGROOT='~/pkgs'  PKGROOT='~/pkgs'
52  SLEEP_INITIAL=300  SLEEP_INITIAL=180
53  SLEEP_REPEAT=30  SLEEP_REPEAT=30
54  SLEEP_TIMES=20  SLEEP_TIMES=20
55    
# Line 274  class SpecFile(object): Line 274  class SpecFile(object):
274              if not force: return False              if not force: return False
275    
276          # XXX - os.path.join is hackish          # XXX - os.path.join is hackish
277          if subprocess.check_output(["svn", "diff", os.path.join(self.path, '..')]) != '':          svn_diff_output = subprocess.check_output(["svn", "diff", os.path.join(self.path, '..')])
278            if svn_diff_output != '':
279                print svn_diff_output
280              print >>sys.stderr, "ERROR: Package has uncommitted changes!"              print >>sys.stderr, "ERROR: Package has uncommitted changes!"
281              return False              if not force:
282                    return False
283    
284                # Forcing package submission: revert changes
285                try:
286                    print >>sys.stderr, "WARNING: Force used; reverting svn changes"
287                    subprocess.check_call(["svn", "revert", "-R", os.path.join(self.path, '..')])
288                except subprocess.CalledProcessError:
289                    return False
290    
291          with open(self.path, "rw") as f:          with open(self.path, "rw") as f:
292              data = f.read()              data = f.read()
# Line 309  class SpecFile(object): Line 319  class SpecFile(object):
319          tries = 0          tries = 0
320          while tries < SLEEP_TIMES:          while tries < SLEEP_TIMES:
321              tries += 1              tries += 1
322                if tries > 1: time.sleep(SLEEP_REPEAT)
323              try:              try:
324                  # Download new tarball                  # Download new tarball
325                  subprocess.check_call(['mgarepo', 'sync', '-d'], cwd=self.cwd)                  subprocess.check_call(['mgarepo', 'sync', '-d'], cwd=self.cwd)
326                    # success, so exit loop
327                  break                  break
328              except subprocess.CalledProcessError, e:              except subprocess.CalledProcessError, e:
329                  # mgarepo sync returns 1 if the tarball cannot be downloaded                  # mgarepo sync returns 1 if the tarball cannot be downloaded
330                  if e.returncode != 1:                  if e.returncode != 1:
331                      return False                      return False
332            else:
333                return False
334    
                 if tries == SLEEP_TIMES:  
                     return False  
   
                 time.sleep(SLEEP_REPEAT)  
335    
336          try:          try:
337              # Check patches still apply              # Check patches still apply
338              subprocess.check_call(['bm', '-p', '--nodeps'], cwd=self.cwd)              subprocess.check_call(['bm', '-p', '--nodeps'], cwd=self.cwd)
339          except subprocess.CalledProcessError:          except subprocess.CalledProcessError:
340                logfile = os.path.join(os.path.dirname(self.path), 'log.%s' % os.path.splitext(os.path.basename(self.path))[0])
341                if os.path.exists(logfile):
342                    subprocess.call(['tail', '-n', '15', logfile])
343              return False              return False
344    
345          return True          return True
# Line 471  class Patch(object): Line 484  class Patch(object):
484    
485          return self._svn_author          return self._svn_author
486    
 def get_upstream_names():  
     urlopen = urllib2.build_opener()  
487    
488      good_dir = re.compile('^[-A-Za-z0-9_+.]+/$')  class Upstream(object):
489    
490        limit = None
491    
492        def __init__(self):
493            urlopen = urllib2.build_opener()
494    
495            good_dir = re.compile('^[-A-Za-z0-9_+.]+/$')
496    
497      # Get the files          # Get the files
498      usock = urlopen.open(URL)          usock = urlopen.open(URL)
499      parser = urllister()          parser = urllister()
500      parser.feed(usock.read())          parser.feed(usock.read())
501      usock.close()          usock.close()
502      parser.close()          parser.close()
503      files = parser.urls          files = parser.urls
504    
505      tarballs = set([filename.replace('/', '') for filename in files if good_dir.search(filename)])          tarballs = set([filename.replace('/', '') for filename in files if good_dir.search(filename)])
506            if self.limit is not None:
507                tarballs.intersection_update(self.limit)
508    
509      return tarballs          self.names = tarballs
510    
511  def get_downstream_names():  class Downstream(object):
512      re_file = re.compile(r'^(?P<module>.*?)[_-](?:(?P<oldversion>([0-9]+[\.])*[0-9]+)-)?(?P<version>([0-9]+[\.\-])*[0-9]+)\.(?P<format>(?:tar\.|diff\.)?[a-z][a-z0-9]*)$')      re_file = re.compile(r'^(?P<module>.*?)[_-](?:(?P<oldversion>([0-9]+[\.])*[0-9]+)-)?(?P<version>([0-9]+[\.\-])*[0-9]+)\.(?P<format>(?:tar\.|diff\.)?[a-z][a-z0-9]*)$')
513    
514      contents = subprocess.check_output(['urpmf', '--qf', '%name|%version|%files', '.', "--media", MEDIA], close_fds=True).strip("\n").splitlines()      def __init__(self):
515            contents = subprocess.check_output(['urpmf', '--qf', '%name|%version|%files', '.', "--media", MEDIA], close_fds=True).strip("\n").splitlines()
516    
517      FILES = {}          FILES = {}
518      TARBALLS = {}          TARBALLS = {}
519    
520      for line in  contents:          for line in  contents:
521          try:              try:
522              srpm, version, filename = line.split("|")                  srpm, version, filename = line.split("|")
523          except ValueError:              except ValueError:
524              print >>sys.stderr, line                  print >>sys.stderr, line
525              continue                  continue
526    
527          if '.tar' in filename:              if '.tar' in filename:
528              r = re_file.match(filename)                  r = self.re_file.match(filename)
529              if r:                  if r:
530                  fileinfo = r.groupdict()                      fileinfo = r.groupdict()
531                  module = fileinfo['module']                      module = fileinfo['module']
532    
533                  if module not in TARBALLS:                      if module not in TARBALLS:
534                      TARBALLS[module] = {}                          TARBALLS[module] = {}
535                  TARBALLS[module][srpm] = version                      TARBALLS[module][srpm] = version
536    
537          if srpm not in FILES:              if srpm not in FILES:
538              FILES[srpm] = set()                  FILES[srpm] = set()
539          FILES[srpm].add(filename)              FILES[srpm].add(filename)
540    
541      return TARBALLS, FILES          self.tarballs = TARBALLS
542            self.files = FILES
543    
544  def get_downstream_from_upstream(upstream, version):  def get_downstream_from_upstream(upstream, version):
545      # Determine the package name      # Determine the package name
546      downstream, downstream_files = get_downstream_names()      downstream = Downstream()
547    #    downstream, downstream_files = get_downstream_names()
548    
549      if upstream not in downstream:      if upstream not in downstream:
550          raise ValueError("No packages for upstream name: %s" % upstream)          raise ValueError("No packages for upstream name: %s" % upstream)
551    
552      if len(downstream[upstream]) != 1:      if len(downstream.tarballs[upstream]) == 1:
553          # XXX - Make it more intelligent          return downstream.tarballs[upstream].keys()
         raise ValueError("Multiple packages found for %s: %s" % (upstream, ", ".join(downstream[upstream].keys())))  
554    
555      return downstream[upstream].keys()      # Directories packages are located in
556        root = os.path.expanduser(PKGROOT)
557    
558        packages = {}
559        for package in downstream.tarballs[upstream].keys():
560            cwd = os.path.join(root, package)
561    
562            # Checkout package to ensure the checkout reflects the latest changes
563            try:
564                subprocess.check_call(['mgarepo', 'co', package], cwd=root)
565            except subprocess.CalledProcessError:
566                raise ValueError("Multiple packages found and cannot checkout %s" % package)
567    
568            # Determine version from spec file
569            try:
570                packages[package] = SpecFile(os.path.join(cwd, "SPECS", "%s.spec" % package)).version
571            except subprocess.CalledProcessError:
572                raise ValueError("Multiple packages found and cannot determine version of %s" % package)
573    
574        # Return all packages reflecting the current version
575        matches = [package for package in packages if packages[package] == version]
576        if len(matches):
577            return matches
578    
579        # Return all packages reflecting the version before the current version
580        latest_version = get_latest_version(packages.values(), max_version=version)
581        matches = [package for package in packages if packages[package] == latest_version]
582        if len(matches):
583            return matches
584    
585        # Give up
586        raise ValueError("Multiple packages found and cannot determine package for version %s" % version)
587    
588  def write_file(path, data):  def write_file(path, data):
589      with tempfile.NamedTemporaryFile(dir=os.path.dirname(path), delete=False) as fdst:      with tempfile.NamedTemporaryFile(dir=os.path.dirname(path), delete=False) as fdst:
# Line 548  def cmd_co(options, parser): Line 601  def cmd_co(options, parser):
601  def join_streams(show_version=False, only_diff_version=False):  def join_streams(show_version=False, only_diff_version=False):
602      root = os.path.expanduser(PKGROOT)      root = os.path.expanduser(PKGROOT)
603    
604      upstream = get_upstream_names()      upstream = Upstream().names
605      downstream, downstream_files = get_downstream_names()      downstream = Downstream()
606    
607      matches = upstream & set(downstream.keys())      matches = upstream & set(downstream.tarballs.keys())
608      for module in matches:      for module in matches:
609          for package in downstream[module].keys():          for package in downstream.tarballs[module].keys():
610              package_version = downstream[module][package]              package_version = downstream.tarballs[module][package]
611              spec_version = None              spec_version = None
612              if show_version or only_diff_version:              if show_version or only_diff_version:
613                  cwd = os.path.join(root, package)                  cwd = os.path.join(root, package)
# Line 566  def join_streams(show_version=False, onl Line 619  def join_streams(show_version=False, onl
619              if only_diff_version and package_version == spec_version:              if only_diff_version and package_version == spec_version:
620                  continue                  continue
621    
622              yield (package, module, package_version, spec_version, downstream_files[package])              yield (package, module, package_version, spec_version, downstream.files[package])
623    
624  def cmd_ls(options, parser):  def cmd_ls(options, parser):
625      for package, module, package_version, spec_version, downstream_files in sorted(join_streams(show_version=options.show_version, only_diff_version=options.diff)):      for package, module, package_version, spec_version, downstream_files in sorted(join_streams(show_version=options.show_version, only_diff_version=options.diff)):
626          print package,"\t",          sys.stdout.write(package)
627          if options.upstream: print module, "\t",          if options.upstream: sys.stdout.write("\t%s" % module)
628          if options.show_version: print spec_version, "\t", package_version, "\t",          if options.show_version: sys.stdout.write("\t%s\t%s" % (spec_version, package_version))
629          print          print
630    
631  def cmd_patches(options, parser):  def cmd_patches(options, parser):
# Line 630  def cmd_package_new_version(options, par Line 683  def cmd_package_new_version(options, par
683              sys.stderr(1)              sys.stderr(1)
684    
685          for filename in sources:          for filename in sources:
686              if not is_valid_hash(os.path.join(cwd, "SOURCES", filename), options.algo, options.hexdigest):              path = os.path.join(cwd, "SOURCES", filename)
687                if not is_valid_hash(path, options.algo, options.hexdigest):
688                  print >>sys.stderr, "ERROR: Hash file failed check for %s!" % path                  print >>sys.stderr, "ERROR: Hash file failed check for %s!" % path
689                  print >>sys.stderr, "ERROR: Reverting changes!"                  print >>sys.stderr, "ERROR: Reverting changes!"
690                  subprocess.call(['svn', 'revert', '-R', cwd], cwd=cwd)                  subprocess.call(['svn', 'revert', '-R', cwd], cwd=cwd)
# Line 642  def cmd_package_new_version(options, par Line 696  def cmd_package_new_version(options, par
696              # checkin changes              # checkin changes
697              subprocess.check_call(['mgarepo', 'ci', '-m', 'new version %s' % options.version], cwd=cwd)              subprocess.check_call(['mgarepo', 'ci', '-m', 'new version %s' % options.version], cwd=cwd)
698              # and submit              # and submit
699              subprocess.check_call(['mgarepo', 'submit'], cwd=cwd)              # XXX HACK NOT TO AUTO SUBMIT ATM
700                if options.hexdigest is None:
701                    subprocess.check_call(['mgarepo', 'submit'], cwd=cwd)
702          except subprocess.CalledProcessError:          except subprocess.CalledProcessError:
703              sys.exit(1)              sys.exit(1)
704    
705  def cmd_parse_ftp_release_list(options, parser):  def cmd_parse_ftp_release_list(options, parser):
706      def _send_reply_mail(contents, orig_msg, to, error=False):      def _send_reply_mail(contents, orig_msg, to, packages=[], error=False):
707          """Send an reply email"""          """Send an reply email"""
708          contents.seek(0)          contents.seek(0)
709          msg = MIMEText(contents.read(), _charset='utf-8')          msg = MIMEText(contents.read(), _charset='utf-8')
710    
711          if error:          if error:
712              # XXX - ugly              # XXX - ugly
713              contents.seek(0)              contents.seek(0)
714              lastline = contents.read().splitlines()[-1]              lastline = contents.read().rstrip().splitlines()[-1]
715              # Remove things like "ERROR: " and so on from the last line              # Remove things like "ERROR: " and so on from the last line
716              lastline = re.sub(r'^(?:[^ :]+:\s+)+', '', lastline)              lastline = re.sub(r'^(?:[^ :]+:\s+)+', '', lastline)
717                # Remove things like "   - " (youri output from mgarepo submit)
718                lastline = re.sub(r'^\s+-\s+', '', lastline)
719              subjecterror = " (ERROR: %s)" % lastline if lastline else " (ERROR)"              subjecterror = " (ERROR: %s)" % lastline if lastline else " (ERROR)"
720          else:          else:
721              subjecterror = ""              subjecterror = ""
722          msg['Subject'] = "Re: %s%s" % (orig_msg['Subject'], subjecterror)  
723            if packages:
724                subject = "%s %s%s" % (", ".join(packages), orig_msg['X-Module-Version'], subjecterror)
725            else:
726                subject = "Re: %s%s" % (orig_msg['Subject'], subjecterror)
727    
728            msg['Subject'] = subject
729          msg['To'] = to          msg['To'] = to
730          msg["In-Reply-To"] = orig_msg["Message-ID"]          msg["In-Reply-To"] = orig_msg["Message-ID"]
731          msg["References"] = orig_msg["Message-ID"]          msg["References"] = orig_msg["Message-ID"]
# Line 704  def cmd_parse_ftp_release_list(options, Line 769  def cmd_parse_ftp_release_list(options,
769          # maildrop aborts and will try to deliver after 5min          # maildrop aborts and will try to deliver after 5min
770          # fork to avoid this          # fork to avoid this
771          if os.fork() != 0: sys.exit(0)          if os.fork() != 0: sys.exit(0)
772          time.sleep(SLEEP_INITIAL)          # wait SLEEP_INITIAL after the message was sent
773            secs = SLEEP_INITIAL
774            t = email.utils.parsedate_tz(msg['Date'])
775            if t is not None:
776                msg_time = email.utils.mktime_tz(t)
777                secs = SLEEP_INITIAL - (time.time() - msg_time)
778    
779            if secs > 0: time.sleep(secs)
780    
781      error = False      error = False
782      for package in packages:      for package in packages:
783          if subprocess.call(['mga-gnome', 'increase', '--submit', '--hash', hexdigest, package, version], stdout=stdout, stderr=stderr):          cmd = ['mga-gnome', 'increase', '--submit', '--hash', hexdigest]
784            if options.force:
785                cmd.append('--force')
786            cmd.extend((package, version))
787            if subprocess.call(cmd, stdout=stdout, stderr=stderr):
788              error = True              error = True
789    
790      if options.mail: _send_reply_mail(stdout, msg, options.mail, error=error)      if options.mail: _send_reply_mail(stdout, msg, options.mail, packages=packages, error=error)
791    
792  def main():  def main():
793      description = """Mageia GNOME commands."""      description = """Mageia GNOME commands."""
794      epilog="""Report bugs to Olav Vitters"""      epilog="""Report bugs to Olav Vitters"""
795      parser = argparse.ArgumentParser(description=description,epilog=epilog)      parser = argparse.ArgumentParser(description=description,epilog=epilog)
796        parser.add_argument("-l", "--limit", type=argparse.FileType('r', 0),
797                            dest="limit_upstream", metavar="FILE",
798                            help="File containing upstream names")
799    
800      # SUBPARSERS      # SUBPARSERS
801      subparsers = parser.add_subparsers(title='subcommands')      subparsers = parser.add_subparsers(title='subcommands')
# Line 772  def main(): Line 851  def main():
851      subparser.add_argument("-m", "--mail", help="Email address to send the progress to")      subparser.add_argument("-m", "--mail", help="Email address to send the progress to")
852      subparser.add_argument("-w", "--wait", action="store_true",      subparser.add_argument("-w", "--wait", action="store_true",
853                                   help="Wait before trying to retrieve the new version")                                   help="Wait before trying to retrieve the new version")
854        subparser.add_argument("-f", "--force", action="store_true",
855                                     help="Force submission")
856      subparser.set_defaults(      subparser.set_defaults(
857          func=cmd_parse_ftp_release_list          func=cmd_parse_ftp_release_list, force=False, wait=False
858      )      )
859    
860      if len(sys.argv) == 1:      if len(sys.argv) == 1:
# Line 781  def main(): Line 862  def main():
862          sys.exit(2)          sys.exit(2)
863    
864      options = parser.parse_args()      options = parser.parse_args()
865        if options.limit_upstream:
866            Upstream.limit = set(options.limit_upstream.read().strip("\n").splitlines())
867    
868      try:      try:
869          options.func(options, parser)          options.func(options, parser)
# Line 796  def main(): Line 879  def main():
879          sys.exit(0)          sys.exit(0)
880    
881  if __name__ == "__main__":  if __name__ == "__main__":
882        os.environ['PYTHONUNBUFFERED'] = '1'
883      main()      main()
Legend:
Removed from v.3555  
changed lines
  Added in v.3650
  ViewVC Help
Powered by ViewVC 1.1.30