[pkgsrc/trunk]: pkgsrc/devel/py-hg-fastimport Add options to fix up the encod...

[joerg <joerg%pkgsrc.org@localhost>]

details:   https://anonhg.NetBSD.org/pkgsrc/rev/3fde09fed0de
branches:  trunk
changeset: 365779:3fde09fed0de
user:      joerg <joerg%pkgsrc.org@localhost>
date:      Tue Jul 25 16:12:56 2017 +0000

description:
Add options to fix up the encoding of commit messages, to put the blobs
in a specific (external) location and to compress the temporary files
with gzip. Blob IDs are hashed and put in subdirectory trees of
configurable depth. Bump revision.

diffstat:

 devel/py-hg-fastimport/Makefile                                   |    5 +-
 devel/py-hg-fastimport/distinfo                                   |    5 +-
 devel/py-hg-fastimport/patches/patch-hgfastimport_____init____.py |   26 +
 devel/py-hg-fastimport/patches/patch-hgfastimport_hgimport.py     |  131 +++++++++-
 4 files changed, 159 insertions(+), 8 deletions(-)

diffs (208 lines):

diff -r 37f933d4c335 -r 3fde09fed0de devel/py-hg-fastimport/Makefile
--- a/devel/py-hg-fastimport/Makefile   Tue Jul 25 16:10:53 2017 +0000
+++ b/devel/py-hg-fastimport/Makefile   Tue Jul 25 16:12:56 2017 +0000
@@ -1,7 +1,8 @@
-# $NetBSD: Makefile,v 1.5 2017/07/22 16:16:25 joerg Exp $
+# $NetBSD: Makefile,v 1.6 2017/07/25 16:12:56 joerg Exp $
 
 DISTNAME=      danielj7-hg-fastimport-d41a42894dfa
 PKGNAME=       ${PYPKGPREFIX}-hg-fastimport-20170211
+PKGREVISION=   1
 CATEGORIES=    devel
 MASTER_SITES=  -https://bitbucket.org/danielj7/hg-fastimport/get/v20170211.tar.bz2
 EXTRACT_SUFX=  .tar.bz2
@@ -11,7 +12,7 @@
 COMMENT=       Mercurial extension for importing from a git fast-import stream
 LICENSE=       gnu-gpl-v2
 
-DEPENDS+=      ${PYPKGPREFIX}-fastimport-[0-9]*:../../devel/py-fastimport
+DEPENDS+=      ${PYPKGPREFIX}-fastimport>=0.9.6nb1:../../devel/py-fastimport
 DEPENDS+=      ${PYPKGPREFIX}-mercurial-[0-9]*:../../devel/py-mercurial
 
 USE_LANGUAGES= # none
diff -r 37f933d4c335 -r 3fde09fed0de devel/py-hg-fastimport/distinfo
--- a/devel/py-hg-fastimport/distinfo   Tue Jul 25 16:10:53 2017 +0000
+++ b/devel/py-hg-fastimport/distinfo   Tue Jul 25 16:12:56 2017 +0000
@@ -1,7 +1,8 @@
-$NetBSD: distinfo,v 1.3 2017/07/22 16:16:25 joerg Exp $
+$NetBSD: distinfo,v 1.4 2017/07/25 16:12:56 joerg Exp $
 
 SHA1 (danielj7-hg-fastimport-d41a42894dfa.tar.bz2) = 8820358107c9de776d6250052a3f15ecccfbcd44
 RMD160 (danielj7-hg-fastimport-d41a42894dfa.tar.bz2) = c6d0f6654509632f3b8266f7a4da519ff6789542
 SHA512 (danielj7-hg-fastimport-d41a42894dfa.tar.bz2) = fc207c90721dc5f32d477bfbd5c4d7634f9a16c28d6e21254bbd942cd4ad7053dbcfd3f0fed91ee797759203f4196985d5bc3132618c58331c06a8fa764124b3
 Size (danielj7-hg-fastimport-d41a42894dfa.tar.bz2) = 30701 bytes
-SHA1 (patch-hgfastimport_hgimport.py) = 42eb4f8e637cf24e52f3cabf6f499ebab9803fa7
+SHA1 (patch-hgfastimport_____init____.py) = b5fd1d95411380e88cdc032f72ef4c67cbf0e3b4
+SHA1 (patch-hgfastimport_hgimport.py) = cb630c9a8e19ea37fcbf8bae0beae485bfa96b0b
diff -r 37f933d4c335 -r 3fde09fed0de devel/py-hg-fastimport/patches/patch-hgfastimport_____init____.py
--- /dev/null   Thu Jan 01 00:00:00 1970 +0000
+++ b/devel/py-hg-fastimport/patches/patch-hgfastimport_____init____.py Tue Jul 25 16:12:56 2017 +0000
@@ -0,0 +1,26 @@
+$NetBSD: patch-hgfastimport_____init____.py,v 1.1 2017/07/25 16:12:56 joerg Exp $
+
+--- hgfastimport/__init__.py.orig      2017-02-12 01:21:40.000000000 +0000
++++ hgfastimport/__init__.py
+@@ -27,7 +27,11 @@ testedwith = '4.1'
+ @command("fastimport",
+          [('', 'branchsort', None, _('try to sort changesets by branches')),
+           ('', 'datesort', None, _('try to sort changesets by date')),
+-          ('', 'sourcesort', None, _('preserve source changesets order'))],
++          ('', 'sourcesort', None, _('preserve source changesets order')),
++          ('', 'compress', False, _('compress temporary blob files')),
++          ('', 'blob-tree', '', _('path for the blob file tree, defaults to .hg/blobs')),
++          ('', 'blob-tree-depth', 2, _('depth of the blob file tree')),
++          ('', 'fallback-message-encoding', '', _('encoding to try for non-UTF-8 messages'))],
+          _('hg fastimport SOURCE ...'),
+          norepo=False)
+ def fastimport(ui, repo, *sources, **opts):
+@@ -56,7 +60,7 @@ def fastimport(ui, repo, *sources, **opt
+ 
+     # sink is the current repo, src is the list of fastimport streams
+     destc = hg.mercurial_sink(ui, repo.root)
+-    srcc = fastimport_source(ui, repo, sources)
++    srcc = fastimport_source(ui, repo, sources, opts['fallback_message_encoding'], opts['compress'], opts['blob_tree_depth'], opts['blob_tree'])
+ 
+     # XXX figuring out sortmode copied straight from hgext/convert/convcmd.py
+     defaultsort = 'branchsort'          # for efficiency and consistency
diff -r 37f933d4c335 -r 3fde09fed0de devel/py-hg-fastimport/patches/patch-hgfastimport_hgimport.py
--- a/devel/py-hg-fastimport/patches/patch-hgfastimport_hgimport.py     Tue Jul 25 16:10:53 2017 +0000
+++ b/devel/py-hg-fastimport/patches/patch-hgfastimport_hgimport.py     Tue Jul 25 16:12:56 2017 +0000
@@ -1,11 +1,134 @@
-$NetBSD: patch-hgfastimport_hgimport.py,v 1.1 2017/07/22 16:16:25 joerg Exp $
+$NetBSD: patch-hgfastimport_hgimport.py,v 1.2 2017/07/25 16:12:56 joerg Exp $
 
---- hgfastimport/hgimport.py.orig      2017-07-22 14:10:21.608040482 +0000
+--- hgfastimport/hgimport.py.orig      2017-02-12 01:21:40.000000000 +0000
 +++ hgfastimport/hgimport.py
-@@ -449,3 +452,6 @@ class HgImportCommitHandler(processor.Co
+@@ -24,6 +24,12 @@ import os
+ import shutil
+ import stat
+ import sys
++import gzip
++import itertools
++
++def grouper(iterable, n, fillvalue=None):
++    args = [iter(iterable)] * n
++    return itertools.izip_longest(fillvalue=fillvalue, *args)
+ 
+ from hgext.convert import common, hg as converthg
+ from mercurial import util
+@@ -31,6 +37,8 @@ from mercurial.i18n import _
+ 
+ from fastimport import processor, parser
+ 
++from hashlib import sha256
++
+ # convertor source objects had a getmode() method up to Mercurial 1.5,
+ # but in 1.6 it was merged with getfile()
+ HAVE_GETMODE = hasattr(converthg.mercurial_source, 'getmode')
+@@ -39,11 +47,12 @@ class fastimport_source(common.converter
+     """Interface between the fastimport processor below and Mercurial's
+     normal conversion infrastructure.
+     """
+-    def __init__(self, ui, repo, sources):
++    def __init__(self, ui, repo, sources, fallback_message_encoding, compress, blob_tree_depth, blob_tree):
+         self.ui = ui
+         self.sources = sources
+-        self.processor = HgImportProcessor(ui, repo)
++        self.processor = HgImportProcessor(ui, repo, compress, blob_tree_depth, blob_tree)
+         self.parsed = False
++        self.fallback_message_encoding = fallback_message_encoding
+ 
+     # converter_source methods
+ 
+@@ -138,7 +147,7 @@ class fastimport_source(common.converter
+             else:
+                 infile = open(source, 'rb')
+             try:
+-                p = parser.ImportParser(infile)
++                p = parser.ImportParser(infile, message_fallback_encoding = self.fallback_message_encoding)
+                 self.processor.process(p.iter_commands)
+             finally:
+                 if infile is not sys.stdin:
+@@ -150,7 +159,7 @@ class HgImportProcessor(processor.Import
+     
+     tagprefix = "refs/tags/"
+ 
+-    def __init__(self, ui, repo):
++    def __init__(self, ui, repo, compress, blob_tree_depth, blob_tree):
+         super(HgImportProcessor, self).__init__()
+         self.ui = ui
+         self.repo = repo
+@@ -168,6 +177,10 @@ class HgImportProcessor(processor.Import
+         self.numblobs = 0               # for progress reporting
+         self.blobdir = None
+ 
++        self.compress = compress
++        self.blob_tree = blob_tree
++        self.blob_tree_depth = blob_tree_depth
++
+     def setup(self):
+         """Setup before processing any streams."""
+         pass
+@@ -187,12 +200,24 @@ class HgImportProcessor(processor.Import
+     def _getblobfilename(self, blobid):
+         if self.blobdir is None:
+             raise RuntimeError("no blobs seen, so no blob directory created")
+-        # XXX should escape ":" for windows
+-        return os.path.join(self.blobdir, "blob-" + blobid)
++        h = sha256(blobid).hexdigest()
++        h_comp = list(grouper(h, 3, ''))
++        if len(h_comp) > self.blob_tree_depth:
++            h_comp = h_comp[:self.blob_tree_depth] + [''.join(list(itertools.chain(*h_comp[self.blob_tree_depth:])))]
++        #self.ui.status("mapping blobid %s to %s\n" % (blobid, h))
++        return os.path.join(self.blobdir, *[''.join(x) for x in h_comp])
+ 
+     def getblob(self, fileid):
+         (commitid, blobid) = fileid
+-        f = open(self._getblobfilename(blobid), "rb")
++        fn = self._getblobfilename(blobid)
++        try:
++            if self.compress:
++                f = gzip.open(fn, "rb")
++            else:
++                f = open(fn, "rb")
++        except:
++            self.ui.status("missing blob %s for fileid %s\n" % (fn, fileid))
++            raise
+         try:
+             return f.read()
+         finally:
+@@ -200,13 +225,23 @@ class HgImportProcessor(processor.Import
+ 
+     def writeblob(self, blobid, data):
+         if self.blobdir is None:        # no blobs seen yet
+-            self.blobdir = os.path.join(self.repo.root, ".hg", "blobs")
+-            os.mkdir(self.blobdir)
++            if self.blob_tree:
++                self.blobdir = self.blob_tree
++            else:
++                self.blobdir = os.path.join(self.repo.root, ".hg", "blobs")
+ 
+         fn = self._getblobfilename(blobid)
+-        blobfile = open(fn, "wb")
+-        #self.ui.debug("writing blob %s to %s (%d bytes)\n"
++        try:
++            os.makedirs(os.path.dirname(fn))
++        except OSError:
++            pass
++
++        #self.ui.status("writing blob %s to %s (%d bytes)\n"
+         #              % (blobid, fn, len(data)))
++        if self.compress:
++            blobfile = gzip.open(fn, "wb", 3)
++        else:
++            blobfile = open(fn, "wb")
+         blobfile.write(data)
+         blobfile.close()
+ 
+@@ -449,3 +484,7 @@ class HgImportCommitHandler(processor.Co
          # copy oldname to newname and delete oldname
          self.copies[filecmd.new_path] = filecmd.old_path
          self.modified.append((filecmd.old_path, None))
 +
 +    def deleteall_handler(self, filecmd):
-+        pass # Drop it for now, used by fossil when no parent exists.
++        # Ignore for now.
++        pass