[pkgsrc/trunk]: pkgsrc/pkgtools/url2pkg/files pkgtools/url2pkg: improve deter...

[rillig <rillig%pkgsrc.org@localhost>]

details:   https://anonhg.NetBSD.org/pkgsrc/rev/8161a2bb94e6
branches:  trunk
changeset: 341697:8161a2bb94e6
user:      rillig <rillig%pkgsrc.org@localhost>
date:      Sun Oct 06 08:24:18 2019 +0000

description:
pkgtools/url2pkg: improve determining WRKSRC

For setting WRKSRC, only the directories in WRKDIR are relevant, not the
files.

diffstat:

 pkgtools/url2pkg/files/url2pkg.py      |  43 ++++++++++-------------
 pkgtools/url2pkg/files/url2pkg_test.py |  60 ++++++++++++++++++++++++++-------
 2 files changed, 65 insertions(+), 38 deletions(-)

diffs (177 lines):

diff -r cb1d26da219e -r 8161a2bb94e6 pkgtools/url2pkg/files/url2pkg.py
--- a/pkgtools/url2pkg/files/url2pkg.py Sun Oct 06 05:53:00 2019 +0000
+++ b/pkgtools/url2pkg/files/url2pkg.py Sun Oct 06 08:24:18 2019 +0000
@@ -1,5 +1,5 @@
 #! @PYTHONBIN@
-# $NetBSD: url2pkg.py,v 1.16 2019/10/06 05:53:00 rillig Exp $
+# $NetBSD: url2pkg.py,v 1.17 2019/10/06 08:24:18 rillig Exp $
 
 # Copyright (c) 2019 The NetBSD Foundation, Inc.
 # All rights reserved.
@@ -785,23 +785,21 @@
         Sets abs_wrksrc depending on abs_wrkdir and the files found there.
         """
 
-        def ignore(f: str) -> bool:
-            return f.startswith('.') \
-                   or f == 'pax_global_header' \
-                   or f == 'package.xml' \
-                   or f.endswith('.gemspec')
+        def relevant(f: Path) -> bool:
+            return f.is_dir() and not f.name.startswith('.')
+
+        subdirs = [f.name for f in self.abs_wrkdir.glob('*') if relevant(f)]
 
-        files = list(filter(lambda x: not ignore(x), os.listdir(self.abs_wrkdir)))
-
-        if len(files) == 1:
-            if files[0] != self.makefile_lines.get('DISTNAME'):
-                self.build_vars.append(Var('WRKSRC', '=', '${WRKDIR}/' + files[0]))
-            self.abs_wrksrc = self.abs_wrkdir / files[0]
-        elif len(files) == 0:
+        if len(subdirs) == 1:
+            if subdirs[0] != self.makefile_lines.get('DISTNAME'):
+                self.build_vars.append(Var('WRKSRC', '=', '${WRKDIR}/' + subdirs[0]))
+            self.abs_wrksrc = self.abs_wrkdir / subdirs[0]
+        elif len(subdirs) == 0:
             self.build_vars.append(Var('WRKSRC', '=', '${WRKDIR}'))
             self.abs_wrksrc = self.abs_wrkdir
         else:
-            wrksrc = '${WRKDIR} # More than one possibility -- please check manually.'
+            choices = ' '.join(subdirs)
+            wrksrc = f'${{WRKDIR}} # TODO: one of {choices}, or leave it as-is'
             self.build_vars.append(Var('WRKSRC', '=', wrksrc))
             self.abs_wrksrc = self.abs_wrkdir
 
@@ -893,17 +891,18 @@
 
     def adjust(self):
 
-        def scan(basedir: Path, pattern: str) -> List[str]:
-            full_paths = basedir.rglob(pattern)
-            return [str(f.relative_to(basedir)) for f in full_paths]
+        def scan(basedir: Path, only: Callable[[Path], bool]) -> List[str]:
+            relevant = (f for f in basedir.rglob('*') if only(f))
+            relative = (str(f.relative_to(basedir)) for f in relevant)
+            return list(sorted((f for f in relative if not f.startswith('.'))))
 
         self.up.debug('Adjusting the Makefile')
         self.makefile_lines = Lines.read_from(self.up.pkgdir / 'Makefile')
 
         self.abs_wrkdir = Path(self.up.show_var('WRKDIR'))
         self.determine_wrksrc()
-        self.wrksrc_files = scan(self.abs_wrksrc, '**')
-        self.wrksrc_dirs = scan(self.abs_wrksrc, '**/')
+        self.wrksrc_dirs = scan(self.abs_wrksrc, Path.is_dir)
+        self.wrksrc_files = scan(self.abs_wrksrc, Path.is_file)
 
         self.adjust_configure()
         self.adjust_cmake()
@@ -939,11 +938,7 @@
     if not re.fullmatch(r'\w+://[!-~]+?/[!-~]+', url):
         sys.exit(f'url2pkg: invalid URL: {url}')
 
-    if not up.pkgdir.glob('w*/.extract_done') or not (up.pkgdir / 'Makefile').is_file():
-        initial_lines = Generator(url).generate_package(up)
-    else:
-        initial_lines = Generator(url).generate_lines()
-
+    initial_lines = Generator(url).generate_package(up)
     Adjuster(up, url, initial_lines).adjust()
 
     up.out.write('\n')
diff -r cb1d26da219e -r 8161a2bb94e6 pkgtools/url2pkg/files/url2pkg_test.py
--- a/pkgtools/url2pkg/files/url2pkg_test.py    Sun Oct 06 05:53:00 2019 +0000
+++ b/pkgtools/url2pkg/files/url2pkg_test.py    Sun Oct 06 08:24:18 2019 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: url2pkg_test.py,v 1.15 2019/10/06 05:53:00 rillig Exp $
+# $NetBSD: url2pkg_test.py,v 1.16 2019/10/06 08:24:18 rillig Exp $
 
 import pytest
 from url2pkg import *
@@ -1237,6 +1237,18 @@
     assert adjuster.abs_wrksrc == adjuster.abs_wrkdir / 'subdir'
 
 
+def test_Adjuster_determine_wrksrc__distname_dir(tmp_path: Path):
+    adjuster = Adjuster(up, '', Lines())
+    adjuster.abs_wrkdir = tmp_path
+    adjuster.makefile_lines.add_vars(Var('DISTNAME', '=', 'distname-1.0'))
+    (tmp_path / 'distname-1.0').mkdir()
+
+    adjuster.determine_wrksrc()
+
+    assert adjuster.abs_wrksrc == adjuster.abs_wrkdir / 'distname-1.0'
+    assert str_vars(adjuster.build_vars) == []
+
+
 def test_Adjuster_determine_wrksrc__several_dirs(tmp_path: Path):
     adjuster = Adjuster(up, '', Lines())
     adjuster.abs_wrkdir = tmp_path
@@ -1247,26 +1259,15 @@
 
     assert adjuster.abs_wrksrc == adjuster.abs_wrkdir
     assert str_vars(adjuster.build_vars) == [
-        'WRKSRC=${WRKDIR} # More than one possibility -- please check manually.',
+        'WRKSRC=${WRKDIR} # TODO: one of subdir1 subdir2, or leave it as-is',
     ]
 
 
 def test_Adjuster_adjust__empty_wrkdir(tmp_path: Path):
-    wrkdir = tmp_path / 'wrkdir'
     up.pkgdir = tmp_path
-    wrkdir.mkdir()
+    up.show_var = lambda varname: {'WRKDIR': str(tmp_path)}[varname]
     adjuster = Adjuster(up, 'https://example.org/distfile-1.0.zip', Lines())
-    adjuster.abs_wrkdir = wrkdir
     (tmp_path / 'Makefile').write_text('# url2pkg-marker\n')
-    fake_path = tmp_path / 'fake'
-    fake_path.write_text(
-        '#! /bin/sh\n'
-        'case $* in\n'
-        f'("show-var VARNAME=WRKDIR") echo "{wrkdir}" ;;\n'
-        '(*) "unknown: $*" ;;\n'
-        'esac\n')
-    fake_path.chmod(0o755)
-    up.make = fake_path
 
     adjuster.adjust()
 
@@ -1277,6 +1278,37 @@
     ]
 
 
+def test_Adjuster_adjust__files_in_wrksrc(tmp_path: Path):
+    wrkdir = tmp_path / 'work'
+    wrkdir.mkdir()
+    (wrkdir / '.hidden').touch()
+    (wrkdir / 'file').touch()
+    (wrkdir / 'dir').mkdir()
+    (wrkdir / 'dir' / '.hidden-dir').mkdir()
+    (wrkdir / 'dir' / 'subdir').mkdir()
+    (wrkdir / 'dir' / 'subdir' / '.hidden').touch()
+    (wrkdir / 'dir' / 'subdir' / 'file').touch()
+    (wrkdir / 'dir2').mkdir()  # to make WRKSRC = WRKDIR
+    up.show_var = lambda varname: {'WRKDIR': str(wrkdir)}[varname]
+    up.pkgdir = tmp_path
+    (tmp_path / 'Makefile').write_text('# url2pkg-marker\n')
+    adjuster = Adjuster(up, 'https://example.org/distfile-1.0.zip', Lines())
+
+    adjuster.adjust()
+
+    assert adjuster.wrksrc_dirs == [
+        'dir',
+        'dir/.hidden-dir',
+        'dir/subdir',
+        'dir2',
+    ]
+    assert adjuster.wrksrc_files == [
+        'dir/subdir/.hidden',
+        'dir/subdir/file',
+        'file',
+    ]
+
+
 def test_Adjuster_adjust_lines_python_module(tmp_path: Path):
     url = 'https://github.com/espressif/esptool/archive/v2.7.tar.gz'
     up.pkgdir = tmp_path