[pkgsrc/trunk]: pkgsrc/www/py-scrapy py-scrapy: updated to 1.6.0

[adam <adam%pkgsrc.org@localhost>]

details:   https://anonhg.NetBSD.org/pkgsrc/rev/aee736c787a9
branches:  trunk
changeset: 329120:aee736c787a9
user:      adam <adam%pkgsrc.org@localhost>
date:      Thu Jan 31 09:07:46 2019 +0000

description:
py-scrapy: updated to 1.6.0

Scrapy 1.6.0:

Highlights:
* better Windows support;
* Python 3.7 compatibility;
* big documentation improvements, including a switch
  from .extract_first() + .extract() API to .get() + .getall()
  API;
* feed exports, FilePipeline and MediaPipeline improvements;
* better extensibility: :signal:item_error and
  :signal:request_reached_downloader signals; from_crawler support
  for feed exporters, feed storages and dupefilters.
* scrapy.contracts fixes and new features;
* telnet console security improvements, first released as a
  backport in :ref:release-1.5.2;
* clean-up of the deprecated code;
* various bug fixes, small new features and usability improvements across
  the codebase.

diffstat:

 www/py-scrapy/Makefile |   16 +-
 www/py-scrapy/PLIST    |  200 +------------------------------------------------
 www/py-scrapy/distinfo |   10 +-
 3 files changed, 14 insertions(+), 212 deletions(-)

diffs (truncated from 318 to 300 lines):

diff -r decba72f8ef0 -r aee736c787a9 www/py-scrapy/Makefile
--- a/www/py-scrapy/Makefile    Thu Jan 31 08:46:37 2019 +0000
+++ b/www/py-scrapy/Makefile    Thu Jan 31 09:07:46 2019 +0000
@@ -1,6 +1,6 @@
-# $NetBSD: Makefile,v 1.7 2019/01/24 14:11:48 adam Exp $
+# $NetBSD: Makefile,v 1.8 2019/01/31 09:07:46 adam Exp $
 
-DISTNAME=      Scrapy-1.5.2
+DISTNAME=      Scrapy-1.6.0
 PKGNAME=       ${PYPKGPREFIX}-${DISTNAME:tl}
 CATEGORIES=    www python
 MASTER_SITES=  ${MASTER_SITE_PYPI:=S/Scrapy/}
@@ -10,15 +10,15 @@
 COMMENT=       High-level Web Crawling and Web Scraping framework
 LICENSE=       modified-bsd
 
-DEPENDS+=      ${PYPKGPREFIX}-OpenSSL-[0-9]*:../../security/py-OpenSSL
-DEPENDS+=      ${PYPKGPREFIX}-cssselect-[0-9]*:../../textproc/py-cssselect
-DEPENDS+=      ${PYPKGPREFIX}-lxml-[0-9]*:../../textproc/py-lxml
-DEPENDS+=      ${PYPKGPREFIX}-parsel>=1.1.0:../../www/py-parsel
+DEPENDS+=      ${PYPKGPREFIX}-OpenSSL>=0.13.1:../../security/py-OpenSSL
+DEPENDS+=      ${PYPKGPREFIX}-cssselect>=0.9:../../textproc/py-cssselect
+DEPENDS+=      ${PYPKGPREFIX}-lxml>=3.2.4:../../textproc/py-lxml
+DEPENDS+=      ${PYPKGPREFIX}-parsel>=1.5:../../www/py-parsel
 DEPENDS+=      ${PYPKGPREFIX}-pydispatcher>=2.0.5:../../devel/py-pydispatcher
-DEPENDS+=      ${PYPKGPREFIX}-queuelib-[0-9]*:../../devel/py-queuelib
+DEPENDS+=      ${PYPKGPREFIX}-queuelib>=1.1.1:../../devel/py-queuelib
 DEPENDS+=      ${PYPKGPREFIX}-service_identity-[0-9]*:../../security/py-service_identity
 DEPENDS+=      ${PYPKGPREFIX}-six>=1.5.2:../../lang/py-six
-DEPENDS+=      ${PYPKGPREFIX}-twisted>=13.1.0:../../net/py-twisted
+DEPENDS+=      ${PYPKGPREFIX}-twisted>=17.9.0:../../net/py-twisted
 DEPENDS+=      ${PYPKGPREFIX}-w3lib>=1.17.0:../../www/py-w3lib
 
 USE_LANGUAGES= # none
diff -r decba72f8ef0 -r aee736c787a9 www/py-scrapy/PLIST
--- a/www/py-scrapy/PLIST       Thu Jan 31 08:46:37 2019 +0000
+++ b/www/py-scrapy/PLIST       Thu Jan 31 09:07:46 2019 +0000
@@ -1,4 +1,4 @@
-@comment $NetBSD: PLIST,v 1.4 2019/01/24 14:11:48 adam Exp $
+@comment $NetBSD: PLIST,v 1.5 2019/01/31 09:07:46 adam Exp $
 bin/scrapy-${PYVERSSUFFIX}
 ${PYSITELIB}/${EGG_INFODIR}/PKG-INFO
 ${PYSITELIB}/${EGG_INFODIR}/SOURCES.txt
@@ -20,9 +20,6 @@
 ${PYSITELIB}/scrapy/cmdline.py
 ${PYSITELIB}/scrapy/cmdline.pyc
 ${PYSITELIB}/scrapy/cmdline.pyo
-${PYSITELIB}/scrapy/command.py
-${PYSITELIB}/scrapy/command.pyc
-${PYSITELIB}/scrapy/command.pyo
 ${PYSITELIB}/scrapy/commands/__init__.py
 ${PYSITELIB}/scrapy/commands/__init__.pyc
 ${PYSITELIB}/scrapy/commands/__init__.pyo
@@ -77,174 +74,6 @@
 ${PYSITELIB}/scrapy/contracts/default.py
 ${PYSITELIB}/scrapy/contracts/default.pyc
 ${PYSITELIB}/scrapy/contracts/default.pyo
-${PYSITELIB}/scrapy/contrib/__init__.py
-${PYSITELIB}/scrapy/contrib/__init__.pyc
-${PYSITELIB}/scrapy/contrib/__init__.pyo
-${PYSITELIB}/scrapy/contrib/closespider.py
-${PYSITELIB}/scrapy/contrib/closespider.pyc
-${PYSITELIB}/scrapy/contrib/closespider.pyo
-${PYSITELIB}/scrapy/contrib/corestats.py
-${PYSITELIB}/scrapy/contrib/corestats.pyc
-${PYSITELIB}/scrapy/contrib/corestats.pyo
-${PYSITELIB}/scrapy/contrib/debug.py
-${PYSITELIB}/scrapy/contrib/debug.pyc
-${PYSITELIB}/scrapy/contrib/debug.pyo
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/__init__.py
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/__init__.pyc
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/__init__.pyo
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/ajaxcrawl.py
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/ajaxcrawl.pyc
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/ajaxcrawl.pyo
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/chunked.py
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/chunked.pyc
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/chunked.pyo
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/cookies.py
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/cookies.pyc
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/cookies.pyo
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/decompression.py
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/decompression.pyc
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/decompression.pyo
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/defaultheaders.py
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/defaultheaders.pyc
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/defaultheaders.pyo
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/downloadtimeout.py
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/downloadtimeout.pyc
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/downloadtimeout.pyo
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/httpauth.py
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/httpauth.pyc
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/httpauth.pyo
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/httpcache.py
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/httpcache.pyc
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/httpcache.pyo
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/httpcompression.py
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/httpcompression.pyc
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/httpcompression.pyo
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/httpproxy.py
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/httpproxy.pyc
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/httpproxy.pyo
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/redirect.py
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/redirect.pyc
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/redirect.pyo
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/retry.py
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/retry.pyc
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/retry.pyo
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/robotstxt.py
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/robotstxt.pyc
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/robotstxt.pyo
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/stats.py
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/stats.pyc
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/stats.pyo
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/useragent.py
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/useragent.pyc
-${PYSITELIB}/scrapy/contrib/downloadermiddleware/useragent.pyo
-${PYSITELIB}/scrapy/contrib/exporter/__init__.py
-${PYSITELIB}/scrapy/contrib/exporter/__init__.pyc
-${PYSITELIB}/scrapy/contrib/exporter/__init__.pyo
-${PYSITELIB}/scrapy/contrib/feedexport.py
-${PYSITELIB}/scrapy/contrib/feedexport.pyc
-${PYSITELIB}/scrapy/contrib/feedexport.pyo
-${PYSITELIB}/scrapy/contrib/httpcache.py
-${PYSITELIB}/scrapy/contrib/httpcache.pyc
-${PYSITELIB}/scrapy/contrib/httpcache.pyo
-${PYSITELIB}/scrapy/contrib/linkextractors/__init__.py
-${PYSITELIB}/scrapy/contrib/linkextractors/__init__.pyc
-${PYSITELIB}/scrapy/contrib/linkextractors/__init__.pyo
-${PYSITELIB}/scrapy/contrib/linkextractors/htmlparser.py
-${PYSITELIB}/scrapy/contrib/linkextractors/htmlparser.pyc
-${PYSITELIB}/scrapy/contrib/linkextractors/htmlparser.pyo
-${PYSITELIB}/scrapy/contrib/linkextractors/lxmlhtml.py
-${PYSITELIB}/scrapy/contrib/linkextractors/lxmlhtml.pyc
-${PYSITELIB}/scrapy/contrib/linkextractors/lxmlhtml.pyo
-${PYSITELIB}/scrapy/contrib/linkextractors/regex.py
-${PYSITELIB}/scrapy/contrib/linkextractors/regex.pyc
-${PYSITELIB}/scrapy/contrib/linkextractors/regex.pyo
-${PYSITELIB}/scrapy/contrib/linkextractors/sgml.py
-${PYSITELIB}/scrapy/contrib/linkextractors/sgml.pyc
-${PYSITELIB}/scrapy/contrib/linkextractors/sgml.pyo
-${PYSITELIB}/scrapy/contrib/loader/__init__.py
-${PYSITELIB}/scrapy/contrib/loader/__init__.pyc
-${PYSITELIB}/scrapy/contrib/loader/__init__.pyo
-${PYSITELIB}/scrapy/contrib/loader/common.py
-${PYSITELIB}/scrapy/contrib/loader/common.pyc
-${PYSITELIB}/scrapy/contrib/loader/common.pyo
-${PYSITELIB}/scrapy/contrib/loader/processor.py
-${PYSITELIB}/scrapy/contrib/loader/processor.pyc
-${PYSITELIB}/scrapy/contrib/loader/processor.pyo
-${PYSITELIB}/scrapy/contrib/logstats.py
-${PYSITELIB}/scrapy/contrib/logstats.pyc
-${PYSITELIB}/scrapy/contrib/logstats.pyo
-${PYSITELIB}/scrapy/contrib/memdebug.py
-${PYSITELIB}/scrapy/contrib/memdebug.pyc
-${PYSITELIB}/scrapy/contrib/memdebug.pyo
-${PYSITELIB}/scrapy/contrib/memusage.py
-${PYSITELIB}/scrapy/contrib/memusage.pyc
-${PYSITELIB}/scrapy/contrib/memusage.pyo
-${PYSITELIB}/scrapy/contrib/pipeline/__init__.py
-${PYSITELIB}/scrapy/contrib/pipeline/__init__.pyc
-${PYSITELIB}/scrapy/contrib/pipeline/__init__.pyo
-${PYSITELIB}/scrapy/contrib/pipeline/files.py
-${PYSITELIB}/scrapy/contrib/pipeline/files.pyc
-${PYSITELIB}/scrapy/contrib/pipeline/files.pyo
-${PYSITELIB}/scrapy/contrib/pipeline/images.py
-${PYSITELIB}/scrapy/contrib/pipeline/images.pyc
-${PYSITELIB}/scrapy/contrib/pipeline/images.pyo
-${PYSITELIB}/scrapy/contrib/pipeline/media.py
-${PYSITELIB}/scrapy/contrib/pipeline/media.pyc
-${PYSITELIB}/scrapy/contrib/pipeline/media.pyo
-${PYSITELIB}/scrapy/contrib/spidermiddleware/__init__.py
-${PYSITELIB}/scrapy/contrib/spidermiddleware/__init__.pyc
-${PYSITELIB}/scrapy/contrib/spidermiddleware/__init__.pyo
-${PYSITELIB}/scrapy/contrib/spidermiddleware/depth.py
-${PYSITELIB}/scrapy/contrib/spidermiddleware/depth.pyc
-${PYSITELIB}/scrapy/contrib/spidermiddleware/depth.pyo
-${PYSITELIB}/scrapy/contrib/spidermiddleware/httperror.py
-${PYSITELIB}/scrapy/contrib/spidermiddleware/httperror.pyc
-${PYSITELIB}/scrapy/contrib/spidermiddleware/httperror.pyo
-${PYSITELIB}/scrapy/contrib/spidermiddleware/offsite.py
-${PYSITELIB}/scrapy/contrib/spidermiddleware/offsite.pyc
-${PYSITELIB}/scrapy/contrib/spidermiddleware/offsite.pyo
-${PYSITELIB}/scrapy/contrib/spidermiddleware/referer.py
-${PYSITELIB}/scrapy/contrib/spidermiddleware/referer.pyc
-${PYSITELIB}/scrapy/contrib/spidermiddleware/referer.pyo
-${PYSITELIB}/scrapy/contrib/spidermiddleware/urllength.py
-${PYSITELIB}/scrapy/contrib/spidermiddleware/urllength.pyc
-${PYSITELIB}/scrapy/contrib/spidermiddleware/urllength.pyo
-${PYSITELIB}/scrapy/contrib/spiders/__init__.py
-${PYSITELIB}/scrapy/contrib/spiders/__init__.pyc
-${PYSITELIB}/scrapy/contrib/spiders/__init__.pyo
-${PYSITELIB}/scrapy/contrib/spiders/crawl.py
-${PYSITELIB}/scrapy/contrib/spiders/crawl.pyc
-${PYSITELIB}/scrapy/contrib/spiders/crawl.pyo
-${PYSITELIB}/scrapy/contrib/spiders/feed.py
-${PYSITELIB}/scrapy/contrib/spiders/feed.pyc
-${PYSITELIB}/scrapy/contrib/spiders/feed.pyo
-${PYSITELIB}/scrapy/contrib/spiders/init.py
-${PYSITELIB}/scrapy/contrib/spiders/init.pyc
-${PYSITELIB}/scrapy/contrib/spiders/init.pyo
-${PYSITELIB}/scrapy/contrib/spiders/sitemap.py
-${PYSITELIB}/scrapy/contrib/spiders/sitemap.pyc
-${PYSITELIB}/scrapy/contrib/spiders/sitemap.pyo
-${PYSITELIB}/scrapy/contrib/spiderstate.py
-${PYSITELIB}/scrapy/contrib/spiderstate.pyc
-${PYSITELIB}/scrapy/contrib/spiderstate.pyo
-${PYSITELIB}/scrapy/contrib/statsmailer.py
-${PYSITELIB}/scrapy/contrib/statsmailer.pyc
-${PYSITELIB}/scrapy/contrib/statsmailer.pyo
-${PYSITELIB}/scrapy/contrib/throttle.py
-${PYSITELIB}/scrapy/contrib/throttle.pyc
-${PYSITELIB}/scrapy/contrib/throttle.pyo
-${PYSITELIB}/scrapy/contrib_exp/__init__.py
-${PYSITELIB}/scrapy/contrib_exp/__init__.pyc
-${PYSITELIB}/scrapy/contrib_exp/__init__.pyo
-${PYSITELIB}/scrapy/contrib_exp/downloadermiddleware/__init__.py
-${PYSITELIB}/scrapy/contrib_exp/downloadermiddleware/__init__.pyc
-${PYSITELIB}/scrapy/contrib_exp/downloadermiddleware/__init__.pyo
-${PYSITELIB}/scrapy/contrib_exp/downloadermiddleware/decompression.py
-${PYSITELIB}/scrapy/contrib_exp/downloadermiddleware/decompression.pyc
-${PYSITELIB}/scrapy/contrib_exp/downloadermiddleware/decompression.pyo
-${PYSITELIB}/scrapy/contrib_exp/iterators.py
-${PYSITELIB}/scrapy/contrib_exp/iterators.pyc
-${PYSITELIB}/scrapy/contrib_exp/iterators.pyo
 ${PYSITELIB}/scrapy/core/__init__.py
 ${PYSITELIB}/scrapy/core/__init__.pyc
 ${PYSITELIB}/scrapy/core/__init__.pyo
@@ -350,9 +179,6 @@
 ${PYSITELIB}/scrapy/downloadermiddlewares/useragent.py
 ${PYSITELIB}/scrapy/downloadermiddlewares/useragent.pyc
 ${PYSITELIB}/scrapy/downloadermiddlewares/useragent.pyo
-${PYSITELIB}/scrapy/dupefilter.py
-${PYSITELIB}/scrapy/dupefilter.pyc
-${PYSITELIB}/scrapy/dupefilter.pyo
 ${PYSITELIB}/scrapy/dupefilters.py
 ${PYSITELIB}/scrapy/dupefilters.pyc
 ${PYSITELIB}/scrapy/dupefilters.pyo
@@ -446,9 +272,6 @@
 ${PYSITELIB}/scrapy/link.py
 ${PYSITELIB}/scrapy/link.pyc
 ${PYSITELIB}/scrapy/link.pyo
-${PYSITELIB}/scrapy/linkextractor.py
-${PYSITELIB}/scrapy/linkextractor.pyc
-${PYSITELIB}/scrapy/linkextractor.pyo
 ${PYSITELIB}/scrapy/linkextractors/__init__.py
 ${PYSITELIB}/scrapy/linkextractors/__init__.pyc
 ${PYSITELIB}/scrapy/linkextractors/__init__.pyo
@@ -498,9 +321,6 @@
 ${PYSITELIB}/scrapy/pipelines/media.py
 ${PYSITELIB}/scrapy/pipelines/media.pyc
 ${PYSITELIB}/scrapy/pipelines/media.pyo
-${PYSITELIB}/scrapy/project.py
-${PYSITELIB}/scrapy/project.pyc
-${PYSITELIB}/scrapy/project.pyo
 ${PYSITELIB}/scrapy/resolver.py
 ${PYSITELIB}/scrapy/resolver.pyc
 ${PYSITELIB}/scrapy/resolver.pyo
@@ -537,15 +357,9 @@
 ${PYSITELIB}/scrapy/signals.py
 ${PYSITELIB}/scrapy/signals.pyc
 ${PYSITELIB}/scrapy/signals.pyo
-${PYSITELIB}/scrapy/spider.py
-${PYSITELIB}/scrapy/spider.pyc
-${PYSITELIB}/scrapy/spider.pyo
 ${PYSITELIB}/scrapy/spiderloader.py
 ${PYSITELIB}/scrapy/spiderloader.pyc
 ${PYSITELIB}/scrapy/spiderloader.pyo
-${PYSITELIB}/scrapy/spidermanager.py
-${PYSITELIB}/scrapy/spidermanager.pyc
-${PYSITELIB}/scrapy/spidermanager.pyo
 ${PYSITELIB}/scrapy/spidermiddlewares/__init__.py
 ${PYSITELIB}/scrapy/spidermiddlewares/__init__.pyc
 ${PYSITELIB}/scrapy/spidermiddlewares/__init__.pyo
@@ -579,18 +393,9 @@
 ${PYSITELIB}/scrapy/spiders/sitemap.py
 ${PYSITELIB}/scrapy/spiders/sitemap.pyc
 ${PYSITELIB}/scrapy/spiders/sitemap.pyo
-${PYSITELIB}/scrapy/squeue.py
-${PYSITELIB}/scrapy/squeue.pyc
-${PYSITELIB}/scrapy/squeue.pyo
 ${PYSITELIB}/scrapy/squeues.py
 ${PYSITELIB}/scrapy/squeues.pyc
 ${PYSITELIB}/scrapy/squeues.pyo
-${PYSITELIB}/scrapy/stats.py
-${PYSITELIB}/scrapy/stats.pyc
-${PYSITELIB}/scrapy/stats.pyo
-${PYSITELIB}/scrapy/statscol.py
-${PYSITELIB}/scrapy/statscol.pyc
-${PYSITELIB}/scrapy/statscol.pyo
 ${PYSITELIB}/scrapy/statscollectors.py
 ${PYSITELIB}/scrapy/statscollectors.pyc
 ${PYSITELIB}/scrapy/statscollectors.pyo
@@ -630,9 +435,6 @@
 ${PYSITELIB}/scrapy/utils/datatypes.py
 ${PYSITELIB}/scrapy/utils/datatypes.pyc
 ${PYSITELIB}/scrapy/utils/datatypes.pyo
-${PYSITELIB}/scrapy/utils/decorator.py
-${PYSITELIB}/scrapy/utils/decorator.pyc
-${PYSITELIB}/scrapy/utils/decorator.pyo