From d219d48cce59e02195e32a80c16e0b7cffb11e75 Mon Sep 17 00:00:00 2001 From: Felix Gruber Date: Wed, 20 Apr 2022 17:28:01 +0000 Subject: gnu: Add python-protego. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * gnu/packages/python-web.scm (python-protego): New variable. Signed-off-by: Ludovic Courtès --- gnu/packages/python-web.scm | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) (limited to 'gnu/packages/python-web.scm') diff --git a/gnu/packages/python-web.scm b/gnu/packages/python-web.scm index 7e333916b6..05104f8fe2 100644 --- a/gnu/packages/python-web.scm +++ b/gnu/packages/python-web.scm @@ -6466,3 +6466,30 @@ as Flask.") "This package provides a Python JSON-RPC 2.0 protocol and server powered by asyncio.") (license license:expat))) + +(define-public python-protego + (package + (name "python-protego") + (version "0.2.1") + (source + (origin + (method url-fetch) + (uri (pypi-uri "Protego" version)) + (sha256 + (base32 "1wigcjyhz8zbk562zhgfbkm733dcn65j1swzvki79dys0i1nsrnz")))) + (build-system python-build-system) + (arguments + `(#:phases + (modify-phases %standard-phases + (replace 'check + (lambda* (#:key tests? #:allow-other-keys) + (when tests? + (invoke "pytest"))))))) + (propagated-inputs (list python-six)) + (native-inputs (list python-pytest)) + (home-page "https://github.com/scrapy/protego") + (synopsis + "Pure-Python robots.txt parser with support for modern conventions") + (description + "Pure-Python robots.txt parser with support for modern conventions.") + (license license:bsd-3))) -- cgit v1.3 From 76b6337650ba35c85e9263c958dd334766268172 Mon Sep 17 00:00:00 2001 From: Felix Gruber Date: Wed, 20 Apr 2022 17:28:02 +0000 Subject: gnu: Add python-parsel. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * gnu/packages/python-web.scm (python-parsel): New variable. Signed-off-by: Ludovic Courtès --- gnu/packages/python-web.scm | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) (limited to 'gnu/packages/python-web.scm') diff --git a/gnu/packages/python-web.scm b/gnu/packages/python-web.scm index 05104f8fe2..b2fbbc86cd 100644 --- a/gnu/packages/python-web.scm +++ b/gnu/packages/python-web.scm @@ -6493,3 +6493,28 @@ by asyncio.") (description "Pure-Python robots.txt parser with support for modern conventions.") (license license:bsd-3))) + +(define-public python-parsel + (package + (name "python-parsel") + (version "1.6.0") + (source + (origin + (method url-fetch) + (uri (pypi-uri "parsel" version)) + (sha256 + (base32 "0yawf9r3r863lwxj0n89i7h3n8xjbsl5b7n6xg76r68scl5yzvvh")))) + (build-system python-build-system) + (propagated-inputs + (list python-cssselect + python-lxml + python-six + python-w3lib)) + (native-inputs + (list python-pytest python-pytest-runner)) + (home-page "https://github.com/scrapy/parsel") + (synopsis "Extract data from HTML and XML using XPath and CSS selectors") + (description "Parsel is a library to extract and remove data from +HTML and XML using XPath and CSS selectors, optionally combined with +regular expressions.") + (license license:bsd-3))) -- cgit v1.3 From f455c1e313ca1cb7cb8d875b9fa00e2b8c5ab3c2 Mon Sep 17 00:00:00 2001 From: Felix Gruber Date: Wed, 20 Apr 2022 17:28:04 +0000 Subject: gnu: Add python-scrapy. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * gnu/packages/python-web.scm (python-scrapy): New variable. Signed-off-by: Ludovic Courtès --- gnu/packages/python-web.scm | 59 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) (limited to 'gnu/packages/python-web.scm') diff --git a/gnu/packages/python-web.scm b/gnu/packages/python-web.scm index b2fbbc86cd..0f9f1e8786 100644 --- a/gnu/packages/python-web.scm +++ b/gnu/packages/python-web.scm @@ -6518,3 +6518,62 @@ by asyncio.") HTML and XML using XPath and CSS selectors, optionally combined with regular expressions.") (license license:bsd-3))) + +(define-public python-scrapy + (package + (name "python-scrapy") + (version "2.6.1") + (source + (origin + (method url-fetch) + (uri (pypi-uri "Scrapy" version)) + (sha256 + (base32 "09rqalbwcz9ix8h0992mzjs50sssxsmmh8w9abkrqchgknjmbzan")))) + (build-system python-build-system) + (arguments + `(#:phases + (modify-phases %standard-phases + (replace 'check + (lambda* (#:key tests? #:allow-other-keys) + (when tests? + (invoke "pytest" + ;; requires network access + "--ignore" "tests/test_command_check.py" + "-k" + (string-append + ;; Failing for unknown reasons + "not test_server_set_cookie_domain_suffix_public_private" + " and not test_user_set_cookie_domain_suffix_public_private" + " and not test_pformat") + "tests"))))))) + (propagated-inputs + (list python-botocore ; Optional: For S3FeedStorage class. + python-cryptography + python-cssselect + python-itemadapter + python-itemloaders + python-lxml + python-parsel + python-protego + python-pydispatcher + python-pyopenssl + python-queuelib + python-service-identity + python-setuptools + python-tldextract + python-twisted + python-w3lib + python-zope-interface)) + (native-inputs + (list python-pytest + python-pyftpdlib + python-sybil + python-testfixtures + python-uvloop)) + (home-page "https://scrapy.org") + (synopsis "High-level Web crawling and Web scraping framework") + (description "Scrapy is a fast high-level web crawling and web +scraping framework, used to crawl websites and extract structured data +from their pages. It can be used for a wide range of purposes, from data +mining to monitoring and automated testing.") + (license license:bsd-3))) -- cgit v1.3