python3Packages.scrapy: 2.4.1 -> 2.5.0
This commit is contained in:
Fabian Affolter
parent
5e10deb72a
commit
8d022afa23
|
|
@ -1,64 +1,89 @@
|
|||
{ lib
|
||||
, stdenv
|
||||
, buildPythonPackage
|
||||
, isPy27
|
||||
, fetchPypi
|
||||
, glibcLocales
|
||||
, pytestCheckHook
|
||||
, testfixtures
|
||||
, pillow
|
||||
, twisted
|
||||
, cryptography
|
||||
, w3lib
|
||||
, lxml
|
||||
, queuelib
|
||||
, pyopenssl
|
||||
, service-identity
|
||||
, parsel
|
||||
, pydispatcher
|
||||
, cssselect
|
||||
, zope_interface
|
||||
, protego
|
||||
, jmespath
|
||||
, sybil
|
||||
, pytest-twisted
|
||||
, botocore
|
||||
, buildPythonPackage
|
||||
, cryptography
|
||||
, cssselect
|
||||
, fetchFromGitHub
|
||||
, fetchpatch
|
||||
, glibcLocales
|
||||
, installShellFiles
|
||||
, itemadapter
|
||||
, itemloaders
|
||||
, jmespath
|
||||
, lxml
|
||||
, parsel
|
||||
, pillow
|
||||
, protego
|
||||
, pydispatcher
|
||||
, pyopenssl
|
||||
, pytest-twisted
|
||||
, pytestCheckHook
|
||||
, pythonOlder
|
||||
, queuelib
|
||||
, service-identity
|
||||
, sybil
|
||||
, testfixtures
|
||||
, twisted
|
||||
, w3lib
|
||||
, zope_interface
|
||||
}:
|
||||
|
||||
buildPythonPackage rec {
|
||||
version = "2.4.1";
|
||||
pname = "Scrapy";
|
||||
pname = "scrapy";
|
||||
version = "2.5.0";
|
||||
disabled = pythonOlder "3.6";
|
||||
|
||||
disabled = isPy27;
|
||||
src = fetchFromGitHub {
|
||||
owner = pname;
|
||||
repo = pname;
|
||||
rev = version;
|
||||
sha256 = "09lxnjz1cw37i9bgk8sci2xxknj20gi2lq8l7i0b3xw7q8bxzp7h";
|
||||
};
|
||||
|
||||
nativeBuildInputs = [
|
||||
installShellFiles
|
||||
];
|
||||
|
||||
propagatedBuildInputs = [
|
||||
cryptography
|
||||
cssselect
|
||||
itemadapter
|
||||
itemloaders
|
||||
lxml
|
||||
parsel
|
||||
protego
|
||||
pydispatcher
|
||||
pyopenssl
|
||||
queuelib
|
||||
service-identity
|
||||
twisted
|
||||
w3lib
|
||||
zope_interface
|
||||
];
|
||||
|
||||
checkInputs = [
|
||||
botocore
|
||||
glibcLocales
|
||||
jmespath
|
||||
pytestCheckHook
|
||||
sybil
|
||||
testfixtures
|
||||
pillow
|
||||
pytest-twisted
|
||||
botocore
|
||||
];
|
||||
|
||||
propagatedBuildInputs = [
|
||||
twisted
|
||||
cryptography
|
||||
cssselect
|
||||
lxml
|
||||
parsel
|
||||
pydispatcher
|
||||
pyopenssl
|
||||
queuelib
|
||||
service-identity
|
||||
w3lib
|
||||
zope_interface
|
||||
protego
|
||||
itemadapter
|
||||
itemloaders
|
||||
patches = [
|
||||
# Require setuptools, https://github.com/scrapy/scrapy/pull/5122
|
||||
(fetchpatch {
|
||||
name = "add-setuptools.patch";
|
||||
url = "https://github.com/scrapy/scrapy/commit/4f500342c8ad4674b191e1fab0d1b2ac944d7d3e.patch";
|
||||
sha256 = "14030sfv1cf7dy4yww02b49mg39cfcg4bv7ys1iwycfqag3xcjda";
|
||||
})
|
||||
# Make Twisted[http2] installation optional, https://github.com/scrapy/scrapy/pull/5113
|
||||
(fetchpatch {
|
||||
name = "remove-h2.patch";
|
||||
url = "https://github.com/scrapy/scrapy/commit/c5b1ee810167266fcd259f263dbfc0fe0204761a.patch";
|
||||
sha256 = "1gw28wg8qcb0al59rz214hm17smspi6j5kg62nr1r850pykyrsqk";
|
||||
})
|
||||
];
|
||||
|
||||
LC_ALL = "en_US.UTF-8";
|
||||
|
|
@ -68,39 +93,48 @@ buildPythonPackage rec {
|
|||
substituteInPlace pytest.ini --replace "--doctest-modules" ""
|
||||
'';
|
||||
|
||||
pytestFlagsArray = [
|
||||
"--ignore=tests/test_proxy_connect.py"
|
||||
"--ignore=tests/test_utils_display.py"
|
||||
"--ignore=tests/test_command_check.py"
|
||||
disabledTestPaths = [
|
||||
"tests/test_proxy_connect.py"
|
||||
"tests/test_utils_display.py"
|
||||
"tests/test_command_check.py"
|
||||
# Don't test the documentation
|
||||
"docs"
|
||||
];
|
||||
|
||||
disabledTests = [
|
||||
# It's unclear if the failures are related to libxml2, https://github.com/NixOS/nixpkgs/pull/123890
|
||||
"test_nested_css"
|
||||
"test_nested_xpath"
|
||||
"test_flavor_detection"
|
||||
# Requires network access
|
||||
"FTPFeedStorageTest"
|
||||
"test_noconnect"
|
||||
"test_retry_dns_error"
|
||||
"FeedExportTest"
|
||||
"test_custom_asyncio_loop_enabled_true"
|
||||
"test_custom_loop_asyncio"
|
||||
"test_custom_loop_asyncio_deferred_signal"
|
||||
"FileFeedStoragePreFeedOptionsTest" # https://github.com/scrapy/scrapy/issues/5157
|
||||
] ++ lib.optionals stdenv.isDarwin [
|
||||
"test_xmliter_encoding"
|
||||
"test_download"
|
||||
];
|
||||
|
||||
src = fetchPypi {
|
||||
inherit pname version;
|
||||
sha256 = "68c48f01a58636bdf0f6fcd5035a19ecf277b58af24bd70c36dc6e556df3e005";
|
||||
};
|
||||
|
||||
postInstall = ''
|
||||
install -m 644 -D extras/scrapy.1 $out/share/man/man1/scrapy.1
|
||||
installManPage extras/scrapy.1
|
||||
install -m 644 -D extras/scrapy_bash_completion $out/share/bash-completion/completions/scrapy
|
||||
install -m 644 -D extras/scrapy_zsh_completion $out/share/zsh/site-functions/_scrapy
|
||||
'';
|
||||
|
||||
pythonImportsCheck = [ "scrapy" ];
|
||||
|
||||
__darwinAllowLocalNetworking = true;
|
||||
|
||||
meta = with lib; {
|
||||
description = "A fast high-level web crawling and web scraping framework, used to crawl websites and extract structured data from their pages";
|
||||
description = "High-level web crawling and web scraping framework";
|
||||
longDescription = ''
|
||||
Scrapy is a fast high-level web crawling and web scraping framework, used to crawl
|
||||
websites and extract structured data from their pages. It can be used for a wide
|
||||
range of purposes, from data mining to monitoring and automated testing.
|
||||
'';
|
||||
homepage = "https://scrapy.org/";
|
||||
license = licenses.bsd3;
|
||||
maintainers = with maintainers; [ drewkett marsam ];
|
||||
|
|
|
|||
Loading…
Reference in a new issue