Improve fallback system (9d7a741d) · Commits · mirrors / poetry

docs/docs/basic-usage.md

+12 −6

Original line number	Diff line number	Diff line
		@@ -69,18 +69,24 @@ It will automatically find a suitable version constraint.
		information due to bad packaging/publishing which means that `poetry` won't
		be able to properly resolve dependencies.

		To workaround it you can set the missing dependencies yourself in your `pyproject.toml`
		or you can tell `poetry` to use a fallback mechanism by setting the
		`settings.pypi.fallback` setting to `true`.
		To workaround it, `poetry` has a fallback mechanism that will download packages
		distributions to check the dependencies.

		While, in most cases, it will lead to a more exhaustive dependency resolution
		it will also considerably slow down the process (up to 30 minutes in some extreme cases
		like `boto3`).

		If you do not want the fallback mechanism, you can deactivate it like so.

		```bash
		poetry config settings.pypi.fallback true
		poetry config settings.pypi.fallback false
		```

		Note that this is temporary and should be avoided as much as possible since
		it increases the dependency resolution time drastically (up to 30 minutes in some cases).
		In this case you will need to specify the missing dependencies in you `pyproject.toml`
		file.

		Any case of missing dependencies should be reported to https://github.com/sdispater/poetry/issues
		and on the repository of the main package.

		### Version constraints

poetry/poetry.py

+1 −11

Original line number	Diff line number	Diff line
		@@ -42,19 +42,9 @@ class Poetry:
		# Always put PyPI last to prefere private repositories
		self._pool.add_repository(
		PyPiRepository(
		fallback=self._config.setting(
		'settings.pypi.fallback',
		False
		fallback=self._config.setting('settings.pypi.fallback', True)
		)
		)
		)

		# Adding a fallback for PyPI for when dependencies
		# are not retrievable via the JSON API
		self._pool.configure({
		'name': 'pypi-fallback',
		'url': 'https://pypi.org/simple'
		})

		@property
		def file(self):

poetry/repositories/pypi_repository.py

+184 −8

Original line number	Diff line number	Diff line
		import os
		import tarfile
		import zipfile

		import pkginfo

		from bz2 import BZ2File
		from gzip import GzipFile
		from typing import List
		from typing import Union

		try:
		import urllib.parse as urlparse
		except ImportError:
		import urlparse

		try:
		from xmlrpc.client import ServerProxy
		except ImportError:
		@@ -9,6 +22,7 @@ except ImportError:
		from cachecontrol import CacheControl
		from cachecontrol.caches.file_cache import FileCache
		from cachy import CacheManager
		from requests import get
		from requests import session

		from poetry.locations import CACHE_DIR
		@@ -18,6 +32,7 @@ from poetry.semver.constraints import Constraint
		from poetry.semver.constraints.base_constraint import BaseConstraint
		from poetry.semver.version_parser import VersionParser
		from poetry.utils._compat import Path
		from poetry.utils.helpers import temporary_directory
		from poetry.version.markers import InvalidMarker

		from .repository import Repository
		@@ -28,7 +43,7 @@ class PyPiRepository(Repository):
		def __init__(self,
		url='https://pypi.org/',
		disable_cache=False,
		fallback=False):
		fallback=True):
		self._url = url
		self._disable_cache = disable_cache
		self._fallback = fallback
		@@ -103,13 +118,11 @@ class PyPiRepository(Repository):
		self._fallback
		and release_info['requires_dist'] is None
		and not release_info['requires_python']
		and '_fallback' not in release_info
		):
		# No dependencies set (along with other information)
		# This might be due to actually no dependencies
		# or badly set metadata when uploading
		# So, we return None so that the fallback repository
		# can pick up more accurate info
		return
		# Force cache update
		self._cache.forget('{}:{}'.format(name, version))
		release_info = self.get_release_info(name, version)

		package = Package(name, version, version)
		requires_dist = release_info['requires_dist'] or []
		@@ -230,7 +243,8 @@ class PyPiRepository(Repository):
		'platform': info['platform'],
		'requires_dist': info['requires_dist'],
		'requires_python': info['requires_python'],
		'digests': []
		'digests': [],
		'_fallback': False
		}

		try:
		@@ -241,6 +255,50 @@ class PyPiRepository(Repository):
		for file_info in version_info:
		data['digests'].append(file_info['digests']['sha256'])

		if (
		self._fallback
		and data['requires_dist'] is None
		and not data['requires_python']
		):
		# No dependencies set (along with other information)
		# This might be due to actually no dependencies
		# or badly set metadata when uploading
		# So, we need to make sure there is actually no
		# dependencies by introspecting packages
		data['_fallback'] = True

		urls = {}
		for url in json_data['urls']:
		# Only get sdist and universal wheels
		dist_type = url['packagetype']
		if dist_type not in ['sdist', 'bdist_wheel']:
		continue

		if dist_type == 'sdist' and 'dist' not in urls:
		urls[url['packagetype']] = url['url']
		continue

		if 'bdist_wheel' in urls:
		continue

		# If bdist_wheel, check if it's universal
		python_version = url['python_version']
		if python_version not in ['py2.py3', 'py3', 'py2']:
		continue

		parts = urlparse.urlparse(url['url'])
		filename = os.path.basename(parts.path)

		if '-none-any' not in filename:
		continue

		if not urls:
		return data

		requires_dist = self._get_requires_dist_from_urls(urls)

		data['requires_dist'] = requires_dist

		return data

		def _get(self, endpoint): # type: (str) -> Union[dict, None]
		@@ -251,3 +309,121 @@ class PyPiRepository(Repository):
		json_data = json_response.json()

		return json_data

		def _get_requires_dist_from_urls(self, urls
		): # type: (dict) -> Union[list, None]
		if 'bdist_wheel' in urls:
		return self._get_requires_dist_from_wheel(urls['bdist_wheek'])

		return self._get_requires_dist_from_sdist(urls['sdist'])

		def _get_requires_dist_from_wheel(self, url
		): # type: (str) -> Union[list, None]
		filename = os.path.basename(urlparse.urlparse(url).path)

		with temporary_directory() as temp_dir:
		filepath = os.path.join(temp_dir, filename)
		self._download(url, filepath)

		meta = pkginfo.Wheel(filepath)

		if meta.requires_dist:
		return meta.requires_dist

		def _get_requires_dist_from_sdist(self, url
		): # type: (str) -> Union[list, None]
		filename = os.path.basename(urlparse.urlparse(url).path)

		with temporary_directory() as temp_dir:
		filepath = Path(temp_dir) / filename
		self._download(url, str(filepath))

		meta = pkginfo.SDist(str(filepath))

		if meta.requires_dist:
		return meta.requires_dist

		# Still not dependencies found
		# So, we unpack and introspect
		suffix = filepath.suffix
		if suffix == '.zip':
		tar = zipfile.ZipFile(str(filepath))
		else:
		if suffix == '.bz2':
		gz = BZ2File(str(filepath))
		else:
		gz = GzipFile(str(filepath))

		tar = tarfile.TarFile(str(filepath), fileobj=gz)

		tar.extractall(os.path.join(temp_dir, 'unpacked'))

		unpacked = Path(temp_dir) / 'unpacked'
		sdist_dir = unpacked / Path(filename).name.rstrip('.tar.gz')

		# Checking for .egg-info
		eggs = list(sdist_dir.glob('*.egg-info'))
		if eggs:
		egg_info = eggs[0]

		requires = egg_info / 'requires.txt'
		if requires.exists():
		with requires.open() as f:
		return self._parse_requires(f.read())

		return

		# Still nothing, assume no dependencies
		# We could probably get them by executing
		# python setup.py egg-info but I don't feel
		# confortable executing a file just for the sake
		# of getting dependencies.
		return

		def _download(self, url, dest): # type: (str, str) -> None
		r = get(url, stream=True)
		with open(dest, 'wb') as f:
		for chunk in r.iter_content(chunk_size=1024):
		if chunk:
		f.write(chunk)

		def _parse_requires(self, requires): # type: (str) -> Union[list, None]
		lines = requires.split('\n')

		requires_dist = []
		in_section = False
		current_marker = None
		for line in lines:
		line = line.strip()
		if not line:
		if in_section:
		in_section = False

		continue

		if line.startswith('['):
		# extras or conditional dependencies
		marker = line.lstrip('[').rstrip(']')
		if ':' not in marker:
		extra, marker = marker, None
		else:
		extra, marker = marker.split(':')

		if extra:
		if marker:
		marker = '{} and extra == "{}"'.format(marker, extra)
		else:
		marker = 'extra == "{}"'.format(extra)

		if marker:
		current_marker = marker

		continue

		if current_marker:
		line = '{}; {}'.format(line, current_marker)

		requires_dist.append(line)

		if requires_dist:
		return requires_dist

poetry/utils/helpers.py

+19 −0

Original line number	Diff line number	Diff line
		import re
		import shutil
		import tempfile

		from contextlib import contextmanager

		_canonicalize_regex = re.compile('[-_.]+')

		@@ -9,3 +13,18 @@ def canonicalize_name(name): # type: (str) -> str

		def module_name(name): # type: (str) -> str
		return canonicalize_name(name).replace('-', '_')


		@contextmanager
		def temporary_directory(args, *kwargs):
		try:
		from tempfile import TemporaryDirectory

		with TemporaryDirectory(args, *kwargs) as name:
		yield name
		except ImportError:
		name = tempfile.mkdtemp(args, *kwargs)

		yield name

		shutil.rmtree(name)

tests/repositories/test_pypi_repository.py

+51 −1

Original line number	Diff line number	Diff line
		@@ -11,7 +11,8 @@ class MockRepository(PyPiRepository):
		def __init__(self):
		super(MockRepository, self).__init__(
		url='http://foo.bar',
		disable_cache=True
		disable_cache=True,
		fallback=False
		)

		def _get(self, url):
		@@ -63,3 +64,52 @@ def test_package_drops_malformed_dependencies():
		dependency_names = [d.name for d in package.requires]

		assert 'setuptools' not in dependency_names


		def test_parse_requires():
		requires = """\
		jsonschema>=2.6.0.0,<3.0.0.0
		lockfile>=0.12.0.0,<0.13.0.0
		pip-tools>=1.11.0.0,<2.0.0.0
		pkginfo>=1.4.0.0,<2.0.0.0
		pyrsistent>=0.14.2.0,<0.15.0.0
		toml>=0.9.0.0,<0.10.0.0
		cleo>=0.6.0.0,<0.7.0.0
		cachy>=0.1.1.0,<0.2.0.0
		cachecontrol>=0.12.4.0,<0.13.0.0
		requests>=2.18.0.0,<3.0.0.0
		msgpack-python>=0.5.0.0,<0.6.0.0
		pyparsing>=2.2.0.0,<3.0.0.0
		requests-toolbelt>=0.8.0.0,<0.9.0.0

		[:(python_version >= "2.7.0.0" and python_version < "2.8.0.0") or (python_version >= "3.4.0.0" and python_version < "3.5.0.0")]
		typing>=3.6.0.0,<4.0.0.0

		[:python_version >= "2.7.0.0" and python_version < "2.8.0.0"]
		virtualenv>=15.2.0.0,<16.0.0.0
		pathlib2>=2.3.0.0,<3.0.0.0

		[:python_version >= "3.4.0.0" and python_version < "3.6.0.0"]
		zipfile36>=0.1.0.0,<0.2.0.0
		"""
		result = MockRepository()._parse_requires(requires)
		expected = [
		'jsonschema>=2.6.0.0,<3.0.0.0',
		'lockfile>=0.12.0.0,<0.13.0.0',
		'pip-tools>=1.11.0.0,<2.0.0.0',
		'pkginfo>=1.4.0.0,<2.0.0.0',
		'pyrsistent>=0.14.2.0,<0.15.0.0',
		'toml>=0.9.0.0,<0.10.0.0',
		'cleo>=0.6.0.0,<0.7.0.0',
		'cachy>=0.1.1.0,<0.2.0.0',
		'cachecontrol>=0.12.4.0,<0.13.0.0',
		'requests>=2.18.0.0,<3.0.0.0',
		'msgpack-python>=0.5.0.0,<0.6.0.0',
		'pyparsing>=2.2.0.0,<3.0.0.0',
		'requests-toolbelt>=0.8.0.0,<0.9.0.0',
		'typing>=3.6.0.0,<4.0.0.0; (python_version >= "2.7.0.0" and python_version < "2.8.0.0") or (python_version >= "3.4.0.0" and python_version < "3.5.0.0")',
		'virtualenv>=15.2.0.0,<16.0.0.0; python_version >= "2.7.0.0" and python_version < "2.8.0.0"',
		'pathlib2>=2.3.0.0,<3.0.0.0; python_version >= "2.7.0.0" and python_version < "2.8.0.0"',
		'zipfile36>=0.1.0.0,<0.2.0.0; python_version >= "3.4.0.0" and python_version < "3.6.0.0"'
		]
		assert result == expected