Unverified Commit ce69554f authored by Sébastien Eustace's avatar Sébastien Eustace
Browse files

No longer depend on twine to upload

parent 692fe626
Loading
Loading
Loading
Loading
+1 −1
Changes for poetry/console/commands/publish.py: 1 added line, 1 removed line.
Original line number Diff line number Diff line
from poetry.masonry.publisher import Publisher
from poetry.masonry.publishing.publisher import Publisher

from .command import Command

+71 −0
Changes for poetry/masonry/metadata.py: 71 added lines, 0 removed lines.
Original line number Diff line number Diff line
from poetry.semver.constraints import MultiConstraint
from poetry.utils.helpers import canonicalize_name


class Metadata:

    metadata_version = '1.2'
    # version 1.0
    name = None
    version = None
    platforms = ()
    supported_platforms = ()
    summary = None
    description = None
    keywords = None
    home_page = None
    download_url = None
    author = None
    author_email = None
    license = None
    # version 1.1
    classifiers = ()
    requires = ()
    provides = ()
    obsoletes = ()
    # version 1.2
    maintainer = None
    maintainer_email = None
    requires_python = None
    requires_external = ()
    requires_dist = ()
    provides_dist = ()
    obsoletes_dist = ()
    project_urls = ()

    @classmethod
    def from_package(cls, package) -> 'Metadata':
        meta = cls()

        meta.name = canonicalize_name(package.name)
        meta.version = package.version
        meta.summary = package.description
        meta.description = package.readme
        meta.keywords = ','.join(package.keywords)
        meta.home_page = package.homepage or package.repository_url
        meta.author = package.author_name
        meta.author_email = package.author_email
        meta.license = package.license
        meta.classifiers = package.classifiers

        # Version 1.1
        meta.requires = [d.to_pep_508() for d in package.requires]

        # Version 1.2
        meta.maintainer = meta.author
        meta.maintainer_email = meta.author_email
        meta.requires_python = package.python_constraint
        meta.requires_dist = [d.to_pep_508() for d in package.requires]

        # Requires python
        constraint = package.python_constraint
        if isinstance(constraint, MultiConstraint):
            python_requires = ','.join(
                [str(c).replace(' ', '') for c in constraint.constraints]
            )
        else:
            python_requires = str(constraint).replace(' ', '')

        meta.requires_python = python_requires

        return meta
+1 −0
Changes for poetry/masonry/publishing/__init__.py: 1 added line, 0 removed lines.
Original line number Diff line number Diff line
from .publisher import Publisher
+201 −130
Changes for poetry/masonry/publishing/publisher.py: 201 added lines, 130 removed lines.
Original line number Diff line number Diff line
# -*- coding: utf-8 -*-
import hashlib
import io
import re

import requests
import toml
import twine.utils

from pathlib import Path

from requests import adapters
from requests.exceptions import HTTPError
from twine.commands.upload import find_dists, skip_upload
from twine.repository import Repository as BaseRepository
from twine.exceptions import PackageNotFound, RedirectDetected
from twine.package import PackageFile
from requests.packages.urllib3 import util
from requests_toolbelt import user_agent
from requests_toolbelt.multipart import (
    MultipartEncoder, MultipartEncoderMonitor
)

from poetry import __version__
from poetry.locations import CONFIG_DIR

from ..metadata import Metadata

class Repository(BaseRepository):

    def __init__(self, io, url, username, password):
        self._io = io

        super(Repository, self).__init__(url, username, password)

    def register(self, package):
        data = package.metadata_dictionary()
        data.update({
            ":action": "submit",
            "protocol_version": "1",
        })

        self._io.writeln(
            " - Registering <info>{0}</>".format(package.basefilename)
        )

        data_to_send = self._convert_data_to_list_of_tuples(data)
        encoder = MultipartEncoder(data_to_send)
        resp = self.session.post(
            self.url,
            data=encoder,
            allow_redirects=False,
            headers={'Content-Type': encoder.content_type}
wheel_file_re = re.compile(
    r"""^(?P<namever>(?P<name>.+?)(-(?P<ver>\d.+?))?)
        ((-(?P<build>\d.*?))?-(?P<pyver>.+?)-(?P<abi>.+?)-(?P<plat>.+?)
        \.whl|\.dist-info)$""",
    re.VERBOSE
)
        # Bug 28. Try to silence a ResourceWarning by releasing the socket.
        resp.close()

        return resp

    def _upload(self, package):
        data = package.metadata_dictionary()
        data.update({
            # action
            ":action": "file_upload",
            "protocol_version": "1",
        })
KEYWORDS_TO_NOT_FLATTEN = {'gpg_signature', 'content'}

        data_to_send = self._convert_data_to_list_of_tuples(data)

        with open(package.filename, "rb") as fp:
            data_to_send.append((
                "content",
                (package.basefilename, fp, "application/octet-stream"),
            ))
            encoder = MultipartEncoder(data_to_send)
            bar = self._io.create_progress_bar(encoder.len)
            bar.set_format(
                " - Uploading <info>{0}</> <comment>%percent%%</>".format(
                    package.basefilename
                )
            )
            monitor = MultipartEncoderMonitor(
                encoder, lambda monitor: bar.set_progress(monitor.bytes_read)
            )

            bar.start()

            resp = self.session.post(
                self.url,
                data=monitor,
                allow_redirects=False,
                headers={'Content-Type': monitor.content_type}
            )

            if resp.ok:
                bar.finish()

                self._io.writeln('')
            else:
                self._io.overwrite('')

        return resp


class Publisher(object):
class Publisher:
    """
    Registers and publishes packages to remote repositories.
    """
@@ -162,44 +100,54 @@ class Publisher(object):
        if not password:
            password = self._io.ask_hidden('Password:')

        repository = Repository(self._io, url, username, password)
        session = requests.session()
        session.auth = (username, password)
        session.headers['User-Agent'] = self._make_user_agent_string()
        for scheme in ('http://', 'https://'):
            session.mount(scheme, self._make_adapter_with_retries())

        # TODO: handle certificates

        self.upload(repository)
        try:
            self.upload(session, url)
        finally:
            session.close()

    def register(self, repository):
    def register(self, session, url):
        """
        Register a package to a repository.
        """
        dist = self._poetry.file.parent / 'dist'
        package = dist / f'{self._package.name}-{self._package.version}.tar.gz'

        if package.exists():
            raise PackageNotFound(
                '"{0}" does not exist on the file system.'.format(package)
            )
        file = dist / f'{self._package.name}-{self._package.version}.tar.gz'

        resp = repository.register(
            PackageFile.from_filename(str(package), None)
        if not file.exists():
            raise RuntimeError(
                '"{0}" does not exist on the file system.'.format(file.name)
            )

        repository.close()
        data = self.post_data(file)
        data.update({
            ":action": "submit",
            "protocol_version": "1",
        })

        if resp.is_redirect:
            raise RedirectDetected(
                ('"{0}" attempted to redirect to "{1}" during registration.'
                 ' Aborting...').format(repository.url,
                                        resp.headers["location"]))
        data_to_send = self._convert_data_to_list_of_tuples(data)
        encoder = MultipartEncoder(data_to_send)
        resp = session.post(
            url,
            data=encoder,
            allow_redirects=False,
            headers={'Content-Type': encoder.content_type},
        )

        resp.raise_for_status()
        return resp

    def upload(self, repository):
    def upload(self, session, url):
        """
        Upload packages for the current project.
        """
        try:
            self._upload(repository)
            self._upload(session, url)
        except HTTPError as e:
            if (
                e.response.status_code not in (403, 400)
@@ -210,53 +158,176 @@ class Publisher(object):
            # It may be the first time we publish the package
            # We'll try to register it and go from there
            try:
                self.register(repository)
                self.register(session, url)
            except HTTPError:
                raise

    def _upload(self, repository):
        skip_existing = False
    def post_data(self, file):
        meta = Metadata.from_package(self._package)

        file_type = self._get_type(file)

        blake2_256_hash = hashlib.blake2b(digest_size=256 // 8)
        md5_hash = hashlib.md5()
        sha2_hash = hashlib.sha256()
        with file.open('rb') as fp:
            for content in iter(lambda: fp.read(io.DEFAULT_BUFFER_SIZE), b''):
                md5_hash.update(content)
                sha2_hash.update(content)
                blake2_256_hash.update(content)

        md5_digest = md5_hash.hexdigest()
        sha2_digest = sha2_hash.hexdigest()
        blake2_256_digest = blake2_256_hash.hexdigest()

        if file_type == 'bdist_wheel':
            wheel_info = wheel_file_re.match(file.name)
            py_version = wheel_info.group("pyver")
        else:
            py_version = None

        return {
            # identify release
            "name": meta.name,
            "version": meta.version,

            # file content
            "filetype": file_type,
            "pyversion": py_version,

            # additional meta-data
            "metadata_version": meta.metadata_version,
            "summary": meta.summary,
            "home_page": meta.home_page,
            "author": meta.author,
            "author_email": meta.author_email,
            "maintainer": meta.maintainer,
            "maintainer_email": meta.maintainer_email,
            "license": meta.license,
            "description": meta.description,
            "keywords": meta.keywords,
            "platform": meta.platforms,
            "classifiers": meta.classifiers,
            "download_url": meta.download_url,
            "supported_platform": meta.supported_platforms,
            "comment": None,
            "md5_digest": md5_digest,
            "sha256_digest": sha2_digest,
            "blake2_256_digest": blake2_256_digest,

            # PEP 314
            "provides": meta.provides,
            "requires": meta.requires,
            "obsoletes": meta.obsoletes,

            # Metadata 1.2
            "project_urls": meta.project_urls,
            "provides_dist": meta.provides_dist,
            "obsoletes_dist": meta.obsoletes_dist,
            "requires_dist": meta.requires_dist,
            "requires_external": meta.requires_external,
            "requires_python": meta.requires_python,
        }

    def _upload(self, session, url):
        dist = self._poetry.file.parent / 'dist'
        packages = list(dist.glob(f'{self._package.name}-{self._package.version}*'))
        dists = find_dists([str(p) for p in packages])
        uploads = [i for i in dists if not i.endswith(".asc")]

        for filename in uploads:
            package = PackageFile.from_filename(filename, None)
            skip_message = (
                " - Skipping <comment>{0}</> because it appears to already exist"
                .format(
                    package.basefilename
                )
            )
        packages = dist.glob(f'{self._package.name}-{self._package.version}*')
        files = (i for i in packages if i.suffix != '.asc')

            # Note: The skip_existing check *needs* to be first, because otherwise
            #       we're going to generate extra HTTP requests against a hardcoded
            #       URL for no reason.
            if skip_existing and repository.package_is_uploaded(package):
                self._io.writeln(skip_message)
                continue
        for file in files:
            # TODO; Check existence

            resp = repository.upload(package)
            resp = self._upload_file(session, url, file)

            # Bug 92. If we get a redirect we should abort because something seems
            # funky. The behaviour is not well defined and redirects being issued
            # by PyPI should never happen in reality. This should catch malicious
            # redirects as well.
            if resp.is_redirect:
                raise RedirectDetected(
                raise RuntimeError(
                    ('"{0}" attempted to redirect to "{1}" during upload.'
                     ' Aborting...').format(repository.url,
                                            resp.headers["location"]))
                     ' Aborting...').format(url, resp.headers["location"]))

            if skip_upload(resp, skip_existing, package):
                self._io.writeln(skip_message)
            resp.raise_for_status()

                continue
    def _upload_file(self, session, url, file):
        data = self.post_data(file)
        data.update({
            # action
            ":action": "file_upload",
            "protocol_version": "1",
        })

            twine.utils.check_status_code(resp)
        data_to_send = self._convert_data_to_list_of_tuples(data)

        # Bug 28. Try to silence a ResourceWarning by clearing the connection
        # pool.
        repository.close()
        with file.open('rb') as fp:
            data_to_send.append((
                "content",
                (file.name, fp, "application/octet-stream"),
            ))
            encoder = MultipartEncoder(data_to_send)
            bar = self._io.create_progress_bar(encoder.len)
            bar.set_format(
                " - Uploading <info>{0}</> <comment>%percent%%</>".format(
                    file.name
                )
            )
            monitor = MultipartEncoderMonitor(
                encoder, lambda monitor: bar.set_progress(monitor.bytes_read)
            )

            bar.start()

            resp = session.post(
                url,
                data=monitor,
                allow_redirects=False,
                headers={'Content-Type': monitor.content_type}
            )

            if resp.ok:
                bar.finish()

                self._io.writeln('')
            else:
                self._io.overwrite('')

        return resp

    def _convert_data_to_list_of_tuples(self, data):
        data_to_send = []
        for key, value in data.items():
            if (key in KEYWORDS_TO_NOT_FLATTEN or
                    not isinstance(value, (list, tuple))):
                data_to_send.append((key, value))
            else:
                for item in value:
                    data_to_send.append((key, item))
        return data_to_send

    def _get_type(self, file):
        exts = file.suffixes
        if exts[-1] == '.whl':
            return 'bdist_wheel'
        elif len(exts) >= 2 and ''.join(exts[-2:]) == '.tar.gz':
            return 'sdist'

        raise ValueError(
            f'Unknown distribution format {"".join(exts)}'
        )

    @staticmethod
    def _make_adapter_with_retries():
        retry = util.Retry(
            connect=5,
            total=10,
            method_whitelist=['GET'],
            status_forcelist=[500, 501, 502, 503],
        )
        return adapters.HTTPAdapter(max_retries=retry)

    @staticmethod
    def _make_user_agent_string():
        return user_agent(
            'twine', __version__,
        )
+32 −0
Changes for poetry/packages/package.py: 32 added lines, 0 removed lines.
Original line number Diff line number Diff line
import re
from typing import Union

from poetry.semver.constraints import Constraint
from poetry.semver.helpers import parse_stability
from poetry.semver.version_parser import VersionParser
from poetry.version import parse as parse_version
@@ -13,6 +14,13 @@ AUTHOR_REGEX = re.compile('(?u)^(?P<name>[- .,\w\d\'’"()]+) <(?P<email>.+?)>$'

class Package:

    AVAILABLE_PYTHONS = {
        '2',
        '2.7',
        '3',
        '3.4', '3.5', '3.6', '3.7'
    }

    supported_link_types = {
        'require': {
            'description': 'requires',
@@ -174,6 +182,30 @@ class Package:
    def platform_constraint(self):
        return self._platform_constraint
    
    @property
    def classifiers(self):
        classifiers = []

        # Automatically set python classifiers
        parser = VersionParser()
        if self.python_versions == '*':
            python_constraint = parser.parse_constraints('~2.7 || ^3.4')
        else:
            python_constraint = self.python_constraint

        for version in sorted(self.AVAILABLE_PYTHONS):
            if len(version) == 1:
                constraint = parser.parse_constraints(version + '.*')
            else:
                constraint = Constraint('=', version)

            if python_constraint.matches(constraint):
                classifiers.append(
                    f'Programming Language :: Python :: {version}'
                )

        return classifiers

    def is_dev(self):
        return self._dev

Loading