Commit 2d80366a authored by Łukasz Langa's avatar Łukasz Langa
Browse files

[blib2to3] Make the grammar pickles faster

parent 65c52a65
Loading
Loading
Loading
Loading
+2 −25
Changes for blib2to3/pgen2/grammar.py: 2 added lines, 25 removed lines.
Original line number Diff line number Diff line
@@ -86,21 +86,9 @@ def __init__(self):
        self.start = 256

    def dump(self, filename):
        """Dump the grammar tables to a pickle file.

        dump() recursively changes all dict to OrderedDict, so the pickled file
        is not exactly the same as what was passed in to dump(). load() uses the
        pickled file to create the tables, but  only changes OrderedDict to dict
        at the top level; it does not recursively change OrderedDict to dict.
        So, the loaded tables are different from the original tables that were
        passed to load() in that some of the OrderedDict (from the pickled file)
        are not changed back to dict. For parsing, this has no effect on
        performance because OrderedDict uses dict's __getitem__ with nothing in
        between.
        """
        """Dump the grammar tables to a pickle file."""
        with open(filename, "wb") as f:
            d = _make_deterministic(self.__dict__)
            pickle.dump(d, f, 2)
            pickle.dump(self.__dict__, f, pickle.HIGHEST_PROTOCOL)

    def load(self, filename):
        """Load the grammar tables from a pickle file."""
@@ -141,17 +129,6 @@ def report(self):
        print("start", self.start)


def _make_deterministic(top):
    if isinstance(top, dict):
        return collections.OrderedDict(
            sorted(((k, _make_deterministic(v)) for k, v in top.items())))
    if isinstance(top, list):
        return [_make_deterministic(e) for e in top]
    if isinstance(top, tuple):
        return tuple(_make_deterministic(e) for e in top)
    return top


# Map from operator to number (since tokenize doesn't do this)

opmap_raw = """