Fix pex3 lock sync for split universal locks.

CHANGES.md

@@ -1,5 +1,11 @@
 # Release Notes
 
+## 2.97.1
+
+This release fixes `pex3 lock sync` to handle split `--style universal` locks.
+
+* Fix `pex3 lock sync` for split universal locks. (#3200)
+
 ## 2.97.0
 
 This release adds support for using `--scie-pbs-stripped` in combination with

pex/cli/commands/lock.py

@@ -81,6 +81,7 @@
 from pex.result import Error, Ok, Result, try_
 from pex.sorted_tuple import SortedTuple
 from pex.targets import LocalInterpreter, Target, Targets
+from pex.third_party.packaging.markers import Marker
 from pex.tracer import TRACER
 from pex.typing import TYPE_CHECKING, cast
 from pex.venv.virtualenv import InvalidVirtualenvError, Virtualenv
@@ -1910,12 +1911,13 @@ def _process_lock_update(
                 requirement
             )
 
-        # TODO(John Sirois): Support multiple constraints for the same project name distinguished
-        #  by markers.
-        original_constraints_by_project_name = OrderedDict(
-            (constraint.project_name, OrderedSet([constraint]))
-            for constraint in lock_file.constraints
-        )
+        original_constraints_by_project_name = (
+            OrderedDict()
+        )  # type: OrderedDict[ProjectName, OrderedSet[Constraint]]
+        for constraint in lock_file.constraints:
+            original_constraints_by_project_name.setdefault(
+                constraint.project_name, OrderedSet()
+            ).add(constraint)
         constraints_by_project_name = original_constraints_by_project_name.copy()
 
         dry_run = self.options.dry_run
@@ -1991,8 +1993,26 @@ def _process_lock_update(
                         # grab the latest version in the range already constrained by an existing
                         # requirement or constraint.
                         if update_req and str(update_req) != update_req.name:
-                            constraints_by_project_name[project_name] = OrderedSet(
-                                [update_req.as_constraint()]
+                            constraints = constraints_by_project_name.setdefault(
+                                project_name, OrderedSet()
+                            )
+                            if resolve_update.updated_resolve.marker and update_req.marker:
+                                marker = Marker(
+                                    "({first}) and ({second})".format(
+                                        first=update_req.marker,
+                                        second=resolve_update.updated_resolve.marker,
+                                    )
+                                )
+                            elif resolve_update.updated_resolve.marker:
+                                marker = resolve_update.updated_resolve.marker
+                            else:
+                                marker = update_req.marker
+                            constraints.add(
+                                Constraint(
+                                    name=update_req.name,
+                                    specifier=update_req.specifier,
+                                    marker=marker,
+                                )
                             )
                     else:
                         print(

pex/resolve/lockfile/targets.py

@@ -86,6 +86,21 @@ def _calculate_split_markers(
     return split_markers
 
 
+def _group_patch_versions(patch_version_iter):
+    # type: (Iterator[Tuple[int, int, int]]) -> OrderedDict[Tuple[int, int], OrderedSet[Tuple[int, int, int]]]
+    grouped = OrderedDict()  # type: OrderedDict[Tuple[int, int], OrderedSet[Tuple[int, int, int]]]
+    for version in patch_version_iter:
+        major_minor = version[:2]  # type: Tuple[int, int]
+        value = grouped.get(major_minor, None)
+        if value is not None:
+            version_set = value
+        else:
+            version_set = OrderedSet()
+            grouped[major_minor] = version_set
+        version_set.add(version)
+    return grouped
+
+
 def _iter_universal_targets(
     universal_target,  # type: UniversalTarget
     split_markers,  # type: Mapping[str, Marker]
@@ -108,14 +123,27 @@ def _iter_universal_targets(
         has_marker(marker, "python_full_version") for marker in split_markers.values()
     )
     if has_python_full_version:
-        requires_pythons.update(
-            SpecifierSet(
-                "=={major}.{minor}.{patch}".format(
-                    major=version[0], minor=version[1], patch=version[2]
-                )
-            )
-            for version in iter_compatible_versions(universal_target.requires_python)
+        # TODO: XXX: when iter_compatible_versions, for any complete range of major.minor, use .*
+        complete_version_sets = _group_patch_versions(iter_compatible_versions([]))
+        actual_version_sets = _group_patch_versions(
+            iter_compatible_versions(universal_target.requires_python)
         )
+        for major_minor, version_set in actual_version_sets.items():
+            if version_set == complete_version_sets.get(major_minor):
+                requires_pythons.add(
+                    SpecifierSet(
+                        "=={major}.{minor}.*".format(major=major_minor[0], minor=major_minor[1])
+                    )
+                )
+            else:
+                for version in version_set:
+                    requires_pythons.add(
+                        SpecifierSet(
+                            "=={major}.{minor}.{patch}".format(
+                                major=version[0], minor=version[1], patch=version[2]
+                            )
+                        )
+                    )
     elif has_python_version:
         requires_pythons.update(
             SpecifierSet("=={major}.{minor}.*".format(major=version[0], minor=version[1]))

pex/resolve/lockfile/updater.py

@@ -12,6 +12,7 @@
 from pex.common import pluralize
 from pex.dependency_configuration import DependencyConfiguration
 from pex.dist_metadata import Constraint, Requirement
+from pex.exceptions import production_assert
 from pex.network_configuration import NetworkConfiguration
 from pex.orderedset import OrderedSet
 from pex.pep_440 import Version
@@ -33,6 +34,7 @@
 from pex.sorted_tuple import SortedTuple
 from pex.targets import Target, Targets
 from pex.third_party.packaging import tags
+from pex.third_party.packaging.markers import Marker
 from pex.typing import TYPE_CHECKING
 from pex.util import named_temporary_file
 
@@ -287,15 +289,16 @@ def derived(
 
         deletes = tuple(original_requirements_by_project_name) if parsed_requirements else ()
 
-        # TODO(John Sirois): Support multiple constraints for the same project name distinguished
-        #  by markers.
-        original_constraints_by_project_name = OrderedDict(
-            (constraint.project_name, constraint) for constraint in lock_file.constraints
-        )  # type: OrderedDict[ProjectName, Constraint]
-
+        original_constraints_by_project_name = (
+            OrderedDict()
+        )  # type: OrderedDict[ProjectName, OrderedSet[Constraint]]
+        for constraint in lock_file.constraints:
+            original_constraints_by_project_name.setdefault(
+                constraint.project_name, OrderedSet()
+            ).add(constraint)
         update_constraints_by_project_name = (
             OrderedDict()
-        )  # type: OrderedDict[ProjectName, Constraint]
+        )  # type: OrderedDict[ProjectName, OrderedSet[Constraint]]
         for requirement in replace_requirements:
             project_name = requirement.project_name
             original_constraint = original_constraints_by_project_name.get(project_name)
@@ -305,7 +308,9 @@ def derived(
                         original=original_constraint, override=requirement
                     )
                 )
-            update_constraints_by_project_name[project_name] = requirement.as_constraint()
+            update_constraints_by_project_name.setdefault(project_name, OrderedSet()).add(
+                requirement.as_constraint()
+            )
 
         original_requirements = (
             OrderedDict()
@@ -355,13 +360,16 @@ def specified(
             (replacement.project_name, replacement) for replacement in replacements
         )
 
-        original_constraints_by_project_name = OrderedDict(
-            (constraint.project_name, constraint) for constraint in lock_file.constraints
-        )  # type: OrderedDict[ProjectName, Constraint]
-
+        original_constraints_by_project_name = (
+            OrderedDict()
+        )  # type: OrderedDict[ProjectName, OrderedSet[Constraint]]
+        for constraint in lock_file.constraints:
+            original_constraints_by_project_name.setdefault(
+                constraint.project_name, OrderedSet()
+            ).add(constraint)
         update_constraints_by_project_name = (
             OrderedDict()
-        )  # type: OrderedDict[ProjectName, Constraint]
+        )  # type: OrderedDict[ProjectName, OrderedSet[Constraint]]
         for change in itertools.chain(updates, replacements):
             project_name = change.project_name
             original_constraint = original_constraints_by_project_name.get(project_name)
@@ -371,7 +379,9 @@ def specified(
                         original=original_constraint, override=change
                     )
                 )
-            update_constraints_by_project_name[project_name] = change.as_constraint()
+            update_constraints_by_project_name.setdefault(project_name, OrderedSet()).add(
+                change.as_constraint()
+            )
 
         return cls(
             requirement_configuration=RequirementConfiguration(
@@ -392,7 +402,9 @@ def specified(
 
     requirement_configuration = attr.ib()  # type: RequirementConfiguration
     original_requirements = attr.ib()  # type: Iterable[Requirement]
-    update_constraints_by_project_name = attr.ib()  # type: Mapping[ProjectName, Constraint]
+    update_constraints_by_project_name = (
+        attr.ib()
+    )  # type: Mapping[ProjectName, Iterable[Constraint]]
     deletes = attr.ib()  # type: Container[ProjectName]
     pure_delete = attr.ib()  # type: bool
     lock_configuration = attr.ib()  # type: LockConfiguration
@@ -437,15 +449,33 @@ def _calculate_requirement_configuration(
             ):
                 continue
 
-            constraint = update_constraints_by_project_name.pop(
-                pin.project_name, pinned_requirement
-            )
-            constraints.append(str(constraint))
+            for constraint in update_constraints_by_project_name.pop(
+                pin.project_name, [pinned_requirement]
+            ):
+                if locked_resolve.marker and constraint.marker:
+                    marker = Marker(
+                        "({first}) and ({second})".format(
+                            first=constraint.marker, second=locked_resolve.marker
+                        )
+                    )
+                elif locked_resolve.marker:
+                    marker = locked_resolve.marker
+                else:
+                    marker = constraint.marker
+                constraints.append(
+                    str(
+                        Constraint(
+                            name=constraint.name, specifier=constraint.specifier, marker=marker
+                        )
+                    )
+                )
 
         # Any update constraints remaining are new requirements to resolve.
         requirements.update(
             as_parsed_requirement(constraint.as_requirement())
-            for constraint in update_constraints_by_project_name.values()
+            for constraint in itertools.chain.from_iterable(
+                update_constraints_by_project_name.values()
+            )
         )
 
         if not constraints:
@@ -536,8 +566,25 @@ def update_resolve(
                             avoid_downloads=self.avoid_downloads,
                         )
                     )
-                    assert 1 == len(updated_lock_file.locked_resolves)
-                    updated_resolve = updated_lock_file.locked_resolves[0]
+                    if len(updated_lock_file.locked_resolves) == 1:
+                        updated_resolve = updated_lock_file.locked_resolves[0]
+                    else:
+                        candidate_updates = []
+                        for (index, lr) in enumerate(updated_lock_file.locked_resolves):
+                            if (
+                                lr.marker == locked_resolve.marker
+                                and lr.platform_tag == locked_resolve.platform_tag
+                            ):
+                                candidate_updates.append(lr)
+                        production_assert(
+                            len(candidate_updates) == 1,
+                            (
+                                "Update of locked resolve should have produced one result, "
+                                "found {count}."
+                            ),
+                            count=len(candidate_updates),
+                        )
+                        updated_resolve = candidate_updates[0]
                     updated_requirements = updated_lock_file.requirements
 
         updates = OrderedDict()  # type: OrderedDict[ProjectName, Optional[Update]]

pex/version.py

@@ -1,4 +1,4 @@
 # Copyright 2015 Pex project contributors.
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-__version__ = "2.97.0"
+__version__ = "2.97.1"