Add RequestGroupSearchContext class

There are cases where exactly the same database query is issued several
times in a request group, such as provider_ids_matching_aggregates().

This patch creates a new object, `RequestGroupSearchContext` to cache
the results not to ask to issue the same query in a request group.

The main refatctoring to reduce the number of the same query will be
done in following patches.

This patch brings a slight optimzation to skip retrieving sharing
providers for `use_same_provder=True` case.

Change-Id: Ifbaac9861f86d85a5bff58573c30a4cf957503d8
Story: 2005712
Task: 31038

placement/exception.py

@@ -83,6 +83,10 @@ class ResourceProviderConcurrentUpdateDetected(ConcurrentUpdateDetected):
                "data. Please retry your update")
 
 
+class ResourceProviderNotFound(NotFound):
+    msg_fmt = "No such resource provider(s)"
+
+
 class InvalidAllocationCapacityExceeded(InvalidInventory):
     msg_fmt = ("Unable to create allocation for '%(resource_class)s' on "
                "resource provider '%(resource_provider)s'. The requested "

placement/objects/allocation_candidate.py

@@ -24,6 +24,8 @@ from sqlalchemy import sql
 
 from placement.db.sqlalchemy import models
 from placement import db_api
+from placement import exception
+from placement.objects import research_context as res_ctx
 from placement.objects import resource_provider as rp_obj
 from placement.objects import trait as trait_obj
 from placement import resource_class_cache as rc_cache
@@ -93,107 +95,57 @@ class AllocationCandidates(object):
         )
 
     @staticmethod
-    def _get_by_one_request(context, request, sharing_providers, has_trees):
+    def _get_by_one_request(rg_ctx):
         """Get allocation candidates for one RequestGroup.
 
         Must be called from within an placement_context_manager.reader
         (or writer) context.
 
-        :param context: Nova RequestContext.
-        :param request: One placement.lib.RequestGroup
-        :param sharing_providers: dict, keyed by resource class internal ID, of
-                                  the set of provider IDs containing shared
-                                  inventory of that resource class
-        :param has_trees: bool indicating there is some level of nesting in the
-                          environment (if there isn't, we take faster, simpler
-                          code paths)
-        :return: A tuple of (allocation_requests, provider_summaries)
-                 satisfying `request`.
+        :param rg_ctx: RequestGroupSearchContext.
         """
-        # Transform resource string names to internal integer IDs
-        resources = {
-            rc_cache.RC_CACHE.id_from_string(key): value
-            for key, value in request.resources.items()
-        }
-
-        # maps the trait name to the trait internal ID
-        required_trait_map = {}
-        forbidden_trait_map = {}
-        for trait_map, traits in (
-                (required_trait_map, request.required_traits),
-                (forbidden_trait_map, request.forbidden_traits)):
-            if traits:
-                trait_map.update(trait_obj.ids_from_names(context, traits))
-
-        member_of = request.member_of
-        forbidden_aggs = request.forbidden_aggs
-
-        tree_root_id = None
-        if request.in_tree:
-            tree_ids = rp_obj.provider_ids_from_uuid(context, request.in_tree)
-            if tree_ids is None:
-                # List operations should simply return an empty list when a
-                # non-existing resource provider UUID is given for in_tree.
-                return [], []
-            tree_root_id = tree_ids.root_id
-            LOG.debug("getting allocation candidates in the same tree "
-                      "with the root provider %s", tree_ids.root_uuid)
-
-        any_sharing = any(sharing_providers.values())
-        if not request.use_same_provider and (has_trees or any_sharing):
+        if not rg_ctx.use_same_provider and (
+                rg_ctx.exists_sharing or rg_ctx.exists_nested):
             # TODO(jaypipes): The check/callout to handle trees goes here.
             # Build a dict, keyed by resource class internal ID, of lists of
             # internal IDs of resource providers that share some inventory for
             # each resource class requested.
             # If there aren't any providers that have any of the
             # required traits, just exit early...
-            if required_trait_map:
+            if rg_ctx.required_trait_map:
                 # TODO(cdent): Now that there is also a forbidden_trait_map
                 # it should be possible to further optimize this attempt at
                 # a quick return, but we leave that to future patches for
                 # now.
                 trait_rps = rp_obj.get_provider_ids_having_any_trait(
-                    context, required_trait_map)
+                    rg_ctx.context, rg_ctx.required_trait_map)
                 if not trait_rps:
                     return [], []
-            rp_candidates = rp_obj.get_trees_matching_all(
-                context, resources, required_trait_map, forbidden_trait_map,
-                sharing_providers, member_of, forbidden_aggs, tree_root_id)
-            return _alloc_candidates_multiple_providers(
-                context, resources, required_trait_map, forbidden_trait_map,
-                rp_candidates)
+            rp_candidates = rp_obj.get_trees_matching_all(rg_ctx)
+            return _alloc_candidates_multiple_providers(rg_ctx, rp_candidates)
 
         # Either we are processing a single-RP request group, or there are no
         # sharing providers that (help) satisfy the request.  Get a list of
         # tuples of (internal provider ID, root provider ID) that have ALL
         # the requested resources and more efficiently construct the
         # allocation requests.
-        rp_tuples = rp_obj.get_provider_ids_matching(
-            context, resources, required_trait_map, forbidden_trait_map,
-            member_of, forbidden_aggs, tree_root_id)
-        return _alloc_candidates_single_provider(context, resources, rp_tuples)
+        rp_tuples = rp_obj.get_provider_ids_matching(rg_ctx)
+        return _alloc_candidates_single_provider(rg_ctx, rp_tuples)
 
     @classmethod
     @db_api.placement_context_manager.reader
     def _get_by_requests(cls, context, requests, limit=None,
                          group_policy=None, nested_aware=True):
-        # TODO(jaypipes): Make a RequestGroupContext object and put these
-        # pieces of information in there, passing the context to the various
-        # internal functions handling that part of the request.
-        sharing = {}
-        for request in requests.values():
-            member_of = request.member_of
-            for rc_name, amount in request.resources.items():
-                rc_id = rc_cache.RC_CACHE.id_from_string(rc_name)
-                if rc_id not in sharing:
-                    sharing[rc_id] = rp_obj.get_providers_with_shared_capacity(
-                        context, rc_id, amount, member_of)
         has_trees = rp_obj.has_provider_trees(context)
 
         candidates = {}
         for suffix, request in requests.items():
-            alloc_reqs, summaries = cls._get_by_one_request(
-                context, request, sharing, has_trees)
+            try:
+                rg_ctx = res_ctx.RequestGroupSearchContext(
+                    context, request, has_trees)
+            except exception.ResourceProviderNotFound:
+                return [], []
+
+            alloc_reqs, summaries = cls._get_by_one_request(rg_ctx)
             LOG.debug("%s (suffix '%s') returned %d matches",
                       str(request), str(suffix), len(alloc_reqs))
             if not alloc_reqs:
@@ -331,9 +283,7 @@ class ProviderSummaryResource(object):
         self.max_unit = max_unit
 
 
-def _alloc_candidates_multiple_providers(
-    ctx, requested_resources, required_traits, forbidden_traits,
-        rp_candidates):
+def _alloc_candidates_multiple_providers(rg_ctx, rp_candidates):
     """Returns a tuple of (allocation requests, provider summaries) for a
     supplied set of requested resource amounts and tuples of
     (rp_id, root_id, rc_id). The supplied resource provider trees have
@@ -345,16 +295,7 @@ def _alloc_candidates_multiple_providers(
     providers within the same provider tree including sharing providers to
     satisfy different resources involved in a single request group.
 
-    :param ctx: placement.context.RequestContext object
-    :param requested_resources: dict, keyed by resource class ID, of amounts
-                                being requested for that resource class
-    :param required_traits: A map, keyed by trait string name, of required
-                            trait internal IDs that each *allocation request's
-                            set of providers* must *collectively* have
-                            associated with them
-    :param forbidden_traits: A map, keyed by trait string name, of trait
-                             internal IDs that a resource provider must
-                             not have.
+    :param rg_ctx: RequestGroupSearchContext.
     :param rp_candidates: RPCandidates object representing the providers
                           that satisfy the request for resources.
     """
@@ -367,15 +308,16 @@ def _alloc_candidates_multiple_providers(
     root_ids = rp_candidates.all_rps
 
     # Grab usage summaries for each provider in the trees
-    usages = _get_usages_by_provider_tree(ctx, root_ids)
+    usages = _get_usages_by_provider_tree(rg_ctx.context, root_ids)
 
     # Get a dict, keyed by resource provider internal ID, of trait string names
     # that provider has associated with it
-    prov_traits = trait_obj.get_traits_by_provider_tree(ctx, root_ids)
+    prov_traits = trait_obj.get_traits_by_provider_tree(
+        rg_ctx.context, root_ids)
 
     # Get a dict, keyed by resource provider internal ID, of ProviderSummary
     # objects for all providers
-    summaries = _build_provider_summaries(ctx, usages, prov_traits)
+    summaries = _build_provider_summaries(rg_ctx.context, usages, prov_traits)
 
     # Get a dict, keyed by root provider internal ID, of a dict, keyed by
     # resource class internal ID, of lists of AllocationRequestResource objects
@@ -387,7 +329,7 @@ def _alloc_candidates_multiple_providers(
             AllocationRequestResource(
                 resource_provider=rp_summary.resource_provider,
                 resource_class=rc_cache.RC_CACHE.string_from_id(rp.rc_id),
-                amount=requested_resources[rp.rc_id]))
+                amount=rg_ctx.resources[rp.rc_id]))
 
     # Next, build up a set of allocation requests. These allocation requests
     # are AllocationRequest objects, containing resource provider UUIDs,
@@ -422,8 +364,9 @@ def _alloc_candidates_multiple_providers(
         #  (ARR(rc1, ss2), ARR(rc2, ss2), ARR(rc3, ss1))]
         for res_requests in itertools.product(*request_groups):
             if not _check_traits_for_alloc_request(
-                    res_requests, summaries, required_traits,
-                    forbidden_traits):
+                    res_requests, summaries,
+                    rg_ctx.required_trait_map,
+                    rg_ctx.forbidden_trait_map):
                 # This combination doesn't satisfy trait constraints
                 continue
             root_alloc_reqs.add(
@@ -435,7 +378,7 @@ def _alloc_candidates_multiple_providers(
     return list(alloc_requests), list(summaries.values())
 
 
-def _alloc_candidates_single_provider(ctx, requested_resources, rp_tuples):
+def _alloc_candidates_single_provider(rg_ctx, rp_tuples):
     """Returns a tuple of (allocation requests, provider summaries) for a
     supplied set of requested resource amounts and resource providers. The
     supplied resource providers have capacity to satisfy ALL of the resources
@@ -450,9 +393,7 @@ def _alloc_candidates_single_provider(ctx, requested_resources, rp_tuples):
     AllocationRequest and ProviderSummary objects due to not having to
     determine requests across multiple providers.
 
-    :param ctx: placement.context.RequestContext object
-    :param requested_resources: dict, keyed by resource class ID, of amounts
-                                being requested for that resource class
+    :param rg_ctx: RequestGroupSearchContext
     :param rp_tuples: List of two-tuples of (provider ID, root provider ID)s
                       for providers that matched the requested resources
     """
@@ -463,15 +404,16 @@ def _alloc_candidates_single_provider(ctx, requested_resources, rp_tuples):
     root_ids = set(p[1] for p in rp_tuples)
 
     # Grab usage summaries for each provider
-    usages = _get_usages_by_provider_tree(ctx, root_ids)
+    usages = _get_usages_by_provider_tree(rg_ctx.context, root_ids)
 
     # Get a dict, keyed by resource provider internal ID, of trait string names
     # that provider has associated with it
-    prov_traits = trait_obj.get_traits_by_provider_tree(ctx, root_ids)
+    prov_traits = trait_obj.get_traits_by_provider_tree(
+        rg_ctx.context, root_ids)
 
     # Get a dict, keyed by resource provider internal ID, of ProviderSummary
     # objects for all providers
-    summaries = _build_provider_summaries(ctx, usages, prov_traits)
+    summaries = _build_provider_summaries(rg_ctx.context, usages, prov_traits)
 
     # Next, build up a list of allocation requests. These allocation requests
     # are AllocationRequest objects, containing resource provider UUIDs,
@@ -480,14 +422,14 @@ def _alloc_candidates_single_provider(ctx, requested_resources, rp_tuples):
     for rp_id, root_id in rp_tuples:
         rp_summary = summaries[rp_id]
         req_obj = _allocation_request_for_provider(
-            ctx, requested_resources, rp_summary.resource_provider)
+            rg_ctx.context, rg_ctx.resources, rp_summary.resource_provider)
         alloc_requests.append(req_obj)
         # If this is a sharing provider, we have to include an extra
         # AllocationRequest for every possible anchor.
         traits = rp_summary.traits
         if os_traits.MISC_SHARES_VIA_AGGREGATE in traits:
             anchors = set([p[1] for p in rp_obj.anchors_for_sharing_providers(
-                ctx, [rp_summary.resource_provider.id])])
+                rg_ctx.context, [rp_summary.resource_provider.id])])
             for anchor in anchors:
                 # We already added self
                 if anchor == rp_summary.resource_provider.root_provider_uuid:

placement/objects/research_context.py

@@ -0,0 +1,115 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+"""Utility methods for getting allocation candidates."""
+
+from oslo_log import log as logging
+
+from placement import exception
+from placement.objects import resource_provider as rp_obj
+from placement.objects import trait as trait_obj
+from placement import resource_class_cache as rc_cache
+
+
+LOG = logging.getLogger(__name__)
+
+
+class RequestGroupSearchContext(object):
+    """An adapter object that represents the search for allocation candidates
+    for a single request group.
+    """
+    def __init__(self, context, request, has_trees):
+        """Initializes the object retrieving and caching matching providers
+        for each conditions like resource and aggregates from database.
+
+        :raises placement.exception.ResourceProviderNotFound if there is no
+                provider found which satisfies the request.
+        """
+        # TODO(tetsuro): split this into smaller functions reordering
+        self.context = context
+
+        # A dict, keyed by resource class internal ID, of the amounts of that
+        # resource class being requested by the group.
+        self.resources = {
+            rc_cache.RC_CACHE.id_from_string(key): value
+            for key, value in request.resources.items()
+        }
+
+        # A list of lists of aggregate UUIDs that the providers matching for
+        # that request group must be members of
+        self.member_of = request.member_of
+
+        # A list of aggregate UUIDs that the providers matching for
+        # that request group must not be members of
+        self.forbidden_aggs = request.forbidden_aggs
+
+        # If True, this RequestGroup represents requests which must be
+        # satisfied by a single resource provider.  If False, represents a
+        # request for resources in any resource provider in the same tree,
+        # or a sharing provider.
+        self.use_same_provider = request.use_same_provider
+
+        # maps the trait name to the trait internal ID
+        self.required_trait_map = {}
+        self.forbidden_trait_map = {}
+        for trait_map, traits in (
+                (self.required_trait_map, request.required_traits),
+                (self.forbidden_trait_map, request.forbidden_traits)):
+            if traits:
+                trait_map.update(trait_obj.ids_from_names(context, traits))
+
+        # Internal id of a root provider. If provided, this RequestGroup must
+        # be satisfied by resource provider(s) under the root provider.
+        self.tree_root_id = None
+        if request.in_tree:
+            tree_ids = rp_obj.provider_ids_from_uuid(context, request.in_tree)
+            if tree_ids is None:
+                raise exception.ResourceProviderNotFound()
+            self.tree_root_id = tree_ids.root_id
+            LOG.debug("getting allocation candidates in the same tree "
+                      "with the root provider %s", tree_ids.root_uuid)
+
+        # a dict, keyed by resource class ID, of the set of resource
+        # provider IDs that share some inventory for each resource class
+        # This is only used for unnumbered request group path where
+        # use_same_provider is set to False
+        self._sharing_providers = {}
+        if not self.use_same_provider:
+            for rc_id, amount in self.resources.items():
+                # We may want to have a concept of "sharable resource class"
+                # so that we can skip this lookup.
+                # if not rc_id in (sharable_rc_ids):
+                #     continue
+                self._sharing_providers[rc_id] = \
+                    rp_obj.get_providers_with_shared_capacity(
+                        context, rc_id, amount, self.member_of)
+
+        # bool indicating there is some level of nesting in the environment
+        self.has_trees = has_trees
+
+    @property
+    def exists_sharing(self):
+        """bool indicating there is sharing providers in the environment for
+        the requested resource class (if there isn't, we take faster, simpler
+        code paths)
+        """
+        return any(self._sharing_providers.values())
+
+    @property
+    def exists_nested(self):
+        """bool indicating there is some level of nesting in the environment
+        (if there isn't, we take faster, simpler code paths)
+        """
+        # NOTE: This could be refactored to see the requested resources
+        return self.has_trees
+
+    def get_rps_with_shared_capacity(self, rc_id):
+        return self._sharing_providers.get(rc_id)

placement/objects/resource_provider.py

@@ -646,6 +646,7 @@ def provider_ids_from_rp_ids(context, rp_ids):
     return ret
 
 
+@db_api.placement_context_manager.reader
 def provider_ids_from_uuid(context, uuid):
     """Given the UUID of a resource provider, returns a namedtuple
     (ProviderIds) with the internal ID, the UUID, the parent provider's
@@ -688,6 +689,7 @@ def provider_ids_from_uuid(context, uuid):
     return ProviderIds(**dict(res))
 
 
+@db_api.placement_context_manager.reader
 def provider_ids_matching_aggregates(context, member_of, rp_ids=None):
     """Given a list of lists of aggregate UUIDs, return the internal IDs of all
     resource providers associated with the aggregates.
@@ -1499,9 +1501,7 @@ def _normalize_trait_map(ctx, traits):
 
 
 @db_api.placement_context_manager.reader
-def get_provider_ids_matching(ctx, resources, required_traits,
-                              forbidden_traits, member_of, forbidden_aggs,
-                              tree_root_id):
+def get_provider_ids_matching(rg_ctx):
     """Returns a list of tuples of (internal provider ID, root provider ID)
     that have available inventory to satisfy all the supplied requests for
     resources. If no providers match, the empty list is returned.
@@ -1516,31 +1516,12 @@ def get_provider_ids_matching(ctx, resources, required_traits,
            satisfied by other providers in the same tree or shared via
            aggregate.
 
-    :param ctx: Session context to use
-    :param resources: A dict, keyed by resource class ID, of the amount
-                      requested of that resource class.
-    :param required_traits: A map, keyed by trait string name, of required
-                            trait internal IDs that each provider must have
-                            associated with it
-    :param forbidden_traits: A map, keyed by trait string name, of forbidden
-                             trait internal IDs that each provider must not
-                             have associated with it
-    :param member_of: An optional list of list of aggregate UUIDs. If provided,
-                      the allocation_candidates returned will only be for
-                      resource providers that are members of one or more of the
-                      supplied aggregates of each aggregate UUID list.
-    :param forbidden_aggs: An optional list of aggregate UUIDs. If provided,
-                           the allocation_candidates returned will only be for
-                           resource providers that are NOT members of supplied
-                           aggregates.
-    :param tree_root_id: An optional root resource provider ID. If provided,
-                         the result will be restricted to providers in the tree
-                         with this root ID.
+    :param rg_ctx: RequestGroupSearchContext
     """
-    # The iteratively filtered set of resource provider internal IDs that match
-    # all the constraints in the request
+    # TODO(tetsuro): refactor this to have only the rg_ctx argument
     filtered_rps, forbidden_rp_ids = get_provider_ids_for_traits_and_aggs(
-        ctx, required_traits, forbidden_traits, member_of, forbidden_aggs)
+        rg_ctx.context, rg_ctx.required_trait_map, rg_ctx.forbidden_trait_map,
+        rg_ctx.member_of, rg_ctx.forbidden_aggs)
     if filtered_rps is None:
         # If no providers match the traits/aggs, we can short out
         return []
@@ -1560,10 +1541,10 @@ def get_provider_ids_matching(ctx, resources, required_traits,
     # eventual results list
     provs_with_resource = set()
     first = True
-    for rc_id, amount in resources.items():
+    for rc_id, amount in rg_ctx.resources.items():
         rc_name = rc_cache.RC_CACHE.string_from_id(rc_id)
         provs_with_resource = get_providers_with_resource(
-            ctx, rc_id, amount, tree_root_id=tree_root_id)
+            rg_ctx.context, rc_id, amount, tree_root_id=rg_ctx.tree_root_id)
         LOG.debug("found %d providers with available %d %s",
                   len(provs_with_resource), amount, rc_name)
         if not provs_with_resource:
@@ -1751,8 +1732,7 @@ def _get_trees_with_traits(ctx, rp_ids, required_traits, forbidden_traits):
 
 
 @db_api.placement_context_manager.reader
-def get_trees_matching_all(ctx, resources, required_traits, forbidden_traits,
-                           sharing, member_of, forbidden_aggs, tree_root_id):
+def get_trees_matching_all(rg_ctx):
     """Returns a RPCandidates object representing the providers that satisfy
     the request for resources.
 
@@ -1778,42 +1758,22 @@ def get_trees_matching_all(ctx, resources, required_traits, forbidden_traits,
     to use multiple providers within the same provider tree including sharing
     providers to satisfy different resources involved in a single RequestGroup.
 
-    :param ctx: Session context to use
-    :param resources: A dict, keyed by resource class ID, of the amount
-                      requested of that resource class.
-    :param required_traits: A map, keyed by trait string name, of required
-                            trait internal IDs that each provider TREE must
-                            COLLECTIVELY have associated with it
-    :param forbidden_traits: A map, keyed by trait string name, of trait
-                             internal IDs that a resource provider must
-                             not have.
-    :param sharing: dict, keyed by resource class ID, of lists of resource
-                    provider IDs that share that resource class and can
-                    contribute to the overall allocation request
-    :param member_of: An optional list of lists of aggregate UUIDs. If
-                      provided, the allocation_candidates returned will only be
-                      for resource providers that are members of one or more of
-                      the supplied aggregates in each aggregate UUID list.
-    :param forbidden_aggs: An optional list of aggregate UUIDs. If provided,
-                           the allocation_candidates returned will only be for
-                           resource providers that are NOT members of supplied
-                           aggregates.
-    :param tree_root_id: An optional root provider ID. If provided, the results
-                         are limited to the resource providers under the given
-                         root resource provider.
+    :param rg_ctx: RequestGroupSearchContext
     """
     # If 'member_of' has values, do a separate lookup to identify the
     # resource providers that meet the member_of constraints.
-    if member_of:
-        rps_in_aggs = provider_ids_matching_aggregates(ctx, member_of)
+    if rg_ctx.member_of:
+        rps_in_aggs = provider_ids_matching_aggregates(
+            rg_ctx.context, rg_ctx.member_of)
         if not rps_in_aggs:
             # Short-circuit. The user either asked for a non-existing
             # aggregate or there were no resource providers that matched
             # the requirements...
             return rp_candidates.RPCandidateList()
 
-    if forbidden_aggs:
-        rps_bad_aggs = provider_ids_matching_aggregates(ctx, [forbidden_aggs])
+    if rg_ctx.forbidden_aggs:
+        rps_bad_aggs = provider_ids_matching_aggregates(
+            rg_ctx.context, [rg_ctx.forbidden_aggs])
 
     # To get all trees that collectively have all required resource,
     # aggregates and traits, we use `RPCandidateList` which has a list of
@@ -1822,12 +1782,12 @@ def get_trees_matching_all(ctx, resources, required_traits, forbidden_traits,
     # class ID.
     provs_with_inv = rp_candidates.RPCandidateList()
 
-    for rc_id, amount in resources.items():
+    for rc_id, amount in rg_ctx.resources.items():
         rc_name = rc_cache.RC_CACHE.string_from_id(rc_id)
 
         provs_with_inv_rc = rp_candidates.RPCandidateList()
         rc_provs_with_inv = get_providers_with_resource(
-            ctx, rc_id, amount, tree_root_id=tree_root_id)
+            rg_ctx.context, rc_id, amount, tree_root_id=rg_ctx.tree_root_id)
         provs_with_inv_rc.add_rps(rc_provs_with_inv, rc_id)
         LOG.debug("found %d providers under %d trees with available %d %s",
                   len(provs_with_inv_rc), len(provs_with_inv_rc.trees),
@@ -1837,8 +1797,8 @@ def get_trees_matching_all(ctx, resources, required_traits, forbidden_traits,
             # then we can short-circuit returning an empty RPCandidateList
             return rp_candidates.RPCandidateList()
 
-        sharing_providers = sharing.get(rc_id)
-        if sharing_providers and tree_root_id is None:
+        sharing_providers = rg_ctx.get_rps_with_shared_capacity(rc_id)
+        if sharing_providers and rg_ctx.tree_root_id is None:
             # There are sharing providers for this resource class, so we
             # should also get combinations of (sharing provider, anchor root)
             # in addition to (non-sharing provider, anchor root) we've just
@@ -1846,7 +1806,7 @@ def get_trees_matching_all(ctx, resources, required_traits, forbidden_traits,
             # process if tree_root_id is provided via the ?in_tree=<rp_uuid>
             # queryparam, because it restricts resources from another tree.
             rc_provs_with_inv = anchors_for_sharing_providers(
-                ctx, sharing_providers, get_id=True)
+                rg_ctx.context, sharing_providers, get_id=True)
             provs_with_inv_rc.add_rps(rc_provs_with_inv, rc_id)
             LOG.debug(
                 "considering %d sharing providers with %d %s, "
@@ -1854,25 +1814,25 @@ def get_trees_matching_all(ctx, resources, required_traits, forbidden_traits,
                 len(sharing_providers), amount, rc_name,
                 len(provs_with_inv_rc.trees))
 
-        if member_of:
+        if rg_ctx.member_of:
             # Aggregate on root spans the whole tree, so the rp itself
             # *or its root* should be in the aggregate
             provs_with_inv_rc.filter_by_rp_or_tree(rps_in_aggs)
             LOG.debug("found %d providers under %d trees after applying "
                       "aggregate filter %s",
                       len(provs_with_inv_rc.rps), len(provs_with_inv_rc.trees),
-                      member_of)
+                      rg_ctx.member_of)
             if not provs_with_inv_rc:
                 # Short-circuit returning an empty RPCandidateList
                 return rp_candidates.RPCandidateList()
-        if forbidden_aggs:
+        if rg_ctx.forbidden_aggs:
             # Aggregate on root spans the whole tree, so the rp itself
             # *and its root* should be outside the aggregate
             provs_with_inv_rc.filter_by_rp_nor_tree(rps_bad_aggs)
             LOG.debug("found %d providers under %d trees after applying "
                       "negative aggregate filter %s",
                       len(provs_with_inv_rc.rps), len(provs_with_inv_rc.trees),
-                      forbidden_aggs)
+                      rg_ctx.forbidden_aggs)
             if not provs_with_inv_rc:
                 # Short-circuit returning an empty RPCandidateList
                 return rp_candidates.RPCandidateList()
@@ -1889,8 +1849,8 @@ def get_trees_matching_all(ctx, resources, required_traits, forbidden_traits,
         if not provs_with_inv:
             return rp_candidates.RPCandidateList()
 
-    if (not required_traits and not forbidden_traits) or (
-            any(sharing.values())):
+    if (not rg_ctx.required_trait_map and not rg_ctx.forbidden_trait_map) or (
+            rg_ctx.exists_sharing):
         # If there were no traits required, there's no difference in how we
         # calculate allocation requests between nested and non-nested
         # environments, so just short-circuit and return. Or if sharing
@@ -1902,11 +1862,13 @@ def get_trees_matching_all(ctx, resources, required_traits, forbidden_traits,
     # capacity and that set of providers (grouped by their tree) have all
     # of the required traits and none of the forbidden traits
     rp_tuples_with_trait = _get_trees_with_traits(
-        ctx, provs_with_inv.rps, required_traits, forbidden_traits)
+        rg_ctx.context, provs_with_inv.rps, rg_ctx.required_trait_map,
+        rg_ctx.forbidden_trait_map)
     provs_with_inv.filter_by_rp(rp_tuples_with_trait)
     LOG.debug("found %d providers under %d trees after applying "
               "traits filter - required: %s, forbidden: %s",
               len(provs_with_inv.rps), len(provs_with_inv.trees),
-              list(required_traits), list(forbidden_traits))
+              list(rg_ctx.required_trait_map),
+              list(rg_ctx.forbidden_trait_map))
 
     return provs_with_inv

placement/tests/functional/db/test_allocation_candidates.py

@@ -19,12 +19,35 @@ import sqlalchemy as sa
 from placement import exception
 from placement import lib as placement_lib
 from placement.objects import allocation_candidate as ac_obj
+from placement.objects import research_context as res_ctx
 from placement.objects import resource_class as rc_obj
 from placement.objects import resource_provider as rp_obj
 from placement.objects import trait as trait_obj
 from placement.tests.functional.db import test_base as tb
 
 
+def _req_group_search_context(context, **kwargs):
+    resources = {
+        orc.VCPU: 2,
+        orc.MEMORY_MB: 256,
+        orc.SRIOV_NET_VF: 1,
+    }
+    request = placement_lib.RequestGroup(
+        use_same_provider=False,
+        resources=kwargs.get('resources', resources),
+        required_traits=kwargs.get('required_traits', {}),
+        forbidden_traits=kwargs.get('forbidden_traits', {}),
+        member_of=kwargs.get('member_of', []),
+        forbidden_aggs=kwargs.get('forbidden_aggs', []),
+        in_tree=kwargs.get('in_tree', None),
+    )
+    has_trees = rp_obj.has_provider_trees(context)
+    rg_ctx = res_ctx.RequestGroupSearchContext(
+        context, request, has_trees)
+
+    return rg_ctx
+
+
 class ProviderDBHelperTestCase(tb.PlacementDbBaseTestCase):
 
     def test_get_provider_ids_matching(self):
@@ -142,21 +165,14 @@ class ProviderDBHelperTestCase(tb.PlacementDbBaseTestCase):
         self.allocate_from_provider(excl_extra_avail, 'CUSTOM_SPECIAL', 99)
 
         resources = {
-            orc.STANDARDS.index(orc.VCPU): 5,
-            orc.STANDARDS.index(orc.MEMORY_MB): 1024,
-            orc.STANDARDS.index(orc.DISK_GB): 1500
+            orc.VCPU: 5,
+            orc.MEMORY_MB: 1024,
+            orc.DISK_GB: 1500
         }
 
-        empty_req_traits = {}
-        empty_forbidden_traits = {}
-        empty_agg = []
-        empty_forbidden_aggs = []
-        empty_root_id = None
-
         # Run it!
-        res = rp_obj.get_provider_ids_matching(
-            self.ctx, resources, empty_req_traits, empty_forbidden_traits,
-            empty_agg, empty_forbidden_aggs, empty_root_id)
+        rg_ctx = _req_group_search_context(self.ctx, resources=resources)
+        res = rp_obj.get_provider_ids_matching(rg_ctx)
 
         # We should get all the incl_* RPs
         expected = [incl_biginv_noalloc, incl_extra_full]
@@ -171,44 +187,47 @@ class ProviderDBHelperTestCase(tb.PlacementDbBaseTestCase):
         # get_provider_ids_matching()'s required_traits and forbidden_traits
         # arguments maps, keyed by trait name, of the trait internal ID
         req_traits = {os_traits.HW_CPU_X86_AVX2: avx2_t.id}
-        res = rp_obj.get_provider_ids_matching(
-            self.ctx, resources, req_traits, empty_forbidden_traits, empty_agg,
-            empty_forbidden_aggs, empty_root_id)
+        rg_ctx = _req_group_search_context(
+            self.ctx,
+            resources=resources,
+            required_traits=req_traits,
+        )
+        res = rp_obj.get_provider_ids_matching(rg_ctx)
 
         self.assertEqual([], res)
 
         # Next let's set the required trait to an excl_* RPs.
         # This should result in no results returned as well.
         excl_big_md_noalloc.set_traits([avx2_t])
-        res = rp_obj.get_provider_ids_matching(
-            self.ctx, resources, req_traits, empty_forbidden_traits, empty_agg,
-            empty_forbidden_aggs, empty_root_id)
+        res = rp_obj.get_provider_ids_matching(rg_ctx)
         self.assertEqual([], res)
 
         # OK, now add the trait to one of the incl_* providers and verify that
         # provider now shows up in our results
         incl_biginv_noalloc.set_traits([avx2_t])
-        res = rp_obj.get_provider_ids_matching(
-            self.ctx, resources, req_traits, empty_forbidden_traits, empty_agg,
-            empty_forbidden_aggs, empty_root_id)
+        res = rp_obj.get_provider_ids_matching(rg_ctx)
 
         rp_ids = [r[0] for r in res]
         self.assertEqual([incl_biginv_noalloc.id], rp_ids)
 
-        # Let's see if the tree_root_id filter works
-        root_id = incl_biginv_noalloc.id
-        res = rp_obj.get_provider_ids_matching(
-            self.ctx, resources, empty_req_traits, empty_forbidden_traits,
-            empty_agg, empty_forbidden_aggs, root_id)
+        # Let's see if the in_tree filter works
+        rg_ctx = _req_group_search_context(
+            self.ctx,
+            resources=resources,
+            in_tree=uuids.biginv_noalloc,
+        )
+        res = rp_obj.get_provider_ids_matching(rg_ctx)
         rp_ids = [r[0] for r in res]
         self.assertEqual([incl_biginv_noalloc.id], rp_ids)
 
         # We don't get anything if the specified tree doesn't satisfy the
         # requirements in the first place
-        root_id = excl_allused.id
-        res = rp_obj.get_provider_ids_matching(
-            self.ctx, resources, empty_req_traits, empty_forbidden_traits,
-            empty_agg, empty_forbidden_aggs, root_id)
+        rg_ctx = _req_group_search_context(
+            self.ctx,
+            resources=resources,
+            in_tree=uuids.allused,
+        )
+        res = rp_obj.get_provider_ids_matching(rg_ctx)
         self.assertEqual([], res)
 
     def test_get_provider_ids_matching_with_multiple_forbidden(self):
@@ -223,17 +242,17 @@ class ProviderDBHelperTestCase(tb.PlacementDbBaseTestCase):
         trait_three, = tb.set_traits(rp3, 'CUSTOM_THREE')
         tb.add_inventory(rp3, orc.VCPU, 64)
 
-        resources = {orc.STANDARDS.index(orc.VCPU): 4}
-        empty_req_traits = {}
+        resources = {orc.VCPU: 4}
         forbidden_traits = {trait_two.name: trait_two.id,
                             trait_three.name: trait_three.id}
         member_of = [[uuids.agg1]]
-        empty_forbidden_aggs = []
-        empty_root_id = None
 
-        res = rp_obj.get_provider_ids_matching(
-            self.ctx, resources, empty_req_traits, forbidden_traits, member_of,
-            empty_forbidden_aggs, empty_root_id)
+        rg_ctx = _req_group_search_context(
+            self.ctx,
+            resources=resources,
+            forbidden_traits=forbidden_traits,
+            member_of=member_of)
+        res = rp_obj.get_provider_ids_matching(rg_ctx)
         self.assertEqual({(rp1.id, rp1.id)}, set(res))
 
     def test_get_provider_ids_matching_with_aggregates(self):
@@ -249,70 +268,77 @@ class ProviderDBHelperTestCase(tb.PlacementDbBaseTestCase):
         tb.add_inventory(rp4, orc.VCPU, 64)
         tb.add_inventory(rp5, orc.VCPU, 64)
 
-        resources = {orc.STANDARDS.index(orc.VCPU): 4}
-        empty_req_traits = {}
-        empty_forbidden_traits = {}
-        empty_forbidden_aggs = []
-        empty_root_id = None
-
-        member_of = [[uuids.agg1]]
+        resources = {orc.VCPU: 4}
+        rg_ctx = _req_group_search_context(
+            self.ctx,
+            resources=resources,
+            member_of=[[uuids.agg1]],
+        )
         expected_rp = [rp1, rp4]
 
-        res = rp_obj.get_provider_ids_matching(
-            self.ctx, resources, empty_req_traits, empty_forbidden_traits,
-            member_of, empty_forbidden_aggs, empty_root_id)
+        res = rp_obj.get_provider_ids_matching(rg_ctx)
         self.assertEqual(set((rp.id, rp.id) for rp in expected_rp), set(res))
 
-        member_of = [[uuids.agg1, uuids.agg2]]
+        rg_ctx = _req_group_search_context(
+            self.ctx,
+            resources=resources,
+            member_of=[[uuids.agg1, uuids.agg2]],
+        )
         expected_rp = [rp1, rp2, rp4]
 
-        res = rp_obj.get_provider_ids_matching(
-            self.ctx, resources, empty_req_traits, empty_forbidden_traits,
-            member_of, empty_forbidden_aggs, empty_root_id)
+        res = rp_obj.get_provider_ids_matching(rg_ctx)
         self.assertEqual(set((rp.id, rp.id) for rp in expected_rp), set(res))
 
-        member_of = [[uuids.agg1, uuids.agg2],
-                     [uuids.agg4]]
+        rg_ctx = _req_group_search_context(
+            self.ctx,
+            resources=resources,
+            member_of=[[uuids.agg1, uuids.agg2], [uuids.agg4]],
+        )
         expected_rp = [rp4]
 
-        res = rp_obj.get_provider_ids_matching(
-            self.ctx, resources, empty_req_traits, empty_forbidden_traits,
-            member_of, empty_forbidden_aggs, empty_root_id)
+        res = rp_obj.get_provider_ids_matching(rg_ctx)
         self.assertEqual(set((rp.id, rp.id) for rp in expected_rp), set(res))
 
-        empty_member_of = []
-        forbidden_aggs = [uuids.agg1]
+        rg_ctx = _req_group_search_context(
+            self.ctx,
+            resources=resources,
+            forbidden_aggs=[uuids.agg1],
+        )
         expected_rp = [rp2, rp3, rp5]
 
-        res = rp_obj.get_provider_ids_matching(
-            self.ctx, resources, empty_req_traits, empty_forbidden_traits,
-            empty_member_of, forbidden_aggs, empty_root_id)
+        res = rp_obj.get_provider_ids_matching(rg_ctx)
         self.assertEqual(set((rp.id, rp.id) for rp in expected_rp), set(res))
 
-        forbidden_aggs = [uuids.agg1, uuids.agg2]
+        rg_ctx = _req_group_search_context(
+            self.ctx,
+            resources=resources,
+            forbidden_aggs=[uuids.agg1, uuids.agg2],
+        )
         expected_rp = [rp3, rp5]
 
-        res = rp_obj.get_provider_ids_matching(
-            self.ctx, resources, empty_req_traits, empty_forbidden_traits,
-            empty_member_of, forbidden_aggs, empty_root_id)
+        res = rp_obj.get_provider_ids_matching(rg_ctx)
         self.assertEqual(set((rp.id, rp.id) for rp in expected_rp), set(res))
 
-        member_of = [[uuids.agg1, uuids.agg2]]
-        forbidden_aggs = [uuids.agg3, uuids.agg4]
+        rg_ctx = _req_group_search_context(
+            self.ctx,
+            resources=resources,
+            member_of=[[uuids.agg1, uuids.agg2]],
+            forbidden_aggs=[uuids.agg3, uuids.agg4],
+        )
         expected_rp = [rp1]
 
-        res = rp_obj.get_provider_ids_matching(
-            self.ctx, resources, empty_req_traits, empty_forbidden_traits,
-            member_of, forbidden_aggs, empty_root_id)
+        res = rp_obj.get_provider_ids_matching(rg_ctx)
         self.assertEqual(set((rp.id, rp.id) for rp in expected_rp), set(res))
 
-        member_of = [[uuids.agg1]]
-        forbidden_aggs = [uuids.agg1]
+        rg_ctx = _req_group_search_context(
+            self.ctx,
+            resources=resources,
+            member_of=[[uuids.agg1]],
+            forbidden_aggs=[uuids.agg1],
+        )
         expected_rp = []
 
-        res = rp_obj.get_provider_ids_matching(
-            self.ctx, resources, empty_req_traits, empty_forbidden_traits,
-            member_of, forbidden_aggs, empty_root_id)
+        res = rp_obj.get_provider_ids_matching(rg_ctx)
         self.assertEqual(set((rp.id, rp.id) for rp in expected_rp), set(res))
 
     def test_get_provider_ids_having_all_traits(self):
@@ -391,21 +417,8 @@ class ProviderTreeDBHelperTestCase(tb.PlacementDbBaseTestCase):
             """Helper function to validate the test result"""
             # NOTE(jaypipes): get_trees_matching_all() expects a dict of
             # resource class internal identifiers, not string names
-            resources = {
-                orc.STANDARDS.index(orc.VCPU): 2,
-                orc.STANDARDS.index(orc.MEMORY_MB): 256,
-                orc.STANDARDS.index(orc.SRIOV_NET_VF): 1,
-            }
-            req_traits = kwargs.get('required_traits', {})
-            forbid_traits = kwargs.get('forbidden_traits', {})
-            sharing = kwargs.get('sharing_providers', {})
-            member_of = kwargs.get('member_of', [])
-            forbidden_aggs = kwargs.get('forbidden_aggs', [])
-            tree_root_id = kwargs.get('tree_root_id', None)
-
-            results = rp_obj.get_trees_matching_all(
-                self.ctx, resources, req_traits, forbid_traits, sharing,
-                member_of, forbidden_aggs, tree_root_id)
+            rg_ctx = _req_group_search_context(self.ctx, **kwargs)
+            results = rp_obj.get_trees_matching_all(rg_ctx)
 
             tree_ids = self._get_rp_ids_matching_names(expected_trees)
             rp_ids = self._get_rp_ids_matching_names(expected_rps)
@@ -474,8 +487,7 @@ class ProviderTreeDBHelperTestCase(tb.PlacementDbBaseTestCase):
         # Let's see if the tree_root_id filter works
         expected_trees = ['cn1']
         expected_rps = ['cn1', 'cn1_numa0_pf0', 'cn1_numa1_pf1']
-        tree_root_id = self.get_provider_id_by_name('cn1')
-        _run_test(expected_trees, expected_rps, tree_root_id=tree_root_id)
+        _run_test(expected_trees, expected_rps, in_tree=uuids.cn1)
 
         # Let's see if the aggregate filter works