Lines Matching defs:affinity
2 * kmp_affinity.cpp -- affinity management
67 __kmp_get_affinity_env_var(const kmp_affinity_t &affinity,
69 if (affinity.flags.omp_places) {
74 return affinity.env_var;
80 // The test below is true if affinity is available, but set to "none". Need to
189 // If affinity is supported, check the affinity
851 void kmp_topology_t::set_granularity(kmp_affinity_t &affinity) const {
852 const char *env_var = __kmp_get_affinity_env_var(affinity);
857 if (affinity.core_attr_gran.valid) {
861 affinity, AffIgnoringNonHybrid, env_var,
863 affinity.gran = KMP_HW_CORE;
864 affinity.gran_levels = -1;
865 affinity.core_attr_gran = KMP_AFFINITY_ATTRS_UNKNOWN;
866 affinity.flags.core_types_gran = affinity.flags.core_effs_gran = 0;
867 } else if (affinity.flags.core_types_gran ||
868 affinity.flags.core_effs_gran) {
870 if (affinity.flags.omp_places) {
872 affinity, AffIgnoringNonHybrid, env_var,
876 KMP_AFF_WARNING(affinity, AffGranularityBad, env_var,
880 affinity.gran = KMP_HW_CORE;
881 affinity.gran_levels = -1;
882 affinity.core_attr_gran = KMP_AFFINITY_ATTRS_UNKNOWN;
883 affinity.flags.core_types_gran = affinity.flags.core_effs_gran = 0;
886 // Set the number of affinity granularity levels
887 if (affinity.gran_levels < 0) {
888 kmp_hw_t gran_type = get_equivalent_type(affinity.gran);
901 KMP_AFF_WARNING(affinity, AffGranularityBad, env_var,
902 __kmp_hw_get_catalog_string(affinity.gran),
904 affinity.gran = gran_type;
909 // granularity must be adjusted "down" to processor group affinity
919 KMP_AFF_WARNING(affinity, AffGranTooCoarseProcGroup, env_var,
920 __kmp_hw_get_catalog_string(affinity.gran));
921 affinity.gran = gran_type = KMP_HW_PROC_GROUP;
925 affinity.gran_levels = 0;
927 affinity.gran_levels++;
1417 // Only use Hwloc if affinity isn't explicitly disabled and
1444 // Print the affinity mask to the character array in a pretty format.
1485 // Range with three or more contiguous bits in the affinity mask
1489 // Range with one or two contiguous bits in the affinity mask
1512 // Print the affinity mask to the string buffer object in a pretty format
1549 // Range with three or more contiguous bits in the affinity mask
1553 // Range with one or two contiguous bits in the affinity mask
1567 // Return (possibly empty) affinity mask representing the offline CPUs
2002 // When affinity is off, this routine will still be called to set
2004 // Make sure all these vars are set correctly, and return now if affinity is
2047 // If we aren't affinity capable, then use flat topology
2183 // the current thread's affinity mask to that thread, and then retrieves
2200 // The algorithm used starts by setting the affinity to each available thread
2241 // if the machine is not affinity capable, we assume that HT is off. We have
2262 // Save the affinity mask for the current thread.
2340 // affinity mask before the tail call.
2356 // Restore the old affinity mask for this thread.
2439 // When affinity is off, this routine will still be called to set
2705 // The algorithm used starts by setting the affinity to each available thread
2754 // Save the affinity mask for the current thread.
2933 // affinity map. On AIX, the map is obtained through system SRAD (Scheduler
2983 // of records can't exceed the number of valid bits in the affinity mask.
3503 // When affinity is off, this routine will still be called to set
3505 // Make sure all these vars are set correctly, and return now if affinity is
3606 // Create and return a table of affinity masks, indexed by OS thread ID.
3607 // This routine handles OR'ing together all the affinity masks of threads
3611 kmp_affinity_t &affinity,
3613 // First form a table of affinity masks in order of OS thread id.
3618 const char *env_var = __kmp_get_affinity_env_var(affinity);
3637 affinity.num_os_id_masks = maxOsId + 1;
3638 KMP_CPU_ALLOC_ARRAY(affinity.os_id_masks, affinity.num_os_id_masks);
3639 KMP_ASSERT(affinity.gran_levels >= 0);
3640 if (affinity.flags.verbose && (affinity.gran_levels > 0)) {
3641 KMP_INFORM(ThreadsMigrate, env_var, affinity.gran_levels);
3643 if (affinity.gran_levels >= (int)depth) {
3644 KMP_AFF_WARNING(affinity, AffThreadsMayMigrate);
3664 // affinity mask for this group, and go on to the next thread.
3665 if (__kmp_topology->is_close(leader, i, affinity)) {
3675 kmp_affin_mask_t *mask = KMP_CPU_INDEX(affinity.os_id_masks, osId);
3693 kmp_affin_mask_t *mask = KMP_CPU_INDEX(affinity.os_id_masks, osId);
3702 if (full_mask.restrict_to_mask() && affinity.flags.verbose) {
3709 // Stuff for the affinity proclist parsers. It's easier to declare these vars
3739 KMP_AFF_WARNING(affinity, AffIgnoreInvalidProcID, _osId); \
3745 // Re-parse the proclist (for the explicit affinity type), and form the list
3746 // of affinity newMasks indexed by gtid.
3747 static void __kmp_affinity_process_proclist(kmp_affinity_t &affinity) {
3749 kmp_affin_mask_t **out_masks = &affinity.masks;
3750 unsigned *out_numMasks = &affinity.num_masks;
3751 const char *proclist = affinity.proclist;
3752 kmp_affin_mask_t *osId2Mask = affinity.os_id_masks;
3753 int maxOsId = affinity.num_os_id_masks - 1;
3791 KMP_AFF_WARNING(affinity, AffIgnoreInvalidProcID, num);
3823 KMP_AFF_WARNING(affinity, AffIgnoreInvalidProcID, num);
3958 kmp_affinity_t &affinity, int maxOsId,
3962 kmp_affin_mask_t *osId2Mask = affinity.os_id_masks;
3981 KMP_AFF_WARNING(affinity, AffIgnoreInvalidProcID, start);
4010 KMP_AFF_WARNING(affinity, AffIgnoreInvalidProcID, start);
4057 KMP_AFF_WARNING(affinity, AffIgnoreInvalidProcID, start);
4076 static void __kmp_process_place(const char **scan, kmp_affinity_t &affinity,
4080 kmp_affin_mask_t *osId2Mask = affinity.os_id_masks;
4086 __kmp_process_subplace_list(scan, affinity, maxOsId, tempMask, setSize);
4091 __kmp_process_place(scan, affinity, maxOsId, tempMask, setSize);
4100 KMP_AFF_WARNING(affinity, AffIgnoreInvalidProcID, num);
4112 void __kmp_affinity_process_placelist(kmp_affinity_t &affinity) {
4114 kmp_affin_mask_t **out_masks = &affinity.masks;
4115 unsigned *out_numMasks = &affinity.num_masks;
4116 const char *placelist = affinity.proclist;
4117 kmp_affin_mask_t *osId2Mask = affinity.os_id_masks;
4118 int maxOsId = affinity.num_os_id_masks - 1;
4138 __kmp_process_place(&scan, affinity, maxOsId, tempMask, &setSize);
4219 KMP_AFF_WARNING(affinity, AffIgnoreInvalidProcID, j + stride);
4377 // A thread can then grab not only its affinity mask, but the topology
4379 static void __kmp_affinity_get_topology_info(kmp_affinity_t &affinity) {
4382 if (affinity.type != affinity_none) {
4383 KMP_ASSERT(affinity.num_os_id_masks);
4384 KMP_ASSERT(affinity.os_id_masks);
4386 KMP_ASSERT(affinity.num_masks);
4387 KMP_ASSERT(affinity.masks);
4394 if (!affinity.ids) {
4395 affinity.ids = (kmp_affinity_ids_t *)__kmp_allocate(
4396 sizeof(kmp_affinity_ids_t) * affinity.num_masks);
4398 if (!affinity.attrs) {
4399 affinity.attrs = (kmp_affinity_attrs_t *)__kmp_allocate(
4400 sizeof(kmp_affinity_attrs_t) * affinity.num_masks);
4415 for (unsigned i = 0; i < affinity.num_masks; ++i) {
4416 kmp_affinity_ids_t &ids = affinity.ids[i];
4417 kmp_affinity_attrs_t &attrs = affinity.attrs[i];
4418 kmp_affin_mask_t *mask = KMP_CPU_INDEX(affinity.masks, i);
4424 static void __kmp_aux_affinity_initialize_other_data(kmp_affinity_t &affinity) {
4428 __kmp_affinity_get_topology_info(affinity);
4436 // initial process's affinity mask
4437 static void __kmp_create_affinity_none_places(kmp_affinity_t &affinity) {
4439 KMP_ASSERT(affinity.type == affinity_none);
4441 affinity.num_masks = 1;
4442 KMP_CPU_ALLOC_ARRAY(affinity.masks, affinity.num_masks);
4443 kmp_affin_mask_t *dest = KMP_CPU_INDEX(affinity.masks, 0);
4445 __kmp_aux_affinity_initialize_other_data(affinity);
4448 static void __kmp_aux_affinity_initialize_masks(kmp_affinity_t &affinity) {
4451 // initialization thread's affinity mask. Otherwise, it is all processors that
4453 int verbose = affinity.flags.verbose;
4454 const char *env_var = affinity.env_var;
4470 if (affinity.flags.respect) {
4481 KMP_AFF_WARNING(affinity, ErrorInitializeAffinity);
4482 affinity.type = affinity_none;
4507 // Set the process affinity mask since threads' affinity
4515 static bool __kmp_aux_affinity_initialize_topology(kmp_affinity_t &affinity) {
4517 const char *env_var = affinity.env_var;
4519 int verbose = affinity.flags.verbose;
4530 // another method. We only emit a warning message if affinity is on, or the
4590 // used, then we abort if that method fails. The exception is group affinity,
4655 KMP_AFF_WARNING(affinity, ErrorInitializeAffinity);
4679 static void __kmp_aux_affinity_initialize(kmp_affinity_t &affinity) {
4680 bool is_regular_affinity = (&affinity == &__kmp_affinity);
4681 bool is_hidden_helper_affinity = (&affinity == &__kmp_hh_affinity);
4682 const char *env_var = __kmp_get_affinity_env_var(affinity);
4684 if (affinity.flags.initialized) {
4690 __kmp_aux_affinity_initialize_masks(affinity);
4693 bool success = __kmp_aux_affinity_initialize_topology(affinity);
4697 affinity.type = affinity_none;
4703 // which is the process's initial affinity mask or the number of
4705 if (affinity.type == affinity_none) {
4706 __kmp_create_affinity_none_places(affinity);
4710 affinity.flags.initialized = TRUE;
4714 __kmp_topology->set_granularity(affinity);
4722 if (affinity.core_attr_gran.valid) {
4723 __kmp_create_os_id_masks(&numUnique, affinity, [&](int idx) {
4726 if (__kmp_topology->at(i).attrs.contains(affinity.core_attr_gran))
4730 if (!affinity.os_id_masks) {
4732 if (affinity.core_attr_gran.core_eff != kmp_hw_attr_t::UNKNOWN_CORE_EFF)
4736 KMP_AFF_WARNING(affinity, AffIgnoringNotAvailable, env_var,
4743 if (!affinity.os_id_masks) {
4744 __kmp_create_os_id_masks(&numUnique, affinity, [](int idx) {
4749 if (affinity.gran_levels == 0) {
4753 switch (affinity.type) {
4756 KMP_DEBUG_ASSERT(affinity.proclist != NULL);
4759 __kmp_affinity_process_proclist(affinity);
4761 __kmp_affinity_process_placelist(affinity);
4763 if (affinity.num_masks == 0) {
4764 KMP_AFF_WARNING(affinity, AffNoValidProcID);
4765 affinity.type = affinity_none;
4766 __kmp_create_affinity_none_places(affinity);
4767 affinity.flags.initialized = TRUE;
4772 // The other affinity types rely on sorting the hardware threads according to
4773 // some permutation of the machine topology tree. Set affinity.compact
4774 // and affinity.offset appropriately, then jump to a common code
4775 // fragment to do the sort and create the array of affinity masks.
4777 affinity.compact = 0;
4778 if (affinity.offset) {
4779 affinity.offset =
4780 __kmp_nThreadsPerCore * affinity.offset % __kmp_avail_proc;
4786 affinity.compact = 1;
4787 if (affinity.compact >= depth) {
4788 affinity.compact = 0;
4791 affinity.compact = 0;
4793 if (affinity.offset) {
4794 affinity.offset =
4795 __kmp_nThreadsPerCore * affinity.offset % __kmp_avail_proc;
4800 if (affinity.compact >= depth) {
4801 affinity.compact = 0;
4803 affinity.compact = depth - 1 - affinity.compact;
4808 if (affinity.compact >= depth) {
4809 affinity.compact = depth - 1;
4815 KMP_AFF_WARNING(affinity, AffBalancedNotAvail, env_var);
4816 affinity.type = affinity_none;
4817 __kmp_create_affinity_none_places(affinity);
4818 affinity.flags.initialized = TRUE;
4833 KMP_AFF_WARNING(affinity, AffBalancedNotAvail, env_var);
4834 affinity.type = affinity_none;
4835 __kmp_create_affinity_none_places(affinity);
4836 affinity.flags.initialized = TRUE;
4861 if (affinity.compact >= depth) {
4862 affinity.compact = depth - 1;
4866 // Allocate the gtid->affinity mask table.
4867 if (affinity.flags.dups) {
4868 affinity.num_masks = __kmp_avail_proc;
4870 affinity.num_masks = numUnique;
4875 ((unsigned)__kmp_affinity_num_places < affinity.num_masks) &&
4877 affinity.num_masks = __kmp_affinity_num_places;
4880 KMP_CPU_ALLOC_ARRAY(affinity.masks, affinity.num_masks);
4883 // affinity.compact, then fill out affinity.masks.
4884 __kmp_topology->sort_compact(affinity);
4891 if ((!affinity.flags.dups) && (!__kmp_topology->at(i).leader)) {
4896 kmp_affin_mask_t *src = KMP_CPU_INDEX(affinity.os_id_masks, osId);
4897 kmp_affin_mask_t *dest = KMP_CPU_INDEX(affinity.masks, j);
4901 if (++j >= affinity.num_masks) {
4905 KMP_DEBUG_ASSERT(j == affinity.num_masks);
4907 if (full_mask.restrict_to_mask() && affinity.flags.verbose) {
4916 KMP_ASSERT2(0, "Unexpected affinity setting");
4918 __kmp_aux_affinity_initialize_other_data(affinity);
4919 affinity.flags.initialized = TRUE;
4922 void __kmp_affinity_initialize(kmp_affinity_t &affinity) {
4924 // affinity capable, then affinity type == affinity_none.
4925 // We now explicitly represent this as affinity type == affinity_disabled.
4926 // There are too many checks for affinity type == affinity_none in this code.
4928 // affinity type == affinity_disabled, and if so, slam it with affinity_none,
4929 // call the real initialization routine, then restore affinity type to
4931 int disabled = (affinity.type == affinity_disabled);
4935 affinity.type = affinity_none;
4936 __kmp_aux_affinity_initialize(affinity);
4938 affinity.type = affinity_disabled;
4942 for (kmp_affinity_t *affinity : __kmp_affinities) {
4943 if (affinity->masks != NULL)
4944 KMP_CPU_FREE_ARRAY(affinity->masks, affinity->num_masks);
4945 if (affinity->os_id_masks != NULL)
4946 KMP_CPU_FREE_ARRAY(affinity->os_id_masks, affinity->num_os_id_masks);
4947 if (affinity->proclist != NULL)
4948 __kmp_free(affinity->proclist);
4949 if (affinity->ids != NULL)
4950 __kmp_free(affinity->ids);
4951 if (affinity->attrs != NULL)
4952 __kmp_free(affinity->attrs);
4953 *affinity = KMP_AFFINITY_INIT(affinity->env_var);
4993 static void __kmp_select_mask_by_gtid(int gtid, const kmp_affinity_t *affinity,
5003 KMP_DEBUG_ASSERT(affinity->num_masks > 0);
5004 *place = (mask_idx + affinity->offset) % affinity->num_masks;
5005 *mask = KMP_CPU_INDEX(affinity->masks, *place);
5008 // This function initializes the per-thread data concerning affinity including
5036 const kmp_affinity_t *affinity;
5040 affinity = &__kmp_hh_affinity;
5042 affinity = &__kmp_affinity;
5045 if ((affinity->type == affinity_none) ||
5046 (affinity->type == affinity_balanced) ||
5057 __kmp_select_mask_by_gtid(gtid, affinity, &i, &mask);
5070 __kmp_select_mask_by_gtid(gtid, affinity, &i, &mask);
5078 th->th.th_last_place = affinity->num_masks - 1;
5080 // When using a Non-OMP_PROC_BIND affinity method,
5083 th->th.th_last_place = affinity->num_masks - 1;
5107 const kmp_affinity_t *affinity;
5112 affinity = &__kmp_hh_affinity;
5114 affinity = &__kmp_affinity;
5115 env_var = __kmp_get_affinity_env_var(*affinity, /*for_binding=*/true);
5117 if (affinity->flags.verbose && (affinity->type == affinity_none ||
5119 affinity->type != affinity_balanced)) &&
5129 // On Windows* OS, the process affinity mask might have changed. If the user
5130 // didn't request affinity and this call fails, just continue silently.
5132 if (affinity->type == affinity_none) {
5167 // and set this thread's affinity.
5199 "kmp_set_affinity: setting affinity mask for thread %d = %s\n",
5243 // Turn off 4.0 affinity for the current tread at this parallel level.
5273 "kmp_get_affinity: stored affinity mask for thread %d = %s\n", gtid,
5292 "kmp_get_affinity: system affinity mask for thread %d = %s\n", gtid,
5330 "affinity mask for thread %d = %s\n",
5363 "affinity mask for thread %d = %s\n",
5396 "affinity mask for thread %d = %s\n",
5438 // Dynamic affinity settings - Affinity balanced
5445 // Do not perform balanced affinity for the hidden helper threads
5491 "Illegal set affinity operation when not capable");
5691 // 1) The user gets the current affinity mask
5692 // 2) Then sets the affinity by calling this function
5695 // 5) Reset the affinity to what was stored in step 1)
5714 "affinity not initialized, returning\n"));