Lines Matching defs:affinity
2 * kmp_affinity.cpp -- affinity management
67 __kmp_get_affinity_env_var(const kmp_affinity_t &affinity,
69 if (affinity.flags.omp_places) {
74 return affinity.env_var;
80 // The test below is true if affinity is available, but set to "none". Need to
189 // If affinity is supported, check the affinity
875 void kmp_topology_t::set_granularity(kmp_affinity_t &affinity) const {
876 const char *env_var = __kmp_get_affinity_env_var(affinity);
881 if (affinity.core_attr_gran.valid) {
885 affinity, AffIgnoringNonHybrid, env_var,
887 affinity.gran = KMP_HW_CORE;
888 affinity.gran_levels = -1;
889 affinity.core_attr_gran = KMP_AFFINITY_ATTRS_UNKNOWN;
890 affinity.flags.core_types_gran = affinity.flags.core_effs_gran = 0;
891 } else if (affinity.flags.core_types_gran ||
892 affinity.flags.core_effs_gran) {
894 if (affinity.flags.omp_places) {
896 affinity, AffIgnoringNonHybrid, env_var,
900 KMP_AFF_WARNING(affinity, AffGranularityBad, env_var,
904 affinity.gran = KMP_HW_CORE;
905 affinity.gran_levels = -1;
906 affinity.core_attr_gran = KMP_AFFINITY_ATTRS_UNKNOWN;
907 affinity.flags.core_types_gran = affinity.flags.core_effs_gran = 0;
910 // Set the number of affinity granularity levels
911 if (affinity.gran_levels < 0) {
912 kmp_hw_t gran_type = get_equivalent_type(affinity.gran);
925 KMP_AFF_WARNING(affinity, AffGranularityBad, env_var,
926 __kmp_hw_get_catalog_string(affinity.gran),
928 affinity.gran = gran_type;
933 // granularity must be adjusted "down" to processor group affinity
943 KMP_AFF_WARNING(affinity, AffGranTooCoarseProcGroup, env_var,
944 __kmp_hw_get_catalog_string(affinity.gran));
945 affinity.gran = gran_type = KMP_HW_PROC_GROUP;
949 affinity.gran_levels = 0;
951 affinity.gran_levels++;
1442 // Only use Hwloc if affinity isn't explicitly disabled and
1469 // Print the affinity mask to the character array in a pretty format.
1510 // Range with three or more contiguous bits in the affinity mask
1514 // Range with one or two contiguous bits in the affinity mask
1537 // Print the affinity mask to the string buffer object in a pretty format
1574 // Range with three or more contiguous bits in the affinity mask
1578 // Range with one or two contiguous bits in the affinity mask
1648 // Return (possibly empty) affinity mask representing the offline CPUs
2031 // When affinity is off, this routine will still be called to set
2033 // Make sure all these vars are set correctly, and return now if affinity is
2077 // If we aren't affinity capable, then use flat topology
2241 // the current thread's affinity mask to that thread, and then retrieves
2258 // The algorithm used starts by setting the affinity to each available thread
2299 // if the machine is not affinity capable, we assume that HT is off. We have
2320 // Save the affinity mask for the current thread.
2398 // affinity mask before the tail call.
2414 // Restore the old affinity mask for this thread.
2497 // When affinity is off, this routine will still be called to set
2808 // If affinity is disabled, __kmp_avail_proc may be zero
2867 // The algorithm used starts by setting the affinity to each available thread
2893 // Save the affinity mask for the current thread.
3214 // affinity map. On AIX, the map is obtained through system SRAD (Scheduler
3264 // of records can't exceed the number of valid bits in the affinity mask.
3791 // When affinity is off, this routine will still be called to set
3793 // Make sure all these vars are set correctly, and return now if affinity is
3921 // Create and return a table of affinity masks, indexed by OS thread ID.
3922 // This routine handles OR'ing together all the affinity masks of threads
3926 kmp_affinity_t &affinity,
3928 // First form a table of affinity masks in order of OS thread id.
3933 const char *env_var = __kmp_get_affinity_env_var(affinity);
3952 affinity.num_os_id_masks = maxOsId + 1;
3953 KMP_CPU_ALLOC_ARRAY(affinity.os_id_masks, affinity.num_os_id_masks);
3954 KMP_ASSERT(affinity.gran_levels >= 0);
3955 if (affinity.flags.verbose && (affinity.gran_levels > 0)) {
3956 KMP_INFORM(ThreadsMigrate, env_var, affinity.gran_levels);
3958 if (affinity.gran_levels >= (int)depth) {
3959 KMP_AFF_WARNING(affinity, AffThreadsMayMigrate);
3979 // affinity mask for this group, and go on to the next thread.
3980 if (__kmp_topology->is_close(leader, i, affinity)) {
3990 kmp_affin_mask_t *mask = KMP_CPU_INDEX(affinity.os_id_masks, osId);
4008 kmp_affin_mask_t *mask = KMP_CPU_INDEX(affinity.os_id_masks, osId);
4017 if (full_mask.restrict_to_mask() && affinity.flags.verbose) {
4024 // Stuff for the affinity proclist parsers. It's easier to declare these vars
4054 KMP_AFF_WARNING(affinity, AffIgnoreInvalidProcID, _osId); \
4060 // Re-parse the proclist (for the explicit affinity type), and form the list
4061 // of affinity newMasks indexed by gtid.
4062 static void __kmp_affinity_process_proclist(kmp_affinity_t &affinity) {
4064 kmp_affin_mask_t **out_masks = &affinity.masks;
4065 unsigned *out_numMasks = &affinity.num_masks;
4066 const char *proclist = affinity.proclist;
4067 kmp_affin_mask_t *osId2Mask = affinity.os_id_masks;
4068 int maxOsId = affinity.num_os_id_masks - 1;
4106 KMP_AFF_WARNING(affinity, AffIgnoreInvalidProcID, num);
4138 KMP_AFF_WARNING(affinity, AffIgnoreInvalidProcID, num);
4273 kmp_affinity_t &affinity, int maxOsId,
4277 kmp_affin_mask_t *osId2Mask = affinity.os_id_masks;
4296 KMP_AFF_WARNING(affinity, AffIgnoreInvalidProcID, start);
4325 KMP_AFF_WARNING(affinity, AffIgnoreInvalidProcID, start);
4372 KMP_AFF_WARNING(affinity, AffIgnoreInvalidProcID, start);
4391 static void __kmp_process_place(const char **scan, kmp_affinity_t &affinity,
4395 kmp_affin_mask_t *osId2Mask = affinity.os_id_masks;
4401 __kmp_process_subplace_list(scan, affinity, maxOsId, tempMask, setSize);
4406 __kmp_process_place(scan, affinity, maxOsId, tempMask, setSize);
4415 KMP_AFF_WARNING(affinity, AffIgnoreInvalidProcID, num);
4427 void __kmp_affinity_process_placelist(kmp_affinity_t &affinity) {
4429 kmp_affin_mask_t **out_masks = &affinity.masks;
4430 unsigned *out_numMasks = &affinity.num_masks;
4431 const char *placelist = affinity.proclist;
4432 kmp_affin_mask_t *osId2Mask = affinity.os_id_masks;
4433 int maxOsId = affinity.num_os_id_masks - 1;
4453 __kmp_process_place(&scan, affinity, maxOsId, tempMask, &setSize);
4534 KMP_AFF_WARNING(affinity, AffIgnoreInvalidProcID, j + stride);
4692 // A thread can then grab not only its affinity mask, but the topology
4694 static void __kmp_affinity_get_topology_info(kmp_affinity_t &affinity) {
4697 if (affinity.type != affinity_none) {
4698 KMP_ASSERT(affinity.num_os_id_masks);
4699 KMP_ASSERT(affinity.os_id_masks);
4701 KMP_ASSERT(affinity.num_masks);
4702 KMP_ASSERT(affinity.masks);
4709 if (!affinity.ids) {
4710 affinity.ids = (kmp_affinity_ids_t *)__kmp_allocate(
4711 sizeof(kmp_affinity_ids_t) * affinity.num_masks);
4713 if (!affinity.attrs) {
4714 affinity.attrs = (kmp_affinity_attrs_t *)__kmp_allocate(
4715 sizeof(kmp_affinity_attrs_t) * affinity.num_masks);
4730 for (unsigned i = 0; i < affinity.num_masks; ++i) {
4731 kmp_affinity_ids_t &ids = affinity.ids[i];
4732 kmp_affinity_attrs_t &attrs = affinity.attrs[i];
4733 kmp_affin_mask_t *mask = KMP_CPU_INDEX(affinity.masks, i);
4739 static void __kmp_aux_affinity_initialize_other_data(kmp_affinity_t &affinity) {
4743 __kmp_affinity_get_topology_info(affinity);
4751 // initial process's affinity mask
4752 static void __kmp_create_affinity_none_places(kmp_affinity_t &affinity) {
4754 KMP_ASSERT(affinity.type == affinity_none);
4756 affinity.num_masks = 1;
4757 KMP_CPU_ALLOC_ARRAY(affinity.masks, affinity.num_masks);
4758 kmp_affin_mask_t *dest = KMP_CPU_INDEX(affinity.masks, 0);
4760 __kmp_aux_affinity_initialize_other_data(affinity);
4763 static void __kmp_aux_affinity_initialize_masks(kmp_affinity_t &affinity) {
4766 // initialization thread's affinity mask. Otherwise, it is all processors that
4768 int verbose = affinity.flags.verbose;
4769 const char *env_var = affinity.env_var;
4785 if (affinity.flags.respect) {
4796 KMP_AFF_WARNING(affinity, ErrorInitializeAffinity);
4797 affinity.type = affinity_none;
4822 // Set the process affinity mask since threads' affinity
4830 static bool __kmp_aux_affinity_initialize_topology(kmp_affinity_t &affinity) {
4832 const char *env_var = affinity.env_var;
4834 int verbose = affinity.flags.verbose;
4845 // another method. We only emit a warning message if affinity is on, or the
4905 // used, then we abort if that method fails. The exception is group affinity,
4970 KMP_AFF_WARNING(affinity, ErrorInitializeAffinity);
4994 static void __kmp_aux_affinity_initialize(kmp_affinity_t &affinity) {
4995 bool is_regular_affinity = (&affinity == &__kmp_affinity);
4996 bool is_hidden_helper_affinity = (&affinity == &__kmp_hh_affinity);
4997 const char *env_var = __kmp_get_affinity_env_var(affinity);
4999 if (affinity.flags.initialized) {
5005 __kmp_aux_affinity_initialize_masks(affinity);
5008 bool success = __kmp_aux_affinity_initialize_topology(affinity);
5012 affinity.type = affinity_none;
5018 // which is the process's initial affinity mask or the number of
5020 if (affinity.type == affinity_none) {
5021 __kmp_create_affinity_none_places(affinity);
5025 affinity.flags.initialized = TRUE;
5029 __kmp_topology->set_granularity(affinity);
5037 if (affinity.core_attr_gran.valid) {
5038 __kmp_create_os_id_masks(&numUnique, affinity, [&](int idx) {
5041 if (__kmp_topology->at(i).attrs.contains(affinity.core_attr_gran))
5045 if (!affinity.os_id_masks) {
5047 if (affinity.core_attr_gran.core_eff != kmp_hw_attr_t::UNKNOWN_CORE_EFF)
5051 KMP_AFF_WARNING(affinity, AffIgnoringNotAvailable, env_var,
5059 if (!affinity.os_id_masks) {
5060 int gran = affinity.gran_levels;
5061 int gran_level = depth - 1 - affinity.gran_levels;
5064 &numUnique, affinity, [depth, numAddrs, &affinity](int idx) {
5066 int gran = affinity.gran_levels;
5067 int gran_level = depth - 1 - affinity.gran_levels;
5078 if (!affinity.os_id_masks) {
5079 __kmp_create_os_id_masks(&numUnique, affinity, [](int idx) {
5085 switch (affinity.type) {
5088 KMP_DEBUG_ASSERT(affinity.proclist != NULL);
5091 __kmp_affinity_process_proclist(affinity);
5093 __kmp_affinity_process_placelist(affinity);
5095 if (affinity.num_masks == 0) {
5096 KMP_AFF_WARNING(affinity, AffNoValidProcID);
5097 affinity.type = affinity_none;
5098 __kmp_create_affinity_none_places(affinity);
5099 affinity.flags.initialized = TRUE;
5104 // The other affinity types rely on sorting the hardware threads according to
5105 // some permutation of the machine topology tree. Set affinity.compact
5106 // and affinity.offset appropriately, then jump to a common code
5107 // fragment to do the sort and create the array of affinity masks.
5109 affinity.compact = 0;
5110 if (affinity.offset) {
5111 affinity.offset =
5112 __kmp_nThreadsPerCore * affinity.offset % __kmp_avail_proc;
5118 affinity.compact = 1;
5119 if (affinity.compact >= depth) {
5120 affinity.compact = 0;
5123 affinity.compact = 0;
5125 if (affinity.offset) {
5126 affinity.offset =
5127 __kmp_nThreadsPerCore * affinity.offset % __kmp_avail_proc;
5132 if (affinity.compact >= depth) {
5133 affinity.compact = 0;
5135 affinity.compact = depth - 1 - affinity.compact;
5140 if (affinity.compact >= depth) {
5141 affinity.compact = depth - 1;
5147 KMP_AFF_WARNING(affinity, AffBalancedNotAvail, env_var);
5148 affinity.type = affinity_none;
5149 __kmp_create_affinity_none_places(affinity);
5150 affinity.flags.initialized = TRUE;
5165 KMP_AFF_WARNING(affinity, AffBalancedNotAvail, env_var);
5166 affinity.type = affinity_none;
5167 __kmp_create_affinity_none_places(affinity);
5168 affinity.flags.initialized = TRUE;
5193 if (affinity.compact >= depth) {
5194 affinity.compact = depth - 1;
5198 // Allocate the gtid->affinity mask table.
5199 if (affinity.flags.dups) {
5200 affinity.num_masks = __kmp_avail_proc;
5202 affinity.num_masks = numUnique;
5207 ((unsigned)__kmp_affinity_num_places < affinity.num_masks) &&
5209 affinity.num_masks = __kmp_affinity_num_places;
5212 KMP_CPU_ALLOC_ARRAY(affinity.masks, affinity.num_masks);
5215 // affinity.compact, then fill out affinity.masks.
5216 __kmp_topology->sort_compact(affinity);
5223 if ((!affinity.flags.dups) && (!__kmp_topology->at(i).leader)) {
5228 kmp_affin_mask_t *src = KMP_CPU_INDEX(affinity.os_id_masks, osId);
5231 kmp_affin_mask_t *dest = KMP_CPU_INDEX(affinity.masks, j);
5235 if (++j >= affinity.num_masks) {
5239 KMP_DEBUG_ASSERT(j == affinity.num_masks);
5241 if (full_mask.restrict_to_mask() && affinity.flags.verbose) {
5250 KMP_ASSERT2(0, "Unexpected affinity setting");
5252 __kmp_aux_affinity_initialize_other_data(affinity);
5253 affinity.flags.initialized = TRUE;
5256 void __kmp_affinity_initialize(kmp_affinity_t &affinity) {
5258 // affinity capable, then affinity type == affinity_none.
5259 // We now explicitly represent this as affinity type == affinity_disabled.
5260 // There are too many checks for affinity type == affinity_none in this code.
5262 // affinity type == affinity_disabled, and if so, slam it with affinity_none,
5263 // call the real initialization routine, then restore affinity type to
5265 int disabled = (affinity.type == affinity_disabled);
5269 affinity.type = affinity_none;
5270 __kmp_aux_affinity_initialize(affinity);
5272 affinity.type = affinity_disabled;
5276 for (kmp_affinity_t *affinity : __kmp_affinities) {
5277 if (affinity->masks != NULL)
5278 KMP_CPU_FREE_ARRAY(affinity->masks, affinity->num_masks);
5279 if (affinity->os_id_masks != NULL)
5280 KMP_CPU_FREE_ARRAY(affinity->os_id_masks, affinity->num_os_id_masks);
5281 if (affinity->proclist != NULL)
5282 __kmp_free(affinity->proclist);
5283 if (affinity->ids != NULL)
5284 __kmp_free(affinity->ids);
5285 if (affinity->attrs != NULL)
5286 __kmp_free(affinity->attrs);
5287 *affinity = KMP_AFFINITY_INIT(affinity->env_var);
5327 static void __kmp_select_mask_by_gtid(int gtid, const kmp_affinity_t *affinity,
5337 KMP_DEBUG_ASSERT(affinity->num_masks > 0);
5338 *place = (mask_idx + affinity->offset) % affinity->num_masks;
5339 *mask = KMP_CPU_INDEX(affinity->masks, *place);
5342 // This function initializes the per-thread data concerning affinity including
5370 const kmp_affinity_t *affinity;
5374 affinity = &__kmp_hh_affinity;
5376 affinity = &__kmp_affinity;
5379 if ((affinity->type == affinity_none) ||
5380 (affinity->type == affinity_balanced) ||
5391 __kmp_select_mask_by_gtid(gtid, affinity, &i, &mask);
5404 __kmp_select_mask_by_gtid(gtid, affinity, &i, &mask);
5412 th->th.th_last_place = affinity->num_masks - 1;
5414 // When using a Non-OMP_PROC_BIND affinity method,
5417 th->th.th_last_place = affinity->num_masks - 1;
5441 const kmp_affinity_t *affinity;
5446 affinity = &__kmp_hh_affinity;
5448 affinity = &__kmp_affinity;
5449 env_var = __kmp_get_affinity_env_var(*affinity, /*for_binding=*/true);
5451 if (affinity->flags.verbose && (affinity->type == affinity_none ||
5453 affinity->type != affinity_balanced)) &&
5463 // On Windows* OS, the process affinity mask might have changed. If the user
5464 // didn't request affinity and this call fails, just continue silently.
5466 if (affinity->type == affinity_none) {
5501 // and set this thread's affinity.
5533 "kmp_set_affinity: setting affinity mask for thread %d = %s\n",
5577 // Turn off 4.0 affinity for the current tread at this parallel level.
5607 "kmp_get_affinity: stored affinity mask for thread %d = %s\n", gtid,
5626 "kmp_get_affinity: system affinity mask for thread %d = %s\n", gtid,
5664 "affinity mask for thread %d = %s\n",
5697 "affinity mask for thread %d = %s\n",
5730 "affinity mask for thread %d = %s\n",
5772 // Dynamic affinity settings - Affinity balanced
5779 // Do not perform balanced affinity for the hidden helper threads
5825 "Illegal set affinity operation when not capable");
6025 // 1) The user gets the current affinity mask
6026 // 2) Then sets the affinity by calling this function
6029 // 5) Reset the affinity to what was stored in step 1)
6048 "affinity not initialized, returning\n"));