enh: Select platform optimizations at runtime

Replace compile-time AWS platform detection with runtime EFA device
detection using hwloc. This allows a single binary to work on both
AWS and non-AWS environments, automatically enabling optimizations
when EFA hardware is present. Removes autotools platform checks and
always builds AWS platform code.

Signed-off-by: Hershel Shah <[email protected]>

Author: hershys-aws

configure.ac

@@ -137,9 +137,6 @@ CHECK_PKG_HWLOC([],
 CHECK_PKG_VALGRIND()
 CHECK_VAR_REDZONE()
 
-NCCL_OFI_PLATFORM="none"
-AS_IF([test "${NCCL_OFI_PLATFORM}" = "none"], [AX_CHECK_PLATFORM_AWS()])
-
 AS_IF([test "${valgrind_enabled}" = "1" -a "${enable_asan}" = "yes"],
       [AC_MSG_ERROR([Enabling ASAN and valgrind at the same time is not permitted])])
 
@@ -278,5 +275,5 @@ AC_OUTPUT
 echo "*"
 echo "* AWS OFI NCCL plugin has been configured."
 echo "*"
-echo "* Platform-specific optimizations: ${NCCL_OFI_PLATFORM}"
+echo "* Platform optimizations: Runtime detection enabled"
 echo "*"

include/nccl_ofi_param.h

@@ -374,4 +374,10 @@ OFI_NCCL_PARAM_UINT(cm_num_rx_buffers, "CM_NUM_RX_BUFFERS", 32);
 OFI_NCCL_PARAM_VALUE_SET(PROGRESS_MODEL, (UNSPEC)(AUTO)(MANUAL))
 OFI_NCCL_PARAM(PROGRESS_MODEL, progress_model,  "PROGRESS_MODEL", PROGRESS_MODEL::UNSPEC)
 
+/*
+ * Force non-AWS platform detection for testing. When set to true,
+ * the AWS platform will report negative priority even on AWS hardware.
+ */
+OFI_NCCL_PARAM(bool, force_non_aws, "FORCE_NON_AWS", false);
+
 #endif // End NCCL_OFI_PARAM_H_

include/nccl_ofi_platform.h

@@ -152,7 +152,12 @@ class PlatformManager {
 	 *
 	 * @return	Reference to highest priority platform
 	 */
-	inline Platform& get_platform() { return *platforms_.rbegin()->second; }
+	inline Platform& get_platform() {
+		Platform& selected = *platforms_.rbegin()->second;
+		NCCL_OFI_INFO(NCCL_INIT, "Selected platform: %s with priority %d",
+		              selected.get_name(), platforms_.rbegin()->first);
+		return selected;
+	}
 
 	/**
 	 * @brief	Get number of registered platforms (for testing)

include/platform-aws.h

@@ -23,7 +23,7 @@
 class PlatformAWS : public Platform {
 public:
 	const char* get_name() const override { return "AWS"; }
-	int get_priority() override { return 100; }
+	int get_priority() override { return BASE_PRIORITY * (is_aws() ? 1 : -1); }
 	int init(const char **provider_filter) override;
 	int config_endpoint(struct fi_info *info, struct fid_ep *ep) override;
 	void sort_rails(struct fi_info **info_list, size_t num_rails, size_t num_groups) override;
@@ -72,7 +72,15 @@ class PlatformAWS : public Platform {
 		return fields ? fields->func_idx : -EIO;
 	}
 
+	// Determine AWS at runtime
+	bool is_aws() const;
+
 private:
+	// Constants
+	static constexpr int BASE_PRIORITY = 100;
+	static constexpr unsigned short AWS_VENDOR_ID = 0x1D0F;
+	static constexpr unsigned short EFA_DEV = 0xEFA0;
+
 	std::mutex mutex_;
 
 	// Cache for GUID fields to avoid repeated sysfs reads

m4/ax_platform_aws.m4

@@ -34,12 +34,9 @@ sources = \
 	nccl_ofi_ep_addr_list.cpp \
 	nccl_ofi_param.cpp \
 	nccl_ofi_platform.cpp \
+	platform-aws.cpp \
 	tracepoint.cpp
 
-if WANT_PLATFORM_AWS
-sources += platform-aws.cpp
-endif
-
 if ENABLE_NEURON
   sources += nccl_ofi_interface_neuron.cpp
 else
@@ -48,13 +45,11 @@ else
 	nccl_ofi_interface_nvidia.cpp
 
   # add the tuner sources into the library
-if WANT_PLATFORM_AWS
-    sources +=  \
+  sources +=  \
 	tuner/nccl_ofi_regions.cpp \
 	tuner/nccl_ofi_tuner.cpp \
 	tuner/nccl_ofi_model.cpp
 endif
-endif
 
 # Build an internal-only library that can be used by unit tests as
 # well as the actual nccl_net.so / nccom_net.so libraries.  This saves
@@ -104,20 +99,18 @@ if ENABLE_NCCL_NET_LIBRARY
   libnccl_net_la_LIBTOOLFLAGS = --tag=CXX
   libnccl_net_la_LDFLAGS = -module -avoid-version
 endif
-if WANT_PLATFORM_AWS
+
 # NCCL standardized on the libnccl-tuner-<interface> format after we released a
 # plugin with the tuner named libnccl-ofi-tuner.so.  Create separate libraries
-# for each name.
-  lib_LTLIBRARIES += libnccl-ofi-tuner.la libnccl-tuner-ofi.la
-  libnccl_ofi_tuner_la_SOURCES =
-  libnccl_ofi_tuner_la_LIBADD = libinternal_plugin.la
-  libnccl_ofi_tuner_la_LIBTOOLFLAGS = --tag=CXX
-  libnccl_ofi_tuner_la_LDFLAGS = -module -avoid-version
+lib_LTLIBRARIES += libnccl-ofi-tuner.la libnccl-tuner-ofi.la
+libnccl_ofi_tuner_la_SOURCES =
+libnccl_ofi_tuner_la_LIBADD = libinternal_plugin.la
+libnccl_ofi_tuner_la_LIBTOOLFLAGS = --tag=CXX
+libnccl_ofi_tuner_la_LDFLAGS = -module -avoid-version
 
-  libnccl_tuner_ofi_la_SOURCES =
-  libnccl_tuner_ofi_la_LIBADD = libinternal_plugin.la
-  libnccl_tuner_ofi_la_LIBTOOLFLAGS = --tag=CXX
-  libnccl_tuner_ofi_la_LDFLAGS = -module -avoid-version
-endif
+libnccl_tuner_ofi_la_SOURCES =
+libnccl_tuner_ofi_la_LIBADD = libinternal_plugin.la
+libnccl_tuner_ofi_la_LIBTOOLFLAGS = --tag=CXX
+libnccl_tuner_ofi_la_LDFLAGS = -module -avoid-version
 
 endif

src/Makefile.am

@@ -30,9 +30,7 @@
 #include "nccl_ofi_idpool.h"
 #include "nccl_ofi_dmabuf.h"
 #include "nccl_ofi_platform.h"
-#ifdef WANT_AWS_PLATFORM
 #include "platform-aws.h"
-#endif
 #include "nccl_ofi_ofiutils.h"
 #include "nccl_ofi_system.h"
 
@@ -178,9 +176,7 @@ int nccl_net_ofi_create_plugin(nccl_net_ofi_plugin_t **plugin_p)
 	nic_dup_conns = ofi_nccl_nic_dup_conns();
 	cq_read_count = ofi_nccl_cq_read_count();
 
-#ifdef WANT_AWS_PLATFORM
 	PlatformManager::get_global().register_platform(std::make_unique<PlatformAWS>());
-#endif
 
 	ret = PlatformManager::get_global().get_platform().init(&provider_filter);
 	if (ret != 0)
@@ -1058,7 +1054,7 @@ int nccl_net_ofi_ep_t::release_ep(bool skip_lock, bool force_cleanup)
 
 	/* Store ref_cnt in local variable in case the endpoint gets deleted */
 	int local_ref_cnt = this->ref_cnt;
-	
+
 	if (local_ref_cnt == 0 || force_cleanup) {
 		/* If this was the endpoint we stored in domain for connection
 		   management, remove that reference as well */

src/nccl_ofi_net.cpp

@@ -21,6 +21,6 @@ void PlatformManager::register_platform(PlatformPtr&& platform) {
 		// TODO: Add proper resolution mechanism for competing priorities
 		priority++;
 	}
-
+	NCCL_OFI_INFO(NCCL_INIT, "Adding %s platform with %d priority", name, priority);
 	platforms_[priority] = std::move(platform);
 }

src/nccl_ofi_platform.cpp

@@ -1072,7 +1072,7 @@ static hwloc_obj_t get_numa_mem_child(hwloc_obj_t node)
 	return child;
 }
 
-/* 
+/*
  * @brief	Return PCI device property of PCI device
  *
  * This function reads first `MAX_DEV_PROPERTY_LENGTH` characters from
@@ -1093,7 +1093,7 @@ static hwloc_obj_t get_numa_mem_child(hwloc_obj_t node)
  *		File name of the device property
  * @return	Pointer to an element of a char array to write device property to.
  *		The array has to be allocated by the caller of this function.
- *		There must be space for at least `MAX_DEV_PROPERTY_LENGTH` 
+ *		There must be space for at least `MAX_DEV_PROPERTY_LENGTH`
  *		characters in addition to the delimiting `\0`.
  * @return	0, on sucess
  *		non-zero, on error

src/nccl_ofi_topo.cpp

@@ -21,6 +21,9 @@
 #include <map>
 #include <mutex>
 #include <string>
+#include <memory>
+#include <hwloc.h>
+#include <unistd.h>
 
 #ifdef HAVE_RDMA_FI_EXT_H
 #include <rdma/fi_ext.h>
@@ -1047,3 +1050,26 @@ void PlatformAWS::sort_rails(struct fi_info **info_list, size_t num_rails, size_
 
 	return;
 }
+
+bool PlatformAWS::is_aws() const {
+	if (ofi_nccl_force_non_aws.get()) {
+		NCCL_OFI_INFO(NCCL_INIT, "Disabling PlatformAWS optimizations");
+		return false;
+	}
+
+	hwloc_topology_t topo;
+	if (hwloc_topology_init(&topo) != 0) return false;
+
+	auto topology = std::shared_ptr<hwloc_topology>(topo, hwloc_topology_destroy);
+	hwloc_topology_set_io_types_filter(topo, HWLOC_TYPE_FILTER_KEEP_ALL);
+	if (hwloc_topology_load(topo) != 0) return false;
+
+	hwloc_obj_t obj = nullptr;
+	while ((obj = hwloc_get_next_pcidev(topo, obj)) != nullptr) {
+		if (obj->attr->pcidev.vendor_id == AWS_VENDOR_ID &&
+		    (obj->attr->pcidev.device_id & 0xFFF0) == EFA_DEV) {
+			return true;
+		}
+	}
+	return false;
+}