enh: Select platform optimizations at runtime

Replace compile-time AWS platform detection with runtime EFA device
detection using hwloc. This allows a single binary to work on both
AWS and non-AWS environments, automatically enabling optimizations
when EFA hardware is present. Removes autotools platform checks and
always builds AWS platform code.

Signed-off-by: Hershel Shah <[email protected]>

Author: hershys-aws

configure.ac

@@ -137,9 +137,6 @@ CHECK_PKG_HWLOC([],
 CHECK_PKG_VALGRIND()
 CHECK_VAR_REDZONE()
 
-NCCL_OFI_PLATFORM="none"
-AS_IF([test "${NCCL_OFI_PLATFORM}" = "none"], [AX_CHECK_PLATFORM_AWS()])
-
 AS_IF([test "${valgrind_enabled}" = "1" -a "${enable_asan}" = "yes"],
       [AC_MSG_ERROR([Enabling ASAN and valgrind at the same time is not permitted])])
 
@@ -278,5 +275,5 @@ AC_OUTPUT
 echo "*"
 echo "* AWS OFI NCCL plugin has been configured."
 echo "*"
-echo "* Platform-specific optimizations: ${NCCL_OFI_PLATFORM}"
+echo "* Platform optimizations: AWS, Default"
 echo "*"

include/nccl_ofi_param.h

@@ -374,4 +374,10 @@ OFI_NCCL_PARAM_UINT(cm_num_rx_buffers, "CM_NUM_RX_BUFFERS", 32);
 OFI_NCCL_PARAM_VALUE_SET(PROGRESS_MODEL, (UNSPEC)(AUTO)(MANUAL))
 OFI_NCCL_PARAM(PROGRESS_MODEL, progress_model,  "PROGRESS_MODEL", PROGRESS_MODEL::UNSPEC)
 
+/*
+ * Override platform selection. Valid options: AWS, DEFAULT, or AUTO for auto-detection.
+ */
+OFI_NCCL_PARAM_VALUE_SET(PLATFORM, (AUTO)(AWS)(DEFAULT))
+OFI_NCCL_PARAM(PLATFORM, platform, "PLATFORM", PLATFORM::AUTO);
+
 #endif // End NCCL_OFI_PARAM_H_

include/nccl_ofi_platform.h

@@ -135,24 +135,21 @@ class PlatformManager {
 	 */
 	static PlatformManager& get_global();
 
-	/**
-	 * @brief	Register a platform with the manager
-	 *
-	 * 		Platforms are automatically sorted by priority in the internal map.
-	 * 		Higher priority values take precedence and duplicates are dropped.
-	 *
-	 * @param	platform	Platform instance to register (moved)
-	 */
-	void register_platform(PlatformPtr&& platform);
-
 	/**
 	 * @brief	Get the highest priority platform instance
 	 *
 	 * 		Returns the platform with the highest priority value.
 	 *
 	 * @return	Reference to highest priority platform
 	 */
-	inline Platform& get_platform() { return *platforms_.rbegin()->second; }
+	inline Platform& get_platform() {
+		// std::map sorts by priority key; rbegin() gives highest priority
+		auto it = platforms_.rbegin();
+		Platform& selected = *it->second;
+		NCCL_OFI_INFO(NCCL_NET | NCCL_INIT, "Selected platform: %s with priority %d",
+		              selected.get_name(), it->first);
+		return selected;
+	}
 
 	/**
 	 * @brief	Get number of registered platforms (for testing)
@@ -167,9 +164,21 @@ class PlatformManager {
 	 *		instance is meant to be used in the plugin and the unit
 	 *		tests leverage the protected scope.
 	 */
-	PlatformManager() {
-		register_platform(std::make_unique<Default>());
-	}
+	PlatformManager();
+
+	/**
+	 * @brief	Register a platform with the manager
+	 *
+	 * 		Platforms are automatically sorted by priority in the internal map.
+	 * 		Higher priority values take precedence and duplicates are dropped.
+	 * 		This can only be done in the constructor as all platform must be
+	 * 		added on the created of this object to allow the tuner and the plugin
+	 * 		to operate consistently.
+	 *
+	 * @param	platform	Platform instance to register (moved)
+	 */
+	void register_platform(PlatformPtr&& platform);
+
 
 private:
 	std::map<int, PlatformPtr> platforms_;

include/nccl_ofi_topo.h

@@ -310,4 +310,12 @@ struct fi_info *nccl_ofi_topo_next_info_list(nccl_ofi_topo_data_iterator_t *iter
  */
 int nccl_ofi_topo_write_nccl_topology(nccl_ofi_topo_t *topo, FILE *file);
 
+/*
+ * @brief	Check if EFA or ENA devices are present in the system
+ *
+ * @return	true, if EFA or ENA devices are detected
+ *		false, otherwise
+ */
+bool nccl_ofi_topo_has_efa_ena_devices();
+
 #endif // End NCCL_NET_OFI_TOPO_H_

include/platform-aws.h

@@ -23,7 +23,7 @@
 class PlatformAWS : public Platform {
 public:
 	const char* get_name() const override { return "AWS"; }
-	int get_priority() override { return 100; }
+	int get_priority() override { return PLATFORM_PRIORITY * (is_ec2_instance() ? 1 : -1); }
 	int init(const char **provider_filter) override;
 	int config_endpoint(struct fi_info *info, struct fid_ep *ep) override;
 	void sort_rails(struct fi_info **info_list, size_t num_rails, size_t num_groups) override;
@@ -72,7 +72,13 @@ class PlatformAWS : public Platform {
 		return fields ? fields->func_idx : -EIO;
 	}
 
+	// Determine if running on Amazon EC2 instance
+	bool is_ec2_instance();
+
 private:
+	// Constants
+	static constexpr int PLATFORM_PRIORITY = 100;
+
 	std::mutex mutex_;
 
 	// Cache for GUID fields to avoid repeated sysfs reads

m4/ax_platform_aws.m4

@@ -34,12 +34,9 @@ sources = \
 	nccl_ofi_ep_addr_list.cpp \
 	nccl_ofi_param.cpp \
 	nccl_ofi_platform.cpp \
+	platform-aws.cpp \
 	tracepoint.cpp
 
-if WANT_PLATFORM_AWS
-sources += platform-aws.cpp
-endif
-
 if ENABLE_NEURON
   sources += nccl_ofi_interface_neuron.cpp
 else
@@ -48,13 +45,11 @@ else
 	nccl_ofi_interface_nvidia.cpp
 
   # add the tuner sources into the library
-if WANT_PLATFORM_AWS
-    sources +=  \
+  sources +=  \
 	tuner/nccl_ofi_regions.cpp \
 	tuner/nccl_ofi_tuner.cpp \
 	tuner/nccl_ofi_model.cpp
 endif
-endif
 
 # Build an internal-only library that can be used by unit tests as
 # well as the actual nccl_net.so / nccom_net.so libraries.  This saves
@@ -104,20 +99,19 @@ if ENABLE_NCCL_NET_LIBRARY
   libnccl_net_la_LIBTOOLFLAGS = --tag=CXX
   libnccl_net_la_LDFLAGS = -module -avoid-version
 endif
-if WANT_PLATFORM_AWS
+
 # NCCL standardized on the libnccl-tuner-<interface> format after we released a
 # plugin with the tuner named libnccl-ofi-tuner.so.  Create separate libraries
 # for each name.
-  lib_LTLIBRARIES += libnccl-ofi-tuner.la libnccl-tuner-ofi.la
-  libnccl_ofi_tuner_la_SOURCES =
-  libnccl_ofi_tuner_la_LIBADD = libinternal_plugin.la
-  libnccl_ofi_tuner_la_LIBTOOLFLAGS = --tag=CXX
-  libnccl_ofi_tuner_la_LDFLAGS = -module -avoid-version
+lib_LTLIBRARIES += libnccl-ofi-tuner.la libnccl-tuner-ofi.la
+libnccl_ofi_tuner_la_SOURCES =
+libnccl_ofi_tuner_la_LIBADD = libinternal_plugin.la
+libnccl_ofi_tuner_la_LIBTOOLFLAGS = --tag=CXX
+libnccl_ofi_tuner_la_LDFLAGS = -module -avoid-version
 
-  libnccl_tuner_ofi_la_SOURCES =
-  libnccl_tuner_ofi_la_LIBADD = libinternal_plugin.la
-  libnccl_tuner_ofi_la_LIBTOOLFLAGS = --tag=CXX
-  libnccl_tuner_ofi_la_LDFLAGS = -module -avoid-version
-endif
+libnccl_tuner_ofi_la_SOURCES =
+libnccl_tuner_ofi_la_LIBADD = libinternal_plugin.la
+libnccl_tuner_ofi_la_LIBTOOLFLAGS = --tag=CXX
+libnccl_tuner_ofi_la_LDFLAGS = -module -avoid-version
 
 endif

src/Makefile.am

@@ -30,9 +30,6 @@
 #include "nccl_ofi_idpool.h"
 #include "nccl_ofi_dmabuf.h"
 #include "nccl_ofi_platform.h"
-#ifdef WANT_AWS_PLATFORM
-#include "platform-aws.h"
-#endif
 #include "nccl_ofi_ofiutils.h"
 #include "nccl_ofi_system.h"
 
@@ -178,10 +175,6 @@ int nccl_net_ofi_create_plugin(nccl_net_ofi_plugin_t **plugin_p)
 	nic_dup_conns = ofi_nccl_nic_dup_conns();
 	cq_read_count = ofi_nccl_cq_read_count();
 
-#ifdef WANT_AWS_PLATFORM
-	PlatformManager::get_global().register_platform(std::make_unique<PlatformAWS>());
-#endif
-
 	ret = PlatformManager::get_global().get_platform().init(&provider_filter);
 	if (ret != 0)
 		goto exit;

src/nccl_ofi_net.cpp

@@ -3,6 +3,12 @@
  */
 
 #include "nccl_ofi_platform.h"
+#include "platform-aws.h"
+
+PlatformManager::PlatformManager() {
+	register_platform(std::make_unique<Default>());
+	register_platform(std::make_unique<PlatformAWS>());
+}
 
 PlatformManager& PlatformManager::get_global() {
 	static PlatformManager manager;
@@ -21,6 +27,6 @@ void PlatformManager::register_platform(PlatformPtr&& platform) {
 		// TODO: Add proper resolution mechanism for competing priorities
 		priority++;
 	}
-
+	NCCL_OFI_INFO(NCCL_INIT | NCCL_NET, "Adding %s platform with %d priority", name, priority);
 	platforms_[priority] = std::move(platform);
 }

src/nccl_ofi_platform.cpp

@@ -1722,3 +1722,29 @@ struct fi_info *nccl_ofi_topo_next_info_list(nccl_ofi_topo_data_iterator_t *iter
 
 	return info_list;
 }
+
+bool nccl_ofi_topo_has_efa_ena_devices()
+{
+	hwloc_topology_t topo;
+	if (hwloc_topology_init(&topo) != 0) {
+		return false;
+	}
+
+	auto topology = std::shared_ptr<hwloc_topology>(topo, hwloc_topology_destroy);
+	enable_hwloc_io_types(topo);
+
+	if (hwloc_topology_load(topo) != 0) {
+		return false;
+	}
+
+	hwloc_obj_t obj = nullptr;
+	while ((obj = hwloc_get_next_pcidev(topo, obj)) != nullptr) {
+		// Check for Amazon vendor id and EFA device or ENA device
+		if (obj->attr->pcidev.vendor_id == 0x1D0F &&
+		    (((obj->attr->pcidev.device_id & 0xFFF0) == 0xEFA0 || (obj->attr->pcidev.device_id & 0xFFF0) == 0xEC20) ||
+		     (obj->attr->pcidev.device_id & 0x0FFF) == 0x0EC2)) {
+			return true;
+		}
+	}
+	return false;
+}