enh: Select platform optimizations at runtime

Replace compile-time AWS platform detection with runtime EFA device
detection using hwloc. This allows a single binary to work on both
AWS and non-AWS environments, automatically enabling optimizations
when EFA hardware is present. Removes autotools platform checks and
always builds AWS platform code.

Signed-off-by: Hershel Shah <[email protected]>

Author: hershys-aws

configure.ac

@@ -278,5 +278,7 @@ AC_OUTPUT
 echo "*"
 echo "* AWS OFI NCCL plugin has been configured."
 echo "*"
-echo "* Platform-specific optimizations: ${NCCL_OFI_PLATFORM}"
+AS_IF([test "${NCCL_OFI_PLATFORM}" = "none"],
+      [echo "* Platform Optimizations: DISABLED"],
+      [echo "* Platform Optimizations: ${NCCL_OFI_PLATFORM}"])
 echo "*"

include/nccl_ofi_param.h

@@ -374,4 +374,9 @@ OFI_NCCL_PARAM_UINT(cm_num_rx_buffers, "CM_NUM_RX_BUFFERS", 32);
 OFI_NCCL_PARAM_VALUE_SET(PROGRESS_MODEL, (UNSPEC)(AUTO)(MANUAL))
 OFI_NCCL_PARAM(PROGRESS_MODEL, progress_model,  "PROGRESS_MODEL", PROGRESS_MODEL::UNSPEC)
 
+/*
+ * Override platform selection. Valid options: "AWS", "Default", or empty string for auto-detection.
+ */
+OFI_NCCL_PARAM(std::string, platform_override, "PLATFORM_OVERRIDE", "");
+
 #endif // End NCCL_OFI_PARAM_H_

include/nccl_ofi_platform.h

@@ -7,10 +7,12 @@
 
 #include <memory>
 #include <map>
+#include <limits>
 
 #include <rdma/fabric.h>
 #include <rdma/fi_endpoint.h>
 
+#include "nccl_ofi_param.h"
 #include "nccl_ofi_system.h"
 
 /**
@@ -135,44 +137,57 @@ class PlatformManager {
 	 */
 	static PlatformManager& get_global();
 
-	/**
-	 * @brief	Register a platform with the manager
-	 *
-	 * 		Platforms are automatically sorted by priority in the internal map.
-	 * 		Higher priority values take precedence and duplicates are dropped.
-	 *
-	 * @param	platform	Platform instance to register (moved)
-	 */
-	void register_platform(PlatformPtr&& platform);
-
 	/**
 	 * @brief	Get the highest priority platform instance
 	 *
 	 * 		Returns the platform with the highest priority value.
 	 *
 	 * @return	Reference to highest priority platform
 	 */
-	inline Platform& get_platform() { return *platforms_.rbegin()->second; }
+	inline Platform& get_platform() {
+		NCCL_OFI_INFO(NCCL_INIT | NCCL_NET, "Selected platform: %s",
+		              platform_->get_name());
+		return *platform_;
+	}
 
-	/**
-	 * @brief	Get number of registered platforms (for testing)
-	 *
-	 * @return	Number of platforms in the manager
-	 */
-	inline size_t get_platform_count() { return platforms_.size(); }
 protected:
 	/**
 	 * @brief	Default constructor
 	 *		Register the default Platform by default. A static global
 	 *		instance is meant to be used in the plugin and the unit
 	 *		tests leverage the protected scope.
 	 */
-	PlatformManager() {
-		register_platform(std::make_unique<Default>());
+	PlatformManager();
+
+	/**
+	 * @brief	Register a platform with the manager
+	 *
+	 * 		Platforms are selected by priority. Higher priority values take
+	 * 		precedence. This can only be done in the constructor as all platforms
+	 * 		must be added during object creation to allow the tuner and plugin
+	 * 		to operate consistently.
+	 *
+	 * @param	platform	Platform instance to register (moved)
+	 */
+	void register_platform(PlatformPtr&& platform) {
+		int priority = platform->get_priority();
+		// Replace if no current platform or higher priority
+		if (!platform_ || priority > current_priority_) {
+			platform_ = std::move(platform);
+			current_priority_ = priority;
+
+			// Set max priority if this matches the override so no other platform can override
+			if (!override_.empty() && override_ == platform_->get_name()) {
+				current_priority_ = std::numeric_limits<int>::max();
+			}
+		}
 	}
 
+
 private:
-	std::map<int, PlatformPtr> platforms_;
+	std::string override_ = "";
+	int current_priority_ = -1;
+	PlatformPtr platform_ = nullptr;
 };
 
 

include/nccl_ofi_topo.h

@@ -5,6 +5,8 @@
 #ifndef NCCL_NET_OFI_TOPO_H_
 #define NCCL_NET_OFI_TOPO_H_
 
+#include <memory>
+
 #include <hwloc.h>
 #include <rdma/fabric.h>
 
@@ -310,4 +312,49 @@ struct fi_info *nccl_ofi_topo_next_info_list(nccl_ofi_topo_data_iterator_t *iter
  */
 int nccl_ofi_topo_write_nccl_topology(nccl_ofi_topo_t *topo, FILE *file);
 
+class TopologyManager {
+private:
+        static inline std::unique_ptr<nccl_ofi_topo_t, decltype(&nccl_ofi_topo_free)> instance{
+                nullptr,
+                nccl_ofi_topo_free
+        };
+
+public:
+        static nccl_ofi_topo_t* get() {
+                return instance.get();
+        }
+
+        static nccl_ofi_topo_t* initialize(struct fi_info *provider_list) {
+                instance.reset(nccl_ofi_topo_create(provider_list));
+                if (instance && nccl_ofi_topo_group(instance.get()) != 0) {
+                        instance.reset();
+                }
+                return instance.get();
+        }
+
+        static void reset() {
+                instance.reset();
+        }
+
+        // TODO: Refactor all topology functions to be part of this class
+        static bool has_efa_ena_devices() {
+                auto* topo_instance = get();
+                if (!topo_instance) return false;
+                
+                hwloc_obj_t obj = nullptr;
+                while ((obj = hwloc_get_next_pcidev(topo_instance->topo, obj)) != nullptr) {
+                        // Check for Amazon vendor id and EFA device or ENA device
+                        if (obj->attr->pcidev.vendor_id == 0x1D0F &&
+                            (((obj->attr->pcidev.device_id & 0xFFF0) == 0xEFA0 || (obj->attr->pcidev.device_id & 0xFFF0) == 0xEC20) ||
+                             (obj->attr->pcidev.device_id & 0x0FFF) == 0x0EC2)) {
+                                return true;
+                        }
+                }
+                return false;
+        }
+
+        // Prevent instantiation
+        TopologyManager() = delete;
+};
+
 #endif // End NCCL_NET_OFI_TOPO_H_

include/platform-aws.h

@@ -23,7 +23,7 @@
 class PlatformAWS : public Platform {
 public:
 	const char* get_name() const override { return "AWS"; }
-	int get_priority() override { return 100; }
+	int get_priority() override { return TopologyManager::has_efa_ena_devices() ? 100 : -1; }
 	int init(const char **provider_filter) override;
 	int config_endpoint(struct fi_info *info, struct fid_ep *ep) override;
 	void sort_rails(struct fi_info **info_list, size_t num_rails, size_t num_groups) override;
@@ -72,6 +72,8 @@ class PlatformAWS : public Platform {
 		return fields ? fields->func_idx : -EIO;
 	}
 
+	// Determine if running on Amazon EC2 instance
+
 private:
 	std::mutex mutex_;
 

src/nccl_ofi_net.cpp

@@ -30,9 +30,6 @@
 #include "nccl_ofi_idpool.h"
 #include "nccl_ofi_dmabuf.h"
 #include "nccl_ofi_platform.h"
-#ifdef WANT_AWS_PLATFORM
-#include "platform-aws.h"
-#endif
 #include "nccl_ofi_ofiutils.h"
 #include "nccl_ofi_system.h"
 
@@ -178,10 +175,6 @@ int nccl_net_ofi_create_plugin(nccl_net_ofi_plugin_t **plugin_p)
 	nic_dup_conns = ofi_nccl_nic_dup_conns();
 	cq_read_count = ofi_nccl_cq_read_count();
 
-#ifdef WANT_AWS_PLATFORM
-	PlatformManager::get_global().register_platform(std::make_unique<PlatformAWS>());
-#endif
-
 	ret = PlatformManager::get_global().get_platform().init(&provider_filter);
 	if (ret != 0)
 		goto exit;

src/nccl_ofi_platform.cpp

@@ -3,24 +3,19 @@
  */
 
 #include "nccl_ofi_platform.h"
+#ifdef WANT_AWS_PLATFORM
+#include "platform-aws.h"
+#endif
+
+PlatformManager::PlatformManager()
+: override_(ofi_nccl_platform_override.get()) {
+	register_platform(std::make_unique<Default>());
+#ifdef WANT_AWS_PLATFORM
+	register_platform(std::make_unique<PlatformAWS>());
+#endif
+}
 
 PlatformManager& PlatformManager::get_global() {
 	static PlatformManager manager;
 	return manager;
 }
-
-void PlatformManager::register_platform(PlatformPtr&& platform) {
-	int priority = platform->get_priority();
-	const char* name = platform->get_name();
-
-	auto it = platforms_.find(priority);
-	if (it != platforms_.end()) {
-		if (strcmp(it->second->get_name(), name) == 0) {
-			return;
-		}
-		// TODO: Add proper resolution mechanism for competing priorities
-		priority++;
-	}
-
-	platforms_[priority] = std::move(platform);
-}

src/nccl_ofi_rdma.cpp

@@ -6934,10 +6934,8 @@ static void get_hints(struct fi_info *hints)
 
 nccl_net_ofi_rdma_plugin_t::~nccl_net_ofi_rdma_plugin_t()
 {
-	if (this->topo != nullptr) {
-		nccl_ofi_topo_free(this->topo);
-		this->topo = nullptr;
-	}
+	// TODO: Refactor so plugin holds reference to TopoploguManager object
+        this->topo = nullptr;
 
 	if (r_comm_cleanup_list != nullptr) {
 		delete r_comm_cleanup_list;
@@ -7013,17 +7011,11 @@ nccl_net_ofi_rdma_plugin_t::nccl_net_ofi_rdma_plugin_t(struct fi_info *provider_
 	int ret = 0;
 	int num_devices = 0;
 
-	/* Create NCCL OFI topology */
-	this->topo = nccl_ofi_topo_create(provider_list);
+	/* Use shared NCCL OFI topology */
+	this->topo = TopologyManager::get();
 	if (!this->topo) {
-		NCCL_OFI_WARN("Failed to create NCCL OFI topology");
-		throw std::runtime_error("rdma plugin constructor: topo creation failed");
-	}
-
-	ret = nccl_ofi_topo_group(this->topo);
-	if (ret != 0) {
-		NCCL_OFI_WARN("Failed to group NICs");
-		throw std::runtime_error("rdma plugin constructor: NIC grouping failed");
+		NCCL_OFI_WARN("Failed to get NCCL OFI topology");
+		throw std::runtime_error("rdma plugin constructor: topo access failed");
 	}
 
 	if (this->topo->max_group_size < 1 || this->topo->max_group_size > MAX_NUM_RAILS) {
@@ -7091,6 +7083,10 @@ int nccl_net_ofi_rdma_init(const char *provider_filter,
 			       FI_MAJOR(api_version),
 			       FI_MINOR(api_version),
 			       FI_VERSION_GE(api_version, FI_VERSION(1, 20)) ? "DMA-BUF" : "GPUDirect RDMA");
+
+		/* Initialize topology manager with provider list */
+		TopologyManager::initialize(provider_list);
+
 		/* The 1.18 API allows providers to use CUDA to
 		 * support HMEM pointers, so just having HMEM doesn't
 		 * tell us anything about the usability of CUDA

src/tuner/nccl_ofi_tuner.cpp

@@ -19,6 +19,7 @@
 #include "nccl_ofi_pthread.h"
 #include "nccl_ofi_system.h"
 #include "nccl_ofi_param.h"
+#include "nccl_ofi_platform.h"
 
 #include "tuner/nccl_ofi_tuner_region.h"
 #include "tuner/nccl_ofi_tuner_model.h"
@@ -63,7 +64,17 @@ static ncclResult_t nccl_ofi_tuner_init(size_t nRanks, size_t nNodes, ncclDebugL
 	 * Retrieve platform type and pass to Region and Model based tuner support check functions.
 	 * If both Region and Model based tuner are not supported, log a warning and exit.
 	 */
-	platform_type = nccl_net_ofi_get_product_name();
+	auto platform_name = PlatformManager::get_global().get_platform().get_name();
+	NCCL_OFI_INFO(NCCL_INIT | NCCL_TUNING, "Tuner init: Platform function returned: %s", platform_name);
+	if (strcmp(platform_name, "AWS") == 0) {
+		platform_type = nccl_net_ofi_get_product_name();
+	} else {
+		/* Default platform or other non-AWS platforms should use internal tuner */
+		NCCL_OFI_INFO(NCCL_INIT | NCCL_TUNING,
+		              "Non-AWS platform detected (%s), falling back to NCCL's internal tuner", platform_name);
+		goto exit;
+	}
+
 	if (platform_type == NULL) {
 		NCCL_OFI_WARN("NCCL_OFI_TUNER is not available because platform type is unavailable.");
 		goto exit;