support fan out for backfill read and fix more

yl-lisen · yl-lisen · commit 92008436a22f · 2023-11-23T17:05:23.000+08:00
diff --git a/src/Storages/Streaming/StorageStream.cpp b/src/Storages/Streaming/StorageStream.cpp
@@ -486,15 +486,19 @@ void StorageStream::readConcat(
     for (auto & stream_shard : shards_to_read)
     {
         auto create_streaming_source = [this, header, storage_snapshot, stream_shard, seek_to_info = query_info.seek_to_info, context_](
-                                           Int64 & max_sn_in_parts) {
+                                           Int64 & max_sn_in_parts) -> SourcePtr {
             if (max_sn_in_parts < 0)
             {
                 /// Fallback to seek streaming store
                 auto offsets = stream_shard->getOffsets(seek_to_info);
                 LOG_INFO(log, "Fused read fallbacks to seek stream for shard={} since there are no historical data", stream_shard->shard);
 
-                return std::make_shared<StreamingStoreSource>(
-                    stream_shard, header, storage_snapshot, context_, offsets[stream_shard->shard], log);
+                if (context_->getSettingsRef().query_resource_group.value == "shared")
+                    return source_multiplexers->createChannel(
+                        stream_shard, header.getNames(), storage_snapshot, context_, offsets[stream_shard->shard]);
+                else
+                    return std::make_shared<StreamingStoreSource>(
+                        stream_shard, header, storage_snapshot, context_, offsets[stream_shard->shard], log);
             }
 
             auto committed = stream_shard->storage->inMemoryCommittedSN();
@@ -526,7 +530,12 @@ void StorageStream::readConcat(
                     max_sn_in_parts,
                     committed);
 
-                return std::make_shared<StreamingStoreSource>(stream_shard, header, storage_snapshot, context_, max_sn_in_parts + 1, log);
+                if (context_->getSettingsRef().query_resource_group.value == "shared")
+                    return source_multiplexers->createChannel(
+                        stream_shard, header.getNames(), storage_snapshot, context_, max_sn_in_parts + 1);
+                else
+                    return std::make_shared<StreamingStoreSource>(
+                        stream_shard, header, storage_snapshot, context_, max_sn_in_parts + 1, log);
             }
             else
             {
@@ -542,8 +551,12 @@ void StorageStream::readConcat(
 
                 /// We need reset max_sn_in_parts to tell caller that we are seeking streaming store directly
                 max_sn_in_parts = -1;
-                return std::make_shared<StreamingStoreSource>(
-                    stream_shard, header, storage_snapshot, context_, offsets[stream_shard->shard], log);
+                if (context_->getSettingsRef().query_resource_group.value == "shared")
+                    return source_multiplexers->createChannel(
+                        stream_shard, header.getNames(), storage_snapshot, context_, offsets[stream_shard->shard]);
+                else
+                    return std::make_shared<StreamingStoreSource>(
+                        stream_shard, header, storage_snapshot, context_, offsets[stream_shard->shard], log);
             }
         };
 
diff --git a/src/Storages/Streaming/StreamingStoreSourceMultiplexer.cpp b/src/Storages/Streaming/StreamingStoreSourceMultiplexer.cpp
@@ -280,6 +280,8 @@ std::pair<String, Int32> StreamingStoreSourceMultiplexer::getStreamShard() const
     return stream_shard->getStreamShard();
 }
 
+std::atomic<uint32_t> StreamingStoreSourceMultiplexers::multiplexer_id = 0;
+
 StreamingStoreSourceMultiplexers::StreamingStoreSourceMultiplexers(ContextPtr global_context_, Poco::Logger * log_)
     : global_context(std::move(global_context_)), log(log_)
 {
@@ -310,7 +312,8 @@ StreamingStoreSourceChannelPtr StreamingStoreSourceMultiplexers::createChannel(
     {
         multiplexers.emplace(
             shard,
-            StreamingStoreSourceMultiplexerPtrs{std::make_shared<StreamingStoreSourceMultiplexer>(0, stream_shard, global_context, log)});
+            StreamingStoreSourceMultiplexerPtrs{
+                std::make_shared<StreamingStoreSourceMultiplexer>(getMultiplexerID(), stream_shard, global_context, log)});
         iter = multiplexers.find(shard);
     }
 
@@ -342,7 +345,7 @@ StreamingStoreSourceChannelPtr StreamingStoreSourceMultiplexers::createChannel(
         /// If min channels is greater than > 20(default value), create another multiplexer for this shard
         if (min_channels > global_context->getSettingsRef().max_channels_per_resource_group.value)
         {
-            best_multiplexer = std::make_shared<StreamingStoreSourceMultiplexer>(iter->second.size(), stream_shard, global_context, log);
+            best_multiplexer = std::make_shared<StreamingStoreSourceMultiplexer>(getMultiplexerID(), stream_shard, global_context, log);
             iter->second.push_back(best_multiplexer);
         }
 
@@ -351,7 +354,7 @@ StreamingStoreSourceChannelPtr StreamingStoreSourceMultiplexers::createChannel(
     else
     {
         /// All multiplexers are shutdown
-        auto multiplexer{std::make_shared<StreamingStoreSourceMultiplexer>(iter->second.size(), stream_shard, global_context, log)};
+        auto multiplexer{std::make_shared<StreamingStoreSourceMultiplexer>(getMultiplexerID(), stream_shard, global_context, log)};
         iter->second.push_back(multiplexer);
         return multiplexer->createChannel(column_names, storage_snapshot, query_context);
     }
@@ -366,7 +369,7 @@ StreamingStoreSourceChannelPtr StreamingStoreSourceMultiplexers::createIndepende
     /// will startup after `StreamingStoreSourceChannel::recover()` and reset recovered sn
     /// The `multiplexer` is cached in created StreamingStoreSourceChannel, we can release this one
     auto multiplexer = std::make_shared<StreamingStoreSourceMultiplexer>(
-        0, std::move(stream_shard), global_context, log, [this](auto multiplexer_) { attachToSharedGroup(multiplexer_); });
+        getMultiplexerID(), std::move(stream_shard), global_context, log, [this](auto multiplexer_) { attachToSharedGroup(multiplexer_); });
     return multiplexer->createChannel(column_names, storage_snapshot, query_context);
 }
 
@@ -379,7 +382,7 @@ StreamingStoreSourceChannelPtr StreamingStoreSourceMultiplexers::createIndepende
 {
     /// The `multiplexer` is cached in created StreamingStoreSourceChannel, we can release this one
     auto multiplexer = std::make_shared<StreamingStoreSourceMultiplexer>(
-        0, std::move(stream_shard), global_context, log, [this](auto multiplexer_) { attachToSharedGroup(multiplexer_); });
+        getMultiplexerID(), std::move(stream_shard), global_context, log, [this](auto multiplexer_) { attachToSharedGroup(multiplexer_); });
     auto channel = multiplexer->createChannel(column_names, storage_snapshot, query_context);
     multiplexer->resetSequenceNumber(start_sn);
     multiplexer->startup();
@@ -394,8 +397,17 @@ void StreamingStoreSourceMultiplexers::attachToSharedGroup(StreamingStoreSourceM
     detached_multiplexers.clear();
 
     auto & multiplexer_list = multiplexers[multiplexer->stream_shard->getShard()];
-    for (auto & shared_multiplexer : multiplexer_list)
+    for (auto it = multiplexer_list.begin(); it != multiplexer_list.end();)
     {
+        if ((*it)->isShutdown())
+        {
+            it = multiplexer_list.erase(it);
+            continue;
+        }
+
+        auto & shared_multiplexer = *it;
+        ++it;
+
         /// Skip multiplexer that already have too many channels
         if (shared_multiplexer->totalChannels() > global_context->getSettingsRef().max_channels_per_resource_group.value)
             continue;
@@ -410,7 +422,6 @@ void StreamingStoreSourceMultiplexers::attachToSharedGroup(StreamingStoreSourceM
     }
 
     /// Not detach channels into any existed shared multiplexer, so we reuse it and join in shared groups
-    multiplexer->id = multiplexer_list.size(); /// it's thread safe
     multiplexer_list.emplace_back(std::move(multiplexer));
 }
 }
diff --git a/src/Storages/Streaming/StreamingStoreSourceMultiplexer.h b/src/Storages/Streaming/StreamingStoreSourceMultiplexer.h
@@ -122,10 +122,14 @@ class StreamingStoreSourceMultiplexers final
 
     void attachToSharedGroup(StreamingStoreSourceMultiplexerPtr multiplexer);
 
+    uint32_t getMultiplexerID() { return multiplexer_id.fetch_add(1); }
+
 private:
     ContextPtr global_context;
     Poco::Logger * log;
 
+    static std::atomic<uint32_t> multiplexer_id;
+
     std::mutex multiplexers_mutex;
     std::unordered_map<Int32, StreamingStoreSourceMultiplexerPtrs> multiplexers;
     StreamingStoreSourceMultiplexerPtrs detached_multiplexers;

Original file line number	Diff line number	Diff line change
`@@ -280,6 +280,8 @@ std::pair<String, Int32> StreamingStoreSourceMultiplexer::getStreamShard() const`
`280`	`280`	`return stream_shard->getStreamShard();`
`281`	`281`	`}`
`282`	`282`
	`283`	`+std::atomic<uint32_t> StreamingStoreSourceMultiplexers::multiplexer_id = 0;`
	`284`	`+`
`283`	`285`	`StreamingStoreSourceMultiplexers::StreamingStoreSourceMultiplexers(ContextPtr global_context_, Poco::Logger * log_)`
`284`	`286`	`: global_context(std::move(global_context_)), log(log_)`
`285`	`287`	`{`
`@@ -310,7 +312,8 @@ StreamingStoreSourceChannelPtr StreamingStoreSourceMultiplexers::createChannel(`
`310`	`312`	`{`
`311`	`313`	`multiplexers.emplace(`
`312`	`314`	`shard,`
`313`		`- StreamingStoreSourceMultiplexerPtrs{std::make_shared<StreamingStoreSourceMultiplexer>(0, stream_shard, global_context, log)});`
	`315`	`+ StreamingStoreSourceMultiplexerPtrs{`
	`316`	`+ std::make_shared<StreamingStoreSourceMultiplexer>(getMultiplexerID(), stream_shard, global_context, log)});`
`314`	`317`	`iter = multiplexers.find(shard);`
`315`	`318`	`}`
`316`	`319`
`@@ -342,7 +345,7 @@ StreamingStoreSourceChannelPtr StreamingStoreSourceMultiplexers::createChannel(`
`342`	`345`	`/// If min channels is greater than > 20(default value), create another multiplexer for this shard`
`343`	`346`	`if (min_channels > global_context->getSettingsRef().max_channels_per_resource_group.value)`
`344`	`347`	`{`
`345`		`- best_multiplexer = std::make_shared<StreamingStoreSourceMultiplexer>(iter->second.size(), stream_shard, global_context, log);`
	`348`	`+ best_multiplexer = std::make_shared<StreamingStoreSourceMultiplexer>(getMultiplexerID(), stream_shard, global_context, log);`
`346`	`349`	`iter->second.push_back(best_multiplexer);`
`347`	`350`	`}`
`348`	`351`
`@@ -351,7 +354,7 @@ StreamingStoreSourceChannelPtr StreamingStoreSourceMultiplexers::createChannel(`
`351`	`354`	`else`
`352`	`355`	`{`
`353`	`356`	`/// All multiplexers are shutdown`
`354`		`- auto multiplexer{std::make_shared<StreamingStoreSourceMultiplexer>(iter->second.size(), stream_shard, global_context, log)};`
	`357`	`+ auto multiplexer{std::make_shared<StreamingStoreSourceMultiplexer>(getMultiplexerID(), stream_shard, global_context, log)};`
`355`	`358`	`iter->second.push_back(multiplexer);`
`356`	`359`	`return multiplexer->createChannel(column_names, storage_snapshot, query_context);`
`357`	`360`	`}`
`@@ -366,7 +369,7 @@ StreamingStoreSourceChannelPtr StreamingStoreSourceMultiplexers::createIndepende`
`366`	`369`	/// will startup after `StreamingStoreSourceChannel::recover()` and reset recovered sn
`367`	`370`	/// The `multiplexer` is cached in created StreamingStoreSourceChannel, we can release this one
`368`	`371`	`auto multiplexer = std::make_shared<StreamingStoreSourceMultiplexer>(`
`369`		`- 0, std::move(stream_shard), global_context, log, [this](auto multiplexer_) { attachToSharedGroup(multiplexer_); });`
	`372`	`+ getMultiplexerID(), std::move(stream_shard), global_context, log, [this](auto multiplexer_) { attachToSharedGroup(multiplexer_); });`
`370`	`373`	`return multiplexer->createChannel(column_names, storage_snapshot, query_context);`
`371`	`374`	`}`
`372`	`375`
`@@ -379,7 +382,7 @@ StreamingStoreSourceChannelPtr StreamingStoreSourceMultiplexers::createIndepende`
`379`	`382`	`{`
`380`	`383`	/// The `multiplexer` is cached in created StreamingStoreSourceChannel, we can release this one
`381`	`384`	`auto multiplexer = std::make_shared<StreamingStoreSourceMultiplexer>(`
`382`		`- 0, std::move(stream_shard), global_context, log, [this](auto multiplexer_) { attachToSharedGroup(multiplexer_); });`
	`385`	`+ getMultiplexerID(), std::move(stream_shard), global_context, log, [this](auto multiplexer_) { attachToSharedGroup(multiplexer_); });`
`383`	`386`	`auto channel = multiplexer->createChannel(column_names, storage_snapshot, query_context);`
`384`	`387`	`multiplexer->resetSequenceNumber(start_sn);`
`385`	`388`	`multiplexer->startup();`
`@@ -394,8 +397,17 @@ void StreamingStoreSourceMultiplexers::attachToSharedGroup(StreamingStoreSourceM`
`394`	`397`	`detached_multiplexers.clear();`
`395`	`398`
`396`	`399`	`auto & multiplexer_list = multiplexers[multiplexer->stream_shard->getShard()];`
`397`		`- for (auto & shared_multiplexer : multiplexer_list)`
	`400`	`+ for (auto it = multiplexer_list.begin(); it != multiplexer_list.end();)`
`398`	`401`	`{`
	`402`	`+ if ((*it)->isShutdown())`
	`403`	`+ {`
	`404`	`+ it = multiplexer_list.erase(it);`
	`405`	`+ continue;`
	`406`	`+ }`
	`407`	`+`
	`408`	`+ auto & shared_multiplexer = *it;`
	`409`	`+ ++it;`
	`410`	`+`
`399`	`411`	`/// Skip multiplexer that already have too many channels`
`400`	`412`	`if (shared_multiplexer->totalChannels() > global_context->getSettingsRef().max_channels_per_resource_group.value)`
`401`	`413`	`continue;`
`@@ -410,7 +422,6 @@ void StreamingStoreSourceMultiplexers::attachToSharedGroup(StreamingStoreSourceM`
`410`	`422`	`}`
`411`	`423`
`412`	`424`	`/// Not detach channels into any existed shared multiplexer, so we reuse it and join in shared groups`
`413`		`- multiplexer->id = multiplexer_list.size(); /// it's thread safe`
`414`	`425`	`multiplexer_list.emplace_back(std::move(multiplexer));`
`415`	`426`	`}`
`416`	`427`	`}`