Skip to content

fix: fix reverse routing for KubeSpan #10074

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix: fix reverse routing for KubeSpan #10074

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .github/workflows/ci.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
#
# Generated on 2025-01-24T14:30:35Z by kres 3075de9.
# Generated on 2025-01-28T16:44:01Z by kres 987bf4d.

name: default
concurrency:
Expand Down Expand Up @@ -2070,6 +2070,7 @@ jobs:
IMAGE_REGISTRY: registry.dev.siderolabs.io\
SHORT_INTEGRATION_TEST: "yes"
WITH_CLUSTER_DISCOVERY: "true"
WITH_CONFIG_PATCH: '@hack/test/patches/kubespan-rp_filter.yaml'
WITH_KUBESPAN: "true"
run: |
sudo -E make e2e-qemu
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/integration-misc-1-cron.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
#
# Generated on 2024-12-24T15:00:58Z by kres fcff05e.
# Generated on 2025-01-15T12:49:27Z by kres 3b3f992.

name: integration-misc-1-cron
concurrency:
Expand Down Expand Up @@ -92,6 +92,7 @@ jobs:
IMAGE_REGISTRY: registry.dev.siderolabs.io\
SHORT_INTEGRATION_TEST: "yes"
WITH_CLUSTER_DISCOVERY: "true"
WITH_CONFIG_PATCH: '@hack/test/patches/kubespan-rp_filter.yaml'
WITH_KUBESPAN: "true"
run: |
sudo -E make e2e-qemu
Expand Down
1 change: 1 addition & 0 deletions .kres.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -754,6 +754,7 @@ spec:
SHORT_INTEGRATION_TEST: yes
WITH_CLUSTER_DISCOVERY: true
WITH_KUBESPAN: true
WITH_CONFIG_PATCH: "@hack/test/patches/kubespan-rp_filter.yaml"
IMAGE_REGISTRY: registry.dev.siderolabs.io\
- name: e2e-default-hostname
command: e2e-qemu
Expand Down
3 changes: 3 additions & 0 deletions hack/test/patches/kubespan-rp_filter.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
machine:
sysctls:
net.ipv4.conf.all.rp_filter: "1"
52 changes: 52 additions & 0 deletions internal/app/machined/pkg/controllers/kubespan/manager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,12 @@ import (

kubespanadapter "github.com/siderolabs/talos/internal/app/machined/pkg/adapters/kubespan"
"github.com/siderolabs/talos/pkg/machinery/constants"
"github.com/siderolabs/talos/pkg/machinery/kernel"
"github.com/siderolabs/talos/pkg/machinery/nethelpers"
"github.com/siderolabs/talos/pkg/machinery/resources/config"
"github.com/siderolabs/talos/pkg/machinery/resources/kubespan"
"github.com/siderolabs/talos/pkg/machinery/resources/network"
"github.com/siderolabs/talos/pkg/machinery/resources/runtime"
)

// DefaultPeerReconcileInterval is interval between peer status reconciliation on timer.
Expand Down Expand Up @@ -108,6 +110,10 @@ func (ctrl *ManagerController) Outputs() []controller.Output {
Type: kubespan.PeerStatusType,
Kind: controller.OutputExclusive,
},
{
Type: runtime.KernelParamSpecType,
Kind: controller.OutputShared,
},
}
}

Expand Down Expand Up @@ -371,13 +377,15 @@ func (ctrl *ManagerController) Run(ctx context.Context, r controller.Runtime, lo
spec.Policy = nethelpers.VerdictAccept

spec.Rules = []network.NfTablesRule{
// Accept outgoing WireGuard packets.
{
MatchMark: &network.NfTablesMark{
Mask: constants.KubeSpanDefaultFirewallMask,
Value: constants.KubeSpanDefaultFirewallMark,
},
Verdict: pointer.To(nethelpers.VerdictAccept),
},
// Mark packets to be sent over the KubeSpan link.
{
MatchDestinationAddress: &network.NfTablesAddressMatch{
IncludeSubnets: allowedIPsSet.Prefixes(),
Expand All @@ -388,6 +396,35 @@ func (ctrl *ManagerController) Run(ctx context.Context, r controller.Runtime, lo
},
Verdict: pointer.To(nethelpers.VerdictAccept),
},
// Remove KubeSpan mark from packets not sent to KubeSpan peers or received from them.
// This is typically the case when deencapsulated VXLAN packets retain envelope's fwmark, thus causing a routing loop.
{
MatchSourceAddress: &network.NfTablesAddressMatch{
Invert: true,
IncludeSubnets: allowedIPsSet.Prefixes(),
},
MatchMark: &network.NfTablesMark{
Mask: constants.KubeSpanDefaultForceFirewallMark,
Value: constants.KubeSpanDefaultForceFirewallMark,
},
SetMark: &network.NfTablesMark{
Mask: 0xffffffff,
Xor: constants.KubeSpanDefaultForceFirewallMark,
},
Verdict: pointer.To(nethelpers.VerdictAccept),
},
// Mark incoming packets from the KubeSpan link for rp_filter to find the correct routing table.
{
MatchIIfName: &network.NfTablesIfNameMatch{
InterfaceNames: []string{constants.KubeSpanLinkName},
Operator: nethelpers.OperatorEqual,
},
SetMark: &network.NfTablesMark{
Mask: ^uint32(constants.KubeSpanDefaultFirewallMask),
Xor: constants.KubeSpanDefaultForceFirewallMark,
},
Verdict: pointer.To(nethelpers.VerdictAccept),
},
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Full rule list in the modified chain for review:

        chain kubespan_prerouting { # handle 50
                type filter hook prerouting priority filter; policy accept;
                meta mark & 0x00000060 == 0x00000020 accept # handle 51
                ip daddr { 172.20.0.3-172.20.0.4 } meta mark set meta mark & 0xffffffdf | 0x00000040 accept # handle 53
                ip6 daddr { fdbd:f075:61fa:5502:2c26:e0ff:fe99:e2a4, fdbd:f075:61fa:5502:e88a:18ff:fed2:32f3 } meta mark set meta mark & 0xffffffdf | 0x00000040 accept # handle 55
                meta mark & 0x00000040 == 0x00000040 ip saddr != { 172.20.0.3-172.20.0.4 } meta mark set meta mark ^ 0x00000040 accept # handle 57
                meta mark & 0x00000040 == 0x00000040 ip6 saddr != { fdbd:f075:61fa:5502:2c26:e0ff:fe99:e2a4, fdbd:f075:61fa:5502:e88a:18ff:fed2:32f3 } meta mark set meta mark ^ 0x00000040 accept # handle 59
                iifname "kubespan" meta mark set meta mark & 0xffffffdf | 0x00000040 accept # handle 60
        }

        chain kubespan_prerouting { # handle 50
                type filter hook prerouting priority filter; policy accept;
                meta mark & 0x00000060 == 0x00000020 accept # handle 51
                ip daddr { 172.20.0.2, 172.20.0.4 } meta mark set meta mark & 0xffffffdf | 0x00000040 accept # handle 53
                ip6 daddr { fdbd:f075:61fa:5502:5cd0:eff:fea3:252f, fdbd:f075:61fa:5502:e88a:18ff:fed2:32f3 } meta mark set meta mark & 0xffffffdf | 0x00000040 accept # handle 55
                meta mark & 0x00000040 == 0x00000040 ip saddr != { 172.20.0.2, 172.20.0.4 } meta mark set meta mark ^ 0x00000040 accept # handle 57
                meta mark & 0x00000040 == 0x00000040 ip6 saddr != { fdbd:f075:61fa:5502:5cd0:eff:fea3:252f, fdbd:f075:61fa:5502:e88a:18ff:fed2:32f3 } meta mark set meta mark ^ 0x00000040 accept # handle 59
                iifname "kubespan" meta mark set meta mark & 0xffffffdf | 0x00000040 accept # handle 60
        }

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

okay, now with kube-router CNI (and coredns) doesn't come up

}

return nil
Expand Down Expand Up @@ -554,6 +591,17 @@ func (ctrl *ManagerController) Run(ctx context.Context, r controller.Runtime, lo
return fmt.Errorf("error modifying link spec: %w", err)
}

if err = safe.WriterModify(ctx, r, runtime.NewKernelParamSpec(
runtime.NamespaceName,
kernel.Sysctl+".net.ipv4.conf."+constants.KubeSpanLinkName+".src_valid_mark",
), func(res *runtime.KernelParamSpec) error {
res.TypedSpec().Value = "1"

return nil
}); err != nil {
return err
}

if rulesMgr == nil {
rulesMgr = ctrl.RulesManagerFactory(constants.KubeSpanDefaultRoutingTable, constants.KubeSpanDefaultForceFirewallMark, constants.KubeSpanDefaultFirewallMask)

Expand Down Expand Up @@ -591,6 +639,10 @@ func (ctrl *ManagerController) cleanup(ctx context.Context, r controller.Runtime
namespace: kubespan.NamespaceName,
typ: kubespan.PeerStatusType,
},
{
namespace: runtime.NamespaceName,
typ: runtime.KernelParamSpecType,
},
} {
// list keys for cleanup
list, err := r.List(ctx, resource.NewMetadata(item.namespace, item.typ, "", resource.VersionUndefined))
Expand Down
36 changes: 34 additions & 2 deletions internal/app/machined/pkg/controllers/kubespan/manager_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"github.com/siderolabs/talos/pkg/machinery/resources/config"
"github.com/siderolabs/talos/pkg/machinery/resources/kubespan"
"github.com/siderolabs/talos/pkg/machinery/resources/network"
"github.com/siderolabs/talos/pkg/machinery/resources/runtime"
)

type ManagerSuite struct {
Expand All @@ -40,6 +41,10 @@ func (suite *ManagerSuite) TestDisabled() {
suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg))

ctest.AssertNoResource[*network.NfTablesChain](suite, "kubespan_outgoing")
ctest.AssertNoResource[*runtime.KernelParamSpec](
suite,
"proc.sys.net.ipv4.conf.kubespan.src_valid_mark",
)
}

type mockWireguardClient struct {
Expand Down Expand Up @@ -244,9 +249,9 @@ func (suite *ManagerSuite) TestReconcile() {
asrt.Equal(nethelpers.ChainPriorityFilter, spec.Priority)
asrt.Equal(nethelpers.VerdictAccept, spec.Policy)

asrt.Len(spec.Rules, 2)
asrt.Len(spec.Rules, 3)

if len(spec.Rules) != 2 {
if len(spec.Rules) != 3 {
return
}

Expand Down Expand Up @@ -277,6 +282,21 @@ func (suite *ManagerSuite) TestReconcile() {
},
spec.Rules[1],
)

asrt.Equal(
network.NfTablesRule{
MatchIIfName: &network.NfTablesIfNameMatch{
InterfaceNames: []string{constants.KubeSpanLinkName},
Operator: nethelpers.OperatorEqual,
},
SetMark: &network.NfTablesMark{
Mask: ^uint32(constants.KubeSpanDefaultFirewallMask),
Xor: constants.KubeSpanDefaultForceFirewallMark,
},
Verdict: pointer.To(nethelpers.VerdictAccept),
},
spec.Rules[2],
)
},
)

Expand Down Expand Up @@ -358,6 +378,14 @@ func (suite *ManagerSuite) TestReconcile() {
},
)

ctest.AssertResource(
suite,
"proc.sys.net.ipv4.conf.kubespan.src_valid_mark",
func(res *runtime.KernelParamSpec, asrt *assert.Assertions) {
asrt.Equal(res.TypedSpec().Value, "1")
},
)

// update config and disable wireguard, everything should be cleaned up
cfg.TypedSpec().Enabled = false
suite.Require().NoError(suite.State().Update(suite.Ctx(), cfg))
Expand All @@ -371,6 +399,10 @@ func (suite *ManagerSuite) TestReconcile() {
suite,
"kubespan_prerouting",
)
ctest.AssertNoResource[*runtime.KernelParamSpec](
suite,
"proc.sys.net.ipv4.conf.kubespan.src_valid_mark",
)
}

func asUDP(addr netip.AddrPort) *net.UDPAddr {
Expand Down
Loading