fix(gateway): prevent blocked CIDs in CAR responses

docs/examples/kubo-as-a-library/go.mod

@@ -7,7 +7,7 @@ go 1.25
 replace github.com/ipfs/kubo => ./../../..
 
 require (
-	github.com/ipfs/boxo v0.34.0
+	github.com/ipfs/boxo v0.34.1-0.20250901000720-b14fd60c243d
 	github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
 	github.com/libp2p/go-libp2p v0.43.0
 	github.com/multiformats/go-multiaddr v0.16.1
@@ -69,7 +69,7 @@ require (
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/huin/goupnp v1.3.0 // indirect
 	github.com/ipfs-shipyard/nopfs v0.0.14 // indirect
-	github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 // indirect
+	github.com/ipfs-shipyard/nopfs/ipfs v0.25.1-0.20250901002929-8d2b62fbab75 // indirect
 	github.com/ipfs/bbloom v0.0.4 // indirect
 	github.com/ipfs/go-bitfield v1.1.0 // indirect
 	github.com/ipfs/go-block-format v0.2.2 // indirect

docs/examples/kubo-as-a-library/go.sum

@@ -283,12 +283,12 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/ipfs-shipyard/nopfs v0.0.14 h1:HFepJt/MxhZ3/GsLZkkAPzIPdNYKaLO1Qb7YmPbWIKk=
 github.com/ipfs-shipyard/nopfs v0.0.14/go.mod h1:mQyd0BElYI2gB/kq/Oue97obP4B3os4eBmgfPZ+hnrE=
-github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcdHUd7SDsUOY=
-github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU=
+github.com/ipfs-shipyard/nopfs/ipfs v0.25.1-0.20250901002929-8d2b62fbab75 h1:FKrvBGGZKdn/zP0Uw6RHOYBu21qu9hXbXu9uMAp8EtM=
+github.com/ipfs-shipyard/nopfs/ipfs v0.25.1-0.20250901002929-8d2b62fbab75/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU=
 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
-github.com/ipfs/boxo v0.34.0 h1:pMP9bAsTs4xVh8R0ZmxIWviV7kjDa60U24QrlGgHb1g=
-github.com/ipfs/boxo v0.34.0/go.mod h1:kzdH/ewDybtO3+M8MCVkpwnIIc/d2VISX95DFrY4vQA=
+github.com/ipfs/boxo v0.34.1-0.20250901000720-b14fd60c243d h1:mAFUAauRQAxaj9QRk/SL13fpUA45oyjsmxfG1MOSvgY=
+github.com/ipfs/boxo v0.34.1-0.20250901000720-b14fd60c243d/go.mod h1:rXql6ncaLZZfLqDG3Cuw9ZYQKd3rMU5bk1TGXF0+ZL0=
 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
 github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk=

go.mod

@@ -21,8 +21,8 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/hashicorp/go-version v1.7.0
 	github.com/ipfs-shipyard/nopfs v0.0.14
-	github.com/ipfs-shipyard/nopfs/ipfs v0.25.0
-	github.com/ipfs/boxo v0.34.0
+	github.com/ipfs-shipyard/nopfs/ipfs v0.25.1-0.20250901002929-8d2b62fbab75
+	github.com/ipfs/boxo v0.34.1-0.20250901000720-b14fd60c243d
 	github.com/ipfs/go-block-format v0.2.2
 	github.com/ipfs/go-cid v0.5.0
 	github.com/ipfs/go-cidutil v0.1.0

go.sum

@@ -350,12 +350,12 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/ipfs-shipyard/nopfs v0.0.14 h1:HFepJt/MxhZ3/GsLZkkAPzIPdNYKaLO1Qb7YmPbWIKk=
 github.com/ipfs-shipyard/nopfs v0.0.14/go.mod h1:mQyd0BElYI2gB/kq/Oue97obP4B3os4eBmgfPZ+hnrE=
-github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcdHUd7SDsUOY=
-github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU=
+github.com/ipfs-shipyard/nopfs/ipfs v0.25.1-0.20250901002929-8d2b62fbab75 h1:FKrvBGGZKdn/zP0Uw6RHOYBu21qu9hXbXu9uMAp8EtM=
+github.com/ipfs-shipyard/nopfs/ipfs v0.25.1-0.20250901002929-8d2b62fbab75/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU=
 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
-github.com/ipfs/boxo v0.34.0 h1:pMP9bAsTs4xVh8R0ZmxIWviV7kjDa60U24QrlGgHb1g=
-github.com/ipfs/boxo v0.34.0/go.mod h1:kzdH/ewDybtO3+M8MCVkpwnIIc/d2VISX95DFrY4vQA=
+github.com/ipfs/boxo v0.34.1-0.20250901000720-b14fd60c243d h1:mAFUAauRQAxaj9QRk/SL13fpUA45oyjsmxfG1MOSvgY=
+github.com/ipfs/boxo v0.34.1-0.20250901000720-b14fd60c243d/go.mod h1:rXql6ncaLZZfLqDG3Cuw9ZYQKd3rMU5bk1TGXF0+ZL0=
 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
 github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk=

test/cli/content_blocking_test.go

@@ -37,8 +37,21 @@ func TestContentBlocking(t *testing.T) {
 
 	// Create CIDs we use in test
 	h.WriteFile("parent-dir/blocked-subdir/indirectly-blocked-file.txt", "indirectly blocked file content")
+	h.WriteFile("parent-dir/safe-subdir/safe-file.txt", "safe file content")
 	allowedParentDirCID := node.IPFS("add", "--raw-leaves", "-Q", "-r", "--pin=false", filepath.Join(h.Dir, "parent-dir")).Stdout.Trimmed()
-	blockedSubDirCID := node.IPFS("add", "--raw-leaves", "-Q", "-r", "--pin=false", filepath.Join(h.Dir, "parent-dir", "blocked-subdir")).Stdout.Trimmed()
+
+	// Get the CID of subdirectories as they exist within the parent DAG
+	// Note: These CIDs are different from adding the directories standalone
+	safeSubDirCID := node.IPFS("resolve", "-r", "/ipfs/"+allowedParentDirCID+"/safe-subdir").Stdout.Trimmed()
+	safeSubDirCID = strings.TrimPrefix(safeSubDirCID, "/ipfs/")
+	blockedSubDirCID := node.IPFS("resolve", "-r", "/ipfs/"+allowedParentDirCID+"/blocked-subdir").Stdout.Trimmed()
+	blockedSubDirCID = strings.TrimPrefix(blockedSubDirCID, "/ipfs/")
+
+	// Get the CID of the safe file within the safe subdirectory
+	safeFileCID := node.IPFS("resolve", "-r", "/ipfs/"+allowedParentDirCID+"/safe-subdir/safe-file.txt").Stdout.Trimmed()
+	safeFileCID = strings.TrimPrefix(safeFileCID, "/ipfs/")
+
+	// Remove the blocked subdirectory from blockstore
 	node.IPFS("block", "rm", blockedSubDirCID)
 
 	h.WriteFile("directly-blocked-file.txt", "directly blocked file content")
@@ -102,45 +115,88 @@ func TestContentBlocking(t *testing.T) {
 		assert.Equal(t, http.StatusOK, resp.StatusCode)
 	})
 
-	// Confirm CAR responses skip blocked subpaths
-	t.Run("Gateway returns CAR without blocked subpath", func(t *testing.T) {
-		resp := client.Get("/ipfs/" + allowedParentDirCID + "/subdir?format=car")
+	// Verify that when requesting a subdirectory as CAR, the response includes
+	// the safe content even when a sibling directory is blocked
+	t.Run("Gateway returns 200 with CAR for safe subdir when sibling is blocked", func(t *testing.T) {
+		// Request the SAFE subdirectory, verifying it's accessible even though a sibling is blocked
+		resp := client.Get("/ipfs/" + allowedParentDirCID + "/safe-subdir?format=car")
 		assert.Equal(t, http.StatusOK, resp.StatusCode)
 
 		bs, err := carstore.NewReadOnly(strings.NewReader(resp.Body), nil)
 		assert.NoError(t, err)
 
-		has, err := bs.Has(context.Background(), cid.MustParse(blockedSubDirCID))
+		roots, err := bs.Roots()
+		assert.NoError(t, err)
+		assert.Equal(t, 1, len(roots))
+		assert.Equal(t, safeSubDirCID, roots[0].String())
+
+		// Verify the safe content IS in the CAR
+		ctx := context.TODO()
+		has, err := bs.Has(ctx, cid.MustParse(safeSubDirCID))
 		assert.NoError(t, err)
-		assert.False(t, has)
+		assert.True(t, has, "CAR should include the safe subdirectory")
+
+		// Verify the safe file within the subdirectory IS in the CAR
+		has, err = bs.Has(ctx, cid.MustParse(safeFileCID))
+		assert.NoError(t, err)
+		assert.True(t, has, "CAR should include the safe file within the subdirectory")
+
+		// Verify the blocked content is NOT in the CAR (it shouldn't be traversed)
+		has, err = bs.Has(ctx, cid.MustParse(blockedSubDirCID))
+		assert.NoError(t, err)
+		assert.False(t, has, "CAR should not include the blocked subdirectory")
 	})
 
-	/* TODO: this was already broken in 0.26, but we should fix it
-	t.Run("Gateway returns CAR without directly blocked CID", func(t *testing.T) {
+	// Test that requesting non-existent path with CAR format returns 404 (not 410)
+	// This ensures we distinguish between blocked content and missing content
+	t.Run("Gateway returns 404 for non-existent path with CAR format", func(t *testing.T) {
+		// Request a non-existent subdirectory as CAR
+		resp := client.Get("/ipfs/" + allowedParentDirCID + "/safe-404?format=car")
+		assert.Equal(t, http.StatusNotFound, resp.StatusCode, "Non-existent path should return 404 Not Found")
+
+		// Verify response body is not a CAR file but an error message
+		// CAR files start with specific magic bytes
+		assert.NotContains(t, resp.Body, "CAR", "404 response should not contain CAR data")
+		// Verify it contains an appropriate error message
+		assert.Contains(t, resp.Body, "no link named", "404 response should contain IPFS path resolution error")
+	})
+
+	// Test that confirms blocking of children skips them from produced CAR.
+	t.Run("Gateway returns 200 with CAR without directly blocked CID", func(t *testing.T) {
+		// First verify that the blocked CID is actually blocked when accessed directly
+		directResp := client.Get("/ipfs/" + blockedCID)
+		assert.Equal(t, http.StatusGone, directResp.StatusCode, "Direct access to blocked CID should return 410")
+
 		allowedDirWithDirectlyBlockedCID := node.IPFS("add", "--raw-leaves", "-Q", "-rw", filepath.Join(h.Dir, "directly-blocked-file.txt")).Stdout.Trimmed()
+		t.Logf("Directory CID containing blocked file: %s", allowedDirWithDirectlyBlockedCID)
+		t.Logf("Blocked CID that should not appear in CAR: %s", blockedCID)
 		resp := client.Get("/ipfs/" + allowedDirWithDirectlyBlockedCID + "?format=car")
+
+		// We expect HTTP 200 because root cid is not blocked, so we start
+		// streaming directory recursively
 		assert.Equal(t, http.StatusOK, resp.StatusCode)
 
 		bs, err := carstore.NewReadOnly(strings.NewReader(resp.Body), nil)
 		assert.NoError(t, err)
 
+		// The blocked CID MUST be omitted from HTTP response
 		has, err := bs.Has(context.Background(), cid.MustParse(blockedCID))
 		assert.NoError(t, err)
 		assert.False(t, has, "Returned CAR should not include blockedCID")
 	})
-	*/
-
-	// Confirm CAR responses skip blocked subpaths
-	t.Run("Gateway returns CAR without blocked subpath", func(t *testing.T) {
-		resp := client.Get("/ipfs/" + allowedParentDirCID + "/subdir?format=car")
-		assert.Equal(t, http.StatusOK, resp.StatusCode)
 
-		bs, err := carstore.NewReadOnly(strings.NewReader(resp.Body), nil)
-		assert.NoError(t, err)
+	// Test that requesting a CAR with a blocked root CID returns 410
+	t.Run("Gateway returns 410 for CAR request with blocked root CID", func(t *testing.T) {
+		resp := client.Get("/ipfs/" + blockedCID + "?format=car")
+		assert.Equal(t, http.StatusGone, resp.StatusCode, "CAR request for blocked root CID should return 410 Gone")
+		assert.Contains(t, resp.Body, blockedMsg, "Error message should indicate content is blocked")
+	})
 
-		has, err := bs.Has(context.Background(), cid.MustParse(blockedSubDirCID))
-		assert.NoError(t, err)
-		assert.False(t, has, "Returned CAR should not include blockedSubDirCID")
+	// Test that requesting raw format with a blocked root CID returns 410
+	t.Run("Gateway returns 410 for raw request with blocked root CID", func(t *testing.T) {
+		resp := client.Get("/ipfs/" + blockedCID + "?format=raw")
+		assert.Equal(t, http.StatusGone, resp.StatusCode, "Raw request for blocked root CID should return 410 Gone")
+		assert.Contains(t, resp.Body, blockedMsg, "Error message should indicate content is blocked")
 	})
 
 	// Ok, now the full list of test cases we want to cover in both CLI and Gateway

test/dependencies/go.mod

@@ -134,7 +134,7 @@ require (
 	github.com/huin/goupnp v1.3.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/ipfs/bbloom v0.0.4 // indirect
-	github.com/ipfs/boxo v0.34.0 // indirect
+	github.com/ipfs/boxo v0.34.1-0.20250901000720-b14fd60c243d // indirect
 	github.com/ipfs/go-bitfield v1.1.0 // indirect
 	github.com/ipfs/go-block-format v0.2.2 // indirect
 	github.com/ipfs/go-cid v0.5.0 // indirect

test/dependencies/go.sum

@@ -332,8 +332,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
-github.com/ipfs/boxo v0.34.0 h1:pMP9bAsTs4xVh8R0ZmxIWviV7kjDa60U24QrlGgHb1g=
-github.com/ipfs/boxo v0.34.0/go.mod h1:kzdH/ewDybtO3+M8MCVkpwnIIc/d2VISX95DFrY4vQA=
+github.com/ipfs/boxo v0.34.1-0.20250901000720-b14fd60c243d h1:mAFUAauRQAxaj9QRk/SL13fpUA45oyjsmxfG1MOSvgY=
+github.com/ipfs/boxo v0.34.1-0.20250901000720-b14fd60c243d/go.mod h1:rXql6ncaLZZfLqDG3Cuw9ZYQKd3rMU5bk1TGXF0+ZL0=
 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
 github.com/ipfs/go-block-format v0.2.2 h1:uecCTgRwDIXyZPgYspaLXoMiMmxQpSx2aq34eNc4YvQ=