Nested Repeating Groups

[deepakvashist-zh]

Context and Problem

Hi team,

I am encountering an issue when attempting to parse a TradeCaptureReport (MsgType=AE) message that contains a nested repeating group. The message parses correctly without a data dictionary but fails when the data dictionary is used, suggesting an issue with how the parser validates or iterates through nested groups.

The failure occurs specifically after I introduced the MiscFeesGrp inside the TrdCapRptSideGrp (TradeCaptureReport → TrdCapRptSideGrp → MiscFeesGrp).

Is this a known issue in the quickfix parser, or am I incorrectly structuring the message according to the FIX specification?

Example

The following Go test demonstrates the problem. The assertion for ParseMessageWithDataDictionary() fails, while the assertion for ParseMessage() (without DD) passes.

package main

import (
	"bytes"
	"testing"

	"github.com/quickfixgo/enum"
	"github.com/quickfixgo/field"
	"github.com/quickfixgo/fix50sp2/tradecapturereport"
	"github.com/quickfixgo/quickfix"
	"github.com/quickfixgo/quickfix/datadictionary"
	"github.com/shopspring/decimal"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestNestedRepeatingGroups(t *testing.T) {
	tcr := tradecapturereport.New(
		field.NewLastQty(decimal.RequireFromString("100"), 2),
		field.NewLastPx(decimal.RequireFromString("1"), 2),
	)

	tcr.SetTradeRequestID("request-id")
	tcr.SetTradeID("trade-id")
	tcr.SetTradeReportID("trade-report-id")
	tcr.SetFirmTradeID("firm-trade-id")
	tcr.SetSymbol("AAA/BBB")
	tcr.SetSecurityID("AAA/BBB")

	sides := tradecapturereport.NewNoSidesRepeatingGroup()

	sideBuy := sides.Add()
	sideBuy.SetSide(enum.Side_BUY)
	sideBuy.SetSideCurrency("AAA")
	sideBuy.SetSettlCurrAmt(decimal.RequireFromString("100"), 2)

	// Adding MiscFeesGrp to side buy
	fees := tradecapturereport.NewNoMiscFeesRepeatingGroup()
	fee1 := fees.Add()
	fee1.SetMiscFeeAmt(decimal.RequireFromString("0.05"), 2)
	fee1.SetMiscFeeCurr("BBB")
	fee1.SetMiscFeeType(enum.MiscFeeType_OTHER)
	fee2 := fees.Add()
	fee2.SetMiscFeeAmt(decimal.RequireFromString("0.07"), 2)
	fee2.SetMiscFeeCurr("BBB")
	fee2.SetMiscFeeType(enum.MiscFeeType_OTHER)
	sideBuy.SetNoMiscFees(fees)

	sideSell := sides.Add()
	sideSell.SetSide(enum.Side_SELL)
	sideSell.SetSideCurrency("BBB")
	sideSell.SetSettlCurrAmt(decimal.RequireFromString("100"), 2)

	tcr.SetNoSides(sides)

	// https://github.com/quickfixgo/quickfix/blob/main/spec/FIXT11.xml
	transportDict, err := datadictionary.Parse("spec/FIXT11.xml")
	require.NoError(t, err)
	// https://github.com/quickfixgo/quickfix/blob/main/spec/FIX50SP2.xml
	dataDict, err := datadictionary.Parse("spec/FIX50SP2.xml")
	require.NoError(t, err)

	// Getting the message from the struct
	bytes := bytes.NewBuffer([]byte(tcr.Message.ToMessage().String()))

	// Parsing the message USING the data dictionary
	msg1 := quickfix.NewMessage()
	err = quickfix.ParseMessageWithDataDictionary(msg1, bytes, transportDict, dataDict)
	require.NoError(t, err)

	// Parsing the message WITHOUT the data dictionary
	msg2 := quickfix.NewMessage()
	err = quickfix.ParseMessage(msg2, bytes)
	require.NoError(t, err)

	expectedTags := []quickfix.Tag{
		quickfix.Tag(568),  // Tag 568 (TradeRequestID)
		quickfix.Tag(1003), // Tag 1003 (TradeID)
		quickfix.Tag(571),  // Tag 571 (TradeReportID)
		quickfix.Tag(1041), // Tag 1041 (FirmTradeID)
		quickfix.Tag(55),   // Tag 55 (Symbol)
		quickfix.Tag(48),   // Tag 48 (SecurityID)
	}

	// Assertion fails
	assert.Subset(t, msg1.Body.Tags(), expectedTags)

	// Assertion passes
	assert.Subset(t, msg2.Body.Tags(), expectedTags)
}

Detailed Findings

1. Failure Condition: The parser fails to retain all expected tags when using quickfix.ParseMessageWithDataDictionary.
2. Trigger: The parsing breaks upon the inclusion of MiscFeesGrp within the side group.
3. Suspected Location: I put a breakpoint in the library and found that the parsing logic in the following section might be related to how repeating groups are handled, particularly when nested:

   quickfix/message.go

   Lines 307 to 372 in e4fe18c

   	// parseGroup iterates through a repeating group to maintain correct order of those fields.
   	func parseGroup(mp *msgParser, tags []Tag) {
   	mp.foundBody = true
   	dm := mp.msg.fields[mp.fieldIndex : mp.fieldIndex+1]
   	fields := getGroupFields(mp.msg, tags, mp.appDataDictionary)
   	for {
   	mp.fieldIndex++
   	mp.parsedFieldBytes = &mp.msg.fields[mp.fieldIndex]
   	mp.rawBytes, _ = extractField(mp.parsedFieldBytes, mp.rawBytes)
   	mp.trailerBytes = mp.rawBytes
   	// Is this field a member for the group.
   	if isGroupMember(mp.parsedFieldBytes.tag, fields) {
   	// Is this field a nested repeating group.
   	if isNumInGroupField(mp.msg, append(tags, mp.parsedFieldBytes.tag), mp.appDataDictionary) {
   	dm = append(dm, *mp.parsedFieldBytes)
   	tags = append(tags, mp.parsedFieldBytes.tag)
   	fields = getGroupFields(mp.msg, tags, mp.appDataDictionary)
   	continue
   	}
   	// Add the field member to the group.
   	dm = append(dm, *mp.parsedFieldBytes)
   	} else if isHeaderField(mp.parsedFieldBytes.tag, mp.transportDataDictionary) {
   	// Found a header tag for some reason..
   	mp.msg.Body.add(dm)
   	mp.msg.Header.add(mp.msg.fields[mp.fieldIndex : mp.fieldIndex+1])
   	break
   	} else if isTrailerField(mp.parsedFieldBytes.tag, mp.transportDataDictionary) {
   	// Found the trailer at the end of the message.
   	mp.msg.Body.add(dm)
   	mp.msg.Trailer.add(mp.msg.fields[mp.fieldIndex : mp.fieldIndex+1])
   	mp.foundTrailer = true
   	break
   	} else {
   	// Found a body field outside the group.
   	searchTags := []Tag{mp.parsedFieldBytes.tag}
   	// Is this a new group not inside the existing group.
   	if isNumInGroupField(mp.msg, searchTags, mp.appDataDictionary) {
   	// Add the current repeating group.
   	mp.msg.Body.add(dm)
   	// Cycle again with the new group.
   	dm = mp.msg.fields[mp.fieldIndex : mp.fieldIndex+1]
   	fields = getGroupFields(mp.msg, searchTags, mp.appDataDictionary)
   	continue
   	}
   	if len(tags) > 1 {
   	searchTags = tags[:len(tags)-1]
   	}
   	// Did this tag occur after a nested group and belongs to the parent group.
   	if isNumInGroupField(mp.msg, searchTags, mp.appDataDictionary) {
   	// Add the field member to the group.
   	dm = append(dm, *mp.parsedFieldBytes)
   	// Continue parsing the parent group.
   	fields = getGroupFields(mp.msg, searchTags, mp.appDataDictionary)
   	continue
   	}
   	// Add the repeating group.
   	mp.msg.Body.add(dm)
   	// Add the next body field.
   	mp.msg.Body.add(mp.msg.fields[mp.fieldIndex : mp.fieldIndex+1])
   	break
   	}
   	}
   	}

Thank you for your help in advance! Please let me know if you need any other details. 🙏