Better roundtriping of HCL

Author: mikefarah

agents.md

@@ -135,6 +135,7 @@ Key methods:
 - Use the candidate_node style attribute to store style information for round-trip. Ask if this needs to be updated with new styles.
 - Use build tags for optional compilation
 - Add comprehensive tests
+- Run the specific encoder/decoder test (e.g. <format>_test.go) whenever you make ay changes to the encoder_<format> or decoder_<format>
 - Handle errors gracefully
 - Add the no build directive, like the xml encoder and decoder, that enables a minimal yq builds. e.g.  `//go:build !yq_<format>`. Be sure to also update the build_small-yq.sh and build-tinygo-yq.sh to not include the new format.
 

pkg/yqlib/candidate_node.go

@@ -97,6 +97,9 @@ type CandidateNode struct {
 	// (e.g. top level cross document merge). This property does not propagate to child nodes.
 	EvaluateTogether bool
 	IsMapKey         bool
+	// For formats like HCL and TOML: indicates that child entries should be emitted as separate blocks/tables
+	// rather than consolidated into nested mappings (default behaviour)
+	EncodeSeparate bool
 }
 
 func (n *CandidateNode) CreateChild() *CandidateNode {
@@ -407,6 +410,8 @@ func (n *CandidateNode) doCopy(cloneContent bool) *CandidateNode {
 
 		EvaluateTogether: n.EvaluateTogether,
 		IsMapKey:         n.IsMapKey,
+
+		EncodeSeparate: n.EncodeSeparate,
 	}
 
 	if cloneContent {

pkg/yqlib/decoder_hcl.go

@@ -161,8 +161,14 @@ func (dec *hclDecoder) Decode() (*CandidateNode, error) {
 	}
 
 	// process blocks
+	// Count blocks by type at THIS level to detect multiple separate blocks
+	blocksByType := make(map[string]int)
 	for _, block := range body.Blocks {
-		addBlockToMapping(root, block, dec.fileBytes)
+		blocksByType[block.Type]++
+	}
+
+	for _, block := range body.Blocks {
+		addBlockToMapping(root, block, dec.fileBytes, blocksByType[block.Type] > 1)
 	}
 
 	dec.documentIndex++
@@ -187,14 +193,23 @@ func hclBodyToNode(body *hclsyntax.Body, src []byte) *CandidateNode {
 
 		node.AddKeyValueChild(key, val)
 	}
+
+	// Process nested blocks, counting blocks by type at THIS level
+	// to detect which block types appear multiple times
+	blocksByType := make(map[string]int)
+	for _, block := range body.Blocks {
+		blocksByType[block.Type]++
+	}
+
 	for _, block := range body.Blocks {
-		addBlockToMapping(node, block, src)
+		addBlockToMapping(node, block, src, blocksByType[block.Type] > 1)
 	}
 	return node
 }
 
 // addBlockToMapping nests block type and labels into the parent mapping, merging children.
-func addBlockToMapping(parent *CandidateNode, block *hclsyntax.Block, src []byte) {
+// isMultipleBlocksOfType indicates if there are multiple blocks of this type at THIS level
+func addBlockToMapping(parent *CandidateNode, block *hclsyntax.Block, src []byte, isMultipleBlocksOfType bool) {
 	bodyNode := hclBodyToNode(block.Body, src)
 	current := parent
 
@@ -208,6 +223,11 @@ func addBlockToMapping(parent *CandidateNode, block *hclsyntax.Block, src []byte
 	}
 	if typeNode == nil {
 		_, typeNode = current.AddKeyValueChild(createStringScalarNode(block.Type), &CandidateNode{Kind: MappingNode})
+		// Mark the type node if there are multiple blocks of this type at this level
+		// This tells the encoder to emit them as separate blocks rather than consolidating them
+		if isMultipleBlocksOfType {
+			typeNode.EncodeSeparate = true
+		}
 	}
 	current = typeNode
 

pkg/yqlib/doc/usage/hcl.md

@@ -175,3 +175,27 @@ message = "Hello, ${name}!"
 shouty_message = upper(message)
 ```
 
+## Roundtrip: Separate blocks with same name.
+Given a sample.hcl file of:
+```hcl
+resource "aws_instance" "web" {
+  ami = "ami-12345"
+}
+resource "aws_instance" "db" {
+  ami = "ami-67890"
+}
+```
+then
+```bash
+yq sample.hcl
+```
+will output
+```hcl
+resource "aws_instance" "web" {
+  ami = "ami-12345"
+}
+resource "aws_instance" "db" {
+  ami = "ami-67890"
+}
+```
+

pkg/yqlib/encoder_hcl.go

@@ -438,6 +438,13 @@ func (he *hclEncoder) encodeBlockIfMapping(body *hclwrite.Body, key string, valu
 		return false
 	}
 
+	// If EncodeSeparate is set, emit children as separate blocks regardless of label extraction
+	if valueNode.EncodeSeparate {
+		if handled, _ := he.encodeMappingChildrenAsBlocks(body, key, valueNode); handled {
+			return true
+		}
+	}
+
 	// Try to extract block labels from a single-entry mapping chain
 	if labels, bodyNode, ok := extractBlockLabels(valueNode); ok {
 		if len(labels) > 1 && mappingChildrenAllMappings(bodyNode) {
@@ -519,17 +526,46 @@ func (he *hclEncoder) encodeMappingChildrenAsBlocks(body *hclwrite.Body, blockTy
 		return false, nil
 	}
 
+	// Only emit as separate blocks if EncodeSeparate is true
+	// This allows the encoder to respect the original block structure preserved by the decoder
+	if !valueNode.EncodeSeparate {
+		return false, nil
+	}
+
 	for i := 0; i < len(valueNode.Content); i += 2 {
 		childKey := valueNode.Content[i].Value
 		childVal := valueNode.Content[i+1]
-		labels := []string{childKey}
-		if extraLabels, bodyNode, ok := extractBlockLabels(childVal); ok {
-			labels = append(labels, extraLabels...)
-			childVal = bodyNode
-		}
-		block := body.AppendNewBlock(blockType, labels)
-		if err := he.encodeNodeAttributes(block.Body(), childVal); err != nil {
-			return true, err
+
+		// Check if this child also represents multiple blocks (all children are mappings)
+		if mappingChildrenAllMappings(childVal) {
+			// Recursively emit each grandchild as a separate block with extended labels
+			for j := 0; j < len(childVal.Content); j += 2 {
+				grandchildKey := childVal.Content[j].Value
+				grandchildVal := childVal.Content[j+1]
+				labels := []string{childKey, grandchildKey}
+
+				// Try to extract additional labels if this is a single-entry chain
+				if extraLabels, bodyNode, ok := extractBlockLabels(grandchildVal); ok {
+					labels = append(labels, extraLabels...)
+					grandchildVal = bodyNode
+				}
+
+				block := body.AppendNewBlock(blockType, labels)
+				if err := he.encodeNodeAttributes(block.Body(), grandchildVal); err != nil {
+					return true, err
+				}
+			}
+		} else {
+			// Single block with this child as label(s)
+			labels := []string{childKey}
+			if extraLabels, bodyNode, ok := extractBlockLabels(childVal); ok {
+				labels = append(labels, extraLabels...)
+				childVal = bodyNode
+			}
+			block := body.AppendNewBlock(blockType, labels)
+			if err := he.encodeNodeAttributes(block.Body(), childVal); err != nil {
+				return true, err
+			}
 		}
 	}
 

pkg/yqlib/hcl_test.go

@@ -345,6 +345,68 @@ var hclFormatScenarios = []formatScenario{
 		expected:     "tags:\n  - \"a\"\n  - \"b\"\n",
 		scenarioType: "decode",
 	},
+	{
+		description:  "roundtrip list of objects",
+		skipDoc:      true,
+		input:        `items = [{ name = "a", value = 1 }, { name = "b", value = 2 }]`,
+		expected:     "items = [{\n  name = \"a\"\n  value = 1\n  }, {\n  name = \"b\"\n  value = 2\n}]\n",
+		scenarioType: "roundtrip",
+	},
+	{
+		description:  "roundtrip nested blocks with same name",
+		skipDoc:      true,
+		input:        "database \"primary\" {\n  host = \"localhost\"\n  port = 5432\n}\ndatabase \"replica\" {\n  host = \"replica.local\"\n  port = 5433\n}",
+		expected:     "database \"primary\" {\n  host = \"localhost\"\n  port = 5432\n}\ndatabase \"replica\" {\n  host = \"replica.local\"\n  port = 5433\n}\n",
+		scenarioType: "roundtrip",
+	},
+	{
+		description:  "roundtrip mixed nested structure",
+		skipDoc:      true,
+		input:        "servers \"web\" {\n  addresses = [\"10.0.1.1\", \"10.0.1.2\"]\n  port = 8080\n}",
+		expected:     "servers \"web\" {\n  addresses = [\"10.0.1.1\", \"10.0.1.2\"]\n  port = 8080\n}\n",
+		scenarioType: "roundtrip",
+	},
+	{
+		description:  "roundtrip null value",
+		skipDoc:      true,
+		input:        `value = null`,
+		expected:     "value = null\n",
+		scenarioType: "roundtrip",
+	},
+	{
+		description:  "roundtrip empty list",
+		skipDoc:      true,
+		input:        `items = []`,
+		expected:     "items = []\n",
+		scenarioType: "roundtrip",
+	},
+	{
+		description:  "roundtrip empty object",
+		skipDoc:      true,
+		input:        `config = {}`,
+		expected:     "config = {}\n",
+		scenarioType: "roundtrip",
+	},
+	{
+		description:  "Roundtrip: Separate blocks with same name.",
+		input:        "resource \"aws_instance\" \"web\" {\n  ami = \"ami-12345\"\n}\nresource \"aws_instance\" \"db\" {\n  ami = \"ami-67890\"\n}",
+		expected:     "resource \"aws_instance\" \"web\" {\n  ami = \"ami-12345\"\n}\nresource \"aws_instance\" \"db\" {\n  ami = \"ami-67890\"\n}\n",
+		scenarioType: "roundtrip",
+	},
+	{
+		description:  "roundtrip deeply nested structure",
+		skipDoc:      true,
+		input:        "app \"database\" \"primary\" \"connection\" {\n  host = \"db.local\"\n  port = 5432\n}",
+		expected:     "app \"database\" \"primary\" \"connection\" {\n  host = \"db.local\"\n  port = 5432\n}\n",
+		scenarioType: "roundtrip",
+	},
+	{
+		description:  "roundtrip with leading comments",
+		skipDoc:      true,
+		input:        "# Main config\nenabled = true\nport = 8080",
+		expected:     "# Main config\nenabled = true\nport = 8080\n",
+		scenarioType: "roundtrip",
+	},
 }
 
 func testHclScenario(t *testing.T, s formatScenario) {