parser/metadecoders: Add CSV lazyQuotes option to transform.Unmarshal - hugo - [fork] hugo port for 9front
HTML git clone git@git.drkhsh.at/hugo.git
DIR Log
DIR Files
DIR Refs
DIR Submodules
DIR README
DIR LICENSE
---
DIR commit 912c6576bb47535bb29819c9855da90580e11906
DIR parent 911bc60a7ab739885908fdfe49d1578531940909
HTML Author: Joe Mooring <joe.mooring@veriphor.com>
Date: Sat, 13 Jan 2024 10:28:39 -0800
parser/metadecoders: Add CSV lazyQuotes option to transform.Unmarshal
If true, a quote may appear in an unquoted field and a non-doubled quote
may appear in a quoted field. It defaults to false.
Closes #11884
Diffstat:
M docs/content/en/functions/transfor… | 3 +++
M parser/metadecoders/decoder.go | 7 +++++++
M tpl/transform/integration_test.go | 26 ++++++++++++++++++++++++++
3 files changed, 36 insertions(+), 0 deletions(-)
---
DIR diff --git a/docs/content/en/functions/transform/Unmarshal.md b/docs/content/en/functions/transform/Unmarshal.md
@@ -125,6 +125,9 @@ delimiter
comment
: (`string`) The comment character used in the CSV. If set, lines beginning with the comment character without preceding whitespace are ignored.
+lazyQuotes
+: (`bool`) If true, a quote may appear in an unquoted field and a non-doubled quote may appear in a quoted field. Default is `false`.
+
```go-html-template
{{ $csv := "a;b;c" | transform.Unmarshal (dict "delimiter" ";") }}
```
DIR diff --git a/parser/metadecoders/decoder.go b/parser/metadecoders/decoder.go
@@ -20,6 +20,7 @@ import (
"fmt"
"log"
"regexp"
+ "strconv"
"strings"
"github.com/gohugoio/hugo/common/herrors"
@@ -41,6 +42,10 @@ type Decoder struct {
// Comment, if not 0, is the comment character used in the CSV decoder. Lines beginning with the
// Comment character without preceding whitespace are ignored.
Comment rune
+
+ // If true, a quote may appear in an unquoted field and a non-doubled quote
+ // may appear in a quoted field. It defaults to false.
+ LazyQuotes bool
}
// OptionsKey is used in cache keys.
@@ -48,6 +53,7 @@ func (d Decoder) OptionsKey() string {
var sb strings.Builder
sb.WriteRune(d.Delimiter)
sb.WriteRune(d.Comment)
+ sb.WriteString(strconv.FormatBool(d.LazyQuotes))
return sb.String()
}
@@ -205,6 +211,7 @@ func (d Decoder) unmarshalCSV(data []byte, v any) error {
r := csv.NewReader(bytes.NewReader(data))
r.Comma = d.Delimiter
r.Comment = d.Comment
+ r.LazyQuotes = d.LazyQuotes
records, err := r.ReadAll()
if err != nil {
DIR diff --git a/tpl/transform/integration_test.go b/tpl/transform/integration_test.go
@@ -107,3 +107,29 @@ disableKinds = ['page','rss','section','sitemap','taxonomy','term']
_, err := b.BuildE()
b.Assert(err.Error(), qt.Contains, "error calling highlight: invalid Highlight option: 0")
}
+
+// Issue #11884
+func TestUnmarshalCSVLazyDecoding(t *testing.T) {
+ t.Parallel()
+
+ files := `
+-- hugo.toml --
+disableKinds = ['page','rss','section','sitemap','taxonomy','term']
+-- assets/pets.csv --
+name,description,age
+Spot,a nice dog,3
+Rover,"a big dog",5
+Felix,a "malicious" cat,7
+Bella,"an "evil" cat",9
+Scar,"a "dead cat",11
+-- layouts/index.html --
+{{ $opts := dict "lazyQuotes" true }}
+{{ $data := resources.Get "pets.csv" | transform.Unmarshal $opts }}
+{{ printf "%v" $data | safeHTML }}
+ `
+ b := hugolib.Test(t, files)
+
+ b.AssertFileContent("public/index.html", `
+[[name description age] [Spot a nice dog 3] [Rover a big dog 5] [Felix a "malicious" cat 7] [Bella an "evil" cat 9] [Scar a "dead cat 11]]
+ `)
+}