tpl/transform: Add transform.Unmarshal func - hugo - [fork] hugo port for 9front
HTML git clone git@git.drkhsh.at/hugo.git
DIR Log
DIR Files
DIR Refs
DIR Submodules
DIR README
DIR LICENSE
---
DIR commit 822dc627a1cfdf1f97882f27761675ac6ace7669
DIR parent 43f9df0194d229805d80b13c9e38a7a0fec12cf4
HTML Author: Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>
Date: Fri, 21 Dec 2018 16:21:13 +0100
tpl/transform: Add transform.Unmarshal func
Fixes #5428
Diffstat:
A cache/namedmemcache/named_cache.go | 84 +++++++++++++++++++++++++++++++
A cache/namedmemcache/named_cache_te… | 80 +++++++++++++++++++++++++++++++
M deps/deps.go | 11 +++++++++++
M helpers/general.go | 7 +++----
M helpers/general_test.go | 4 ++--
M hugolib/resource_chain_test.go | 10 ++++++++++
M media/mediaType.go | 4 ++++
M media/mediaType_test.go | 4 ++++
M parser/metadecoders/format.go | 50 +++++++++++++++++++++++++++++++
M parser/metadecoders/format_test.go | 42 +++++++++++++++++++++++++++++++
M resource/resource.go | 26 +++++++++++++++++++++-----
M resource/resource_test.go | 1 +
M resource/transform.go | 10 +++++++++-
M tpl/transform/init.go | 8 ++++++++
M tpl/transform/remarshal.go | 30 +++++-------------------------
M tpl/transform/remarshal_test.go | 32 -------------------------------
M tpl/transform/transform.go | 14 ++++++++++++--
M tpl/transform/transform_test.go | 7 ++++---
A tpl/transform/unmarshal.go | 98 +++++++++++++++++++++++++++++++
A tpl/transform/unmarshal_test.go | 185 ++++++++++++++++++++++++++++++
20 files changed, 633 insertions(+), 74 deletions(-)
---
DIR diff --git a/cache/namedmemcache/named_cache.go b/cache/namedmemcache/named_cache.go
@@ -0,0 +1,84 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package namedmemcache provides a memory cache with a named lock. This is suitable
+// for situations where creating the cached resource can be time consuming or otherwise
+// resource hungry, or in situations where a "once only per key" is a requirement.
+package namedmemcache
+
+import (
+ "sync"
+
+ "github.com/BurntSushi/locker"
+)
+
+// Cache holds the cached values.
+type Cache struct {
+ nlocker *locker.Locker
+ cache map[string]cacheEntry
+ mu sync.RWMutex
+}
+
+type cacheEntry struct {
+ value interface{}
+ err error
+}
+
+// New creates a new cache.
+func New() *Cache {
+ return &Cache{
+ nlocker: locker.NewLocker(),
+ cache: make(map[string]cacheEntry),
+ }
+}
+
+// Clear clears the cache state.
+func (c *Cache) Clear() {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+
+ c.cache = make(map[string]cacheEntry)
+ c.nlocker = locker.NewLocker()
+
+}
+
+// GetOrCreate tries to get the value with the given cache key, if not found
+// create will be called and cached.
+// This method is thread safe. It also guarantees that the create func for a given
+// key is invoced only once for this cache.
+func (c *Cache) GetOrCreate(key string, create func() (interface{}, error)) (interface{}, error) {
+ c.mu.RLock()
+ entry, found := c.cache[key]
+ c.mu.RUnlock()
+
+ if found {
+ return entry.value, entry.err
+ }
+
+ c.nlocker.Lock(key)
+ defer c.nlocker.Unlock(key)
+
+ // Double check
+ if entry, found := c.cache[key]; found {
+ return entry.value, entry.err
+ }
+
+ // Create it.
+ value, err := create()
+
+ c.mu.Lock()
+ c.cache[key] = cacheEntry{value: value, err: err}
+ c.mu.Unlock()
+
+ return value, err
+}
DIR diff --git a/cache/namedmemcache/named_cache_test.go b/cache/namedmemcache/named_cache_test.go
@@ -0,0 +1,80 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package namedmemcache
+
+import (
+ "fmt"
+ "sync"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestNamedCache(t *testing.T) {
+ t.Parallel()
+ assert := require.New(t)
+
+ cache := New()
+
+ counter := 0
+ create := func() (interface{}, error) {
+ counter++
+ return counter, nil
+ }
+
+ for i := 0; i < 5; i++ {
+ v1, err := cache.GetOrCreate("a1", create)
+ assert.NoError(err)
+ assert.Equal(1, v1)
+ v2, err := cache.GetOrCreate("a2", create)
+ assert.NoError(err)
+ assert.Equal(2, v2)
+ }
+
+ cache.Clear()
+
+ v3, err := cache.GetOrCreate("a2", create)
+ assert.NoError(err)
+ assert.Equal(3, v3)
+}
+
+func TestNamedCacheConcurrent(t *testing.T) {
+ t.Parallel()
+
+ assert := require.New(t)
+
+ var wg sync.WaitGroup
+
+ cache := New()
+
+ create := func(i int) func() (interface{}, error) {
+ return func() (interface{}, error) {
+ return i, nil
+ }
+ }
+
+ for i := 0; i < 10; i++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ for j := 0; j < 100; j++ {
+ id := fmt.Sprintf("id%d", j)
+ v, err := cache.GetOrCreate(id, create(j))
+ assert.NoError(err)
+ assert.Equal(j, v)
+ }
+ }()
+ }
+ wg.Wait()
+}
DIR diff --git a/deps/deps.go b/deps/deps.go
@@ -123,6 +123,9 @@ type Listeners struct {
// Add adds a function to a Listeners instance.
func (b *Listeners) Add(f func()) {
+ if b == nil {
+ return
+ }
b.Lock()
defer b.Unlock()
b.listeners = append(b.listeners, f)
@@ -192,6 +195,14 @@ func New(cfg DepsCfg) (*Deps, error) {
fs = hugofs.NewDefault(cfg.Language)
}
+ if cfg.MediaTypes == nil {
+ cfg.MediaTypes = media.DefaultTypes
+ }
+
+ if cfg.OutputFormats == nil {
+ cfg.OutputFormats = output.DefaultFormats
+ }
+
ps, err := helpers.NewPathSpec(fs, cfg.Language)
if err != nil {
DIR diff --git a/helpers/general.go b/helpers/general.go
@@ -394,11 +394,10 @@ func MD5FromFileFast(r io.ReadSeeker) (string, error) {
return hex.EncodeToString(h.Sum(nil)), nil
}
-// MD5FromFile creates a MD5 hash from the given file.
-// It will not close the file.
-func MD5FromFile(f afero.File) (string, error) {
+// MD5FromReader creates a MD5 hash from the given reader.
+func MD5FromReader(r io.Reader) (string, error) {
h := md5.New()
- if _, err := io.Copy(h, f); err != nil {
+ if _, err := io.Copy(h, r); err != nil {
return "", nil
}
return hex.EncodeToString(h.Sum(nil)), nil
DIR diff --git a/helpers/general_test.go b/helpers/general_test.go
@@ -272,7 +272,7 @@ func TestFastMD5FromFile(t *testing.T) {
req.NoError(err)
req.NotEqual(m3, m4)
- m5, err := MD5FromFile(bf2)
+ m5, err := MD5FromReader(bf2)
req.NoError(err)
req.NotEqual(m4, m5)
}
@@ -293,7 +293,7 @@ func BenchmarkMD5FromFileFast(b *testing.B) {
}
b.StartTimer()
if full {
- if _, err := MD5FromFile(f); err != nil {
+ if _, err := MD5FromReader(f); err != nil {
b.Fatal(err)
}
} else {
DIR diff --git a/hugolib/resource_chain_test.go b/hugolib/resource_chain_test.go
@@ -339,6 +339,16 @@ Publish 2: {{ $cssPublish2.Permalink }}
assert.False(b.CheckExists("public/inline.min.css"), "Inline content should not be copied to /public")
}},
+ {"unmarshal", func() bool { return true }, func(b *sitesBuilder) {
+ b.WithTemplates("home.html", `
+{{ $toml := "slogan = \"Hugo Rocks!\"" | resources.FromString "slogan.toml" | transform.Unmarshal }}
+Slogan: {{ $toml.slogan }}
+
+`)
+ }, func(b *sitesBuilder) {
+ b.AssertFileContent("public/index.html", `Slogan: Hugo Rocks!`)
+ }},
+
{"template", func() bool { return true }, func(b *sitesBuilder) {}, func(b *sitesBuilder) {
}},
}
DIR diff --git a/media/mediaType.go b/media/mediaType.go
@@ -135,6 +135,8 @@ var (
XMLType = Type{MainType: "application", SubType: "xml", Suffixes: []string{"xml"}, Delimiter: defaultDelimiter}
SVGType = Type{MainType: "image", SubType: "svg", mimeSuffix: "xml", Suffixes: []string{"svg"}, Delimiter: defaultDelimiter}
TextType = Type{MainType: "text", SubType: "plain", Suffixes: []string{"txt"}, Delimiter: defaultDelimiter}
+ TOMLType = Type{MainType: "application", SubType: "toml", Suffixes: []string{"toml"}, Delimiter: defaultDelimiter}
+ YAMLType = Type{MainType: "application", SubType: "yaml", Suffixes: []string{"yaml", "yml"}, Delimiter: defaultDelimiter}
OctetType = Type{MainType: "application", SubType: "octet-stream"}
)
@@ -154,6 +156,8 @@ var DefaultTypes = Types{
SVGType,
TextType,
OctetType,
+ YAMLType,
+ TOMLType,
}
func init() {
DIR diff --git a/media/mediaType_test.go b/media/mediaType_test.go
@@ -39,6 +39,8 @@ func TestDefaultTypes(t *testing.T) {
{SVGType, "image", "svg", "svg", "image/svg+xml", "image/svg+xml"},
{TextType, "text", "plain", "txt", "text/plain", "text/plain"},
{XMLType, "application", "xml", "xml", "application/xml", "application/xml"},
+ {TOMLType, "application", "toml", "toml", "application/toml", "application/toml"},
+ {YAMLType, "application", "yaml", "yaml", "application/yaml", "application/yaml"},
} {
require.Equal(t, test.expectedMainType, test.tp.MainType)
require.Equal(t, test.expectedSubType, test.tp.SubType)
@@ -50,6 +52,8 @@ func TestDefaultTypes(t *testing.T) {
}
+ require.Equal(t, 15, len(DefaultTypes))
+
}
func TestGetByType(t *testing.T) {
DIR diff --git a/parser/metadecoders/format.go b/parser/metadecoders/format.go
@@ -17,6 +17,8 @@ import (
"path/filepath"
"strings"
+ "github.com/gohugoio/hugo/media"
+
"github.com/gohugoio/hugo/parser/pageparser"
)
@@ -55,6 +57,18 @@ func FormatFromString(formatStr string) Format {
}
+// FormatFromMediaType gets the Format given a MIME type, empty string
+// if unknown.
+func FormatFromMediaType(m media.Type) Format {
+ for _, suffix := range m.Suffixes {
+ if f := FormatFromString(suffix); f != "" {
+ return f
+ }
+ }
+
+ return ""
+}
+
// FormatFromFrontMatterType will return empty if not supported.
func FormatFromFrontMatterType(typ pageparser.ItemType) Format {
switch typ {
@@ -70,3 +84,39 @@ func FormatFromFrontMatterType(typ pageparser.ItemType) Format {
return ""
}
}
+
+// FormatFromContentString tries to detect the format (JSON, YAML or TOML)
+// in the given string.
+// It return an empty string if no format could be detected.
+func FormatFromContentString(data string) Format {
+ jsonIdx := strings.Index(data, "{")
+ yamlIdx := strings.Index(data, ":")
+ tomlIdx := strings.Index(data, "=")
+
+ if isLowerIndexThan(jsonIdx, yamlIdx, tomlIdx) {
+ return JSON
+ }
+
+ if isLowerIndexThan(yamlIdx, tomlIdx) {
+ return YAML
+ }
+
+ if tomlIdx != -1 {
+ return TOML
+ }
+
+ return ""
+}
+
+func isLowerIndexThan(first int, others ...int) bool {
+ if first == -1 {
+ return false
+ }
+ for _, other := range others {
+ if other != -1 && other < first {
+ return false
+ }
+ }
+
+ return true
+}
DIR diff --git a/parser/metadecoders/format_test.go b/parser/metadecoders/format_test.go
@@ -17,6 +17,8 @@ import (
"fmt"
"testing"
+ "github.com/gohugoio/hugo/media"
+
"github.com/gohugoio/hugo/parser/pageparser"
"github.com/stretchr/testify/require"
@@ -41,6 +43,21 @@ func TestFormatFromString(t *testing.T) {
}
}
+func TestFormatFromMediaType(t *testing.T) {
+ assert := require.New(t)
+ for i, test := range []struct {
+ m media.Type
+ expect Format
+ }{
+ {media.JSONType, JSON},
+ {media.YAMLType, YAML},
+ {media.TOMLType, TOML},
+ {media.CalendarType, ""},
+ } {
+ assert.Equal(test.expect, FormatFromMediaType(test.m), fmt.Sprintf("t%d", i))
+ }
+}
+
func TestFormatFromFrontMatterType(t *testing.T) {
assert := require.New(t)
for i, test := range []struct {
@@ -56,3 +73,28 @@ func TestFormatFromFrontMatterType(t *testing.T) {
assert.Equal(test.expect, FormatFromFrontMatterType(test.typ), fmt.Sprintf("t%d", i))
}
}
+
+func TestFormatFromContentString(t *testing.T) {
+ t.Parallel()
+ assert := require.New(t)
+
+ for i, test := range []struct {
+ data string
+ expect interface{}
+ }{
+ {`foo = "bar"`, TOML},
+ {` foo = "bar"`, TOML},
+ {`foo="bar"`, TOML},
+ {`foo: "bar"`, YAML},
+ {`foo:"bar"`, YAML},
+ {`{ "foo": "bar"`, JSON},
+ {`asdfasdf`, Format("")},
+ {``, Format("")},
+ } {
+ errMsg := fmt.Sprintf("[%d] %s", i, test.data)
+
+ result := FormatFromContentString(test.data)
+
+ assert.Equal(test.expect, result, errMsg)
+ }
+}
DIR diff --git a/resource/resource.go b/resource/resource.go
@@ -50,6 +50,7 @@ var (
_ ResourcesLanguageMerger = (*Resources)(nil)
_ permalinker = (*genericResource)(nil)
_ collections.Slicer = (*genericResource)(nil)
+ _ Identifier = (*genericResource)(nil)
)
var noData = make(map[string]interface{})
@@ -76,6 +77,8 @@ type Cloner interface {
// Resource represents a linkable resource, i.e. a content page, image etc.
type Resource interface {
+ resourceBase
+
// Permalink represents the absolute link to this resource.
Permalink() string
@@ -87,9 +90,6 @@ type Resource interface {
// For content pages, this value is "page".
ResourceType() string
- // MediaType is this resource's MIME type.
- MediaType() media.Type
-
// Name is the logical name of this resource. This can be set in the front matter
// metadata for this resource. If not set, Hugo will assign a value.
// This will in most cases be the base filename.
@@ -109,6 +109,13 @@ type Resource interface {
Params() map[string]interface{}
}
+// resourceBase pulls out the minimal set of operations to define a Resource,
+// to simplify testing etc.
+type resourceBase interface {
+ // MediaType is this resource's MIME type.
+ MediaType() media.Type
+}
+
// ResourcesLanguageMerger describes an interface for merging resources from a
// different language.
type ResourcesLanguageMerger interface {
@@ -121,12 +128,17 @@ type translatedResource interface {
TranslationKey() string
}
+// Identifier identifies a resource.
+type Identifier interface {
+ Key() string
+}
+
// ContentResource represents a Resource that provides a way to get to its content.
// Most Resource types in Hugo implements this interface, including Page.
// This should be used with care, as it will read the file content into memory, but it
// should be cached as effectively as possible by the implementation.
type ContentResource interface {
- Resource
+ resourceBase
// Content returns this resource's content. It will be equivalent to reading the content
// that RelPermalink points to in the published folder.
@@ -143,7 +155,7 @@ type OpenReadSeekCloser func() (hugio.ReadSeekCloser, error)
// ReadSeekCloserResource is a Resource that supports loading its content.
type ReadSeekCloserResource interface {
- Resource
+ resourceBase
ReadSeekCloser() (hugio.ReadSeekCloser, error)
}
@@ -716,6 +728,10 @@ func (l *genericResource) RelPermalink() string {
return l.relPermalinkFor(l.relTargetDirFile.path())
}
+func (l *genericResource) Key() string {
+ return l.relTargetDirFile.path()
+}
+
func (l *genericResource) relPermalinkFor(target string) string {
return l.relPermalinkForRel(target, false)
DIR diff --git a/resource/resource_test.go b/resource/resource_test.go
@@ -50,6 +50,7 @@ func TestGenericResourceWithLinkFacory(t *testing.T) {
assert.Equal("https://example.com/foo/foo.css", r.Permalink())
assert.Equal("/foo/foo.css", r.RelPermalink())
+ assert.Equal("foo.css", r.Key())
assert.Equal("css", r.ResourceType())
}
DIR diff --git a/resource/transform.go b/resource/transform.go
@@ -38,6 +38,7 @@ var (
_ ContentResource = (*transformedResource)(nil)
_ ReadSeekCloserResource = (*transformedResource)(nil)
_ collections.Slicer = (*transformedResource)(nil)
+ _ Identifier = (*transformedResource)(nil)
)
func (s *Spec) Transform(r Resource, t ResourceTransformation) (Resource, error) {
@@ -249,6 +250,13 @@ func (r *transformedResource) MediaType() media.Type {
return m
}
+func (r *transformedResource) Key() string {
+ if err := r.initTransform(false, false); err != nil {
+ return ""
+ }
+ return r.linker.relPermalinkFor(r.Target)
+}
+
func (r *transformedResource) Permalink() string {
if err := r.initTransform(false, true); err != nil {
return ""
@@ -481,8 +489,8 @@ func (r *transformedResource) transform(setContent, publish bool) (err error) {
}
return nil
-
}
+
func (r *transformedResource) initTransform(setContent, publish bool) error {
r.transformInit.Do(func() {
r.published = publish
DIR diff --git a/tpl/transform/init.go b/tpl/transform/init.go
@@ -95,6 +95,14 @@ func init() {
},
)
+ ns.AddMethodMapping(ctx.Unmarshal,
+ []string{"unmarshal"},
+ [][2]string{
+ {`{{ "hello = \"Hello World\"" | transform.Unmarshal }}`, "map[hello:Hello World]"},
+ {`{{ "hello = \"Hello World\"" | resources.FromString "data/greetings.toml" | transform.Unmarshal }}`, "map[hello:Hello World]"},
+ },
+ )
+
return ns
}
DIR diff --git a/tpl/transform/remarshal.go b/tpl/transform/remarshal.go
@@ -2,9 +2,10 @@ package transform
import (
"bytes"
- "errors"
"strings"
+ "github.com/pkg/errors"
+
"github.com/gohugoio/hugo/parser"
"github.com/gohugoio/hugo/parser/metadecoders"
"github.com/spf13/cast"
@@ -34,9 +35,9 @@ func (ns *Namespace) Remarshal(format string, data interface{}) (string, error)
return "", err
}
- fromFormat, err := detectFormat(from)
- if err != nil {
- return "", err
+ fromFormat := metadecoders.FormatFromContentString(from)
+ if fromFormat == "" {
+ return "", errors.New("failed to detect format from content")
}
meta, err := metadecoders.UnmarshalToMap([]byte(from), fromFormat)
@@ -56,24 +57,3 @@ func toFormatMark(format string) (metadecoders.Format, error) {
return "", errors.New("failed to detect target data serialization format")
}
-
-func detectFormat(data string) (metadecoders.Format, error) {
- jsonIdx := strings.Index(data, "{")
- yamlIdx := strings.Index(data, ":")
- tomlIdx := strings.Index(data, "=")
-
- if jsonIdx != -1 && (yamlIdx == -1 || jsonIdx < yamlIdx) && (tomlIdx == -1 || jsonIdx < tomlIdx) {
- return metadecoders.JSON, nil
- }
-
- if yamlIdx != -1 && (tomlIdx == -1 || yamlIdx < tomlIdx) {
- return metadecoders.YAML, nil
- }
-
- if tomlIdx != -1 {
- return metadecoders.TOML, nil
- }
-
- return "", errors.New("failed to detect data serialization format")
-
-}
DIR diff --git a/tpl/transform/remarshal_test.go b/tpl/transform/remarshal_test.go
@@ -18,7 +18,6 @@ import (
"testing"
"github.com/gohugoio/hugo/helpers"
- "github.com/gohugoio/hugo/parser/metadecoders"
"github.com/spf13/viper"
"github.com/stretchr/testify/require"
)
@@ -171,34 +170,3 @@ func TestTestRemarshalError(t *testing.T) {
assert.Error(err)
}
-
-func TestRemarshalDetectFormat(t *testing.T) {
- t.Parallel()
- assert := require.New(t)
-
- for i, test := range []struct {
- data string
- expect interface{}
- }{
- {`foo = "bar"`, metadecoders.TOML},
- {` foo = "bar"`, metadecoders.TOML},
- {`foo="bar"`, metadecoders.TOML},
- {`foo: "bar"`, metadecoders.YAML},
- {`foo:"bar"`, metadecoders.YAML},
- {`{ "foo": "bar"`, metadecoders.JSON},
- {`asdfasdf`, false},
- {``, false},
- } {
- errMsg := fmt.Sprintf("[%d] %s", i, test.data)
-
- result, err := detectFormat(test.data)
-
- if b, ok := test.expect.(bool); ok && !b {
- assert.Error(err, errMsg)
- continue
- }
-
- assert.NoError(err, errMsg)
- assert.Equal(test.expect, result)
- }
-}
DIR diff --git a/tpl/transform/transform.go b/tpl/transform/transform.go
@@ -19,6 +19,8 @@ import (
"html"
"html/template"
+ "github.com/gohugoio/hugo/cache/namedmemcache"
+
"github.com/gohugoio/hugo/deps"
"github.com/gohugoio/hugo/helpers"
"github.com/spf13/cast"
@@ -26,14 +28,22 @@ import (
// New returns a new instance of the transform-namespaced template functions.
func New(deps *deps.Deps) *Namespace {
+ cache := namedmemcache.New()
+ deps.BuildStartListeners.Add(
+ func() {
+ cache.Clear()
+ })
+
return &Namespace{
- deps: deps,
+ cache: cache,
+ deps: deps,
}
}
// Namespace provides template functions for the "transform" namespace.
type Namespace struct {
- deps *deps.Deps
+ cache *namedmemcache.Cache
+ deps *deps.Deps
}
// Emojify returns a copy of s with all emoji codes replaced with actual emojis.
DIR diff --git a/tpl/transform/transform_test.go b/tpl/transform/transform_test.go
@@ -34,7 +34,6 @@ func TestEmojify(t *testing.T) {
t.Parallel()
v := viper.New()
- v.Set("contentDir", "content")
ns := New(newDeps(v))
for i, test := range []struct {
@@ -215,7 +214,6 @@ func TestPlainify(t *testing.T) {
t.Parallel()
v := viper.New()
- v.Set("contentDir", "content")
ns := New(newDeps(v))
for i, test := range []struct {
@@ -241,8 +239,11 @@ func TestPlainify(t *testing.T) {
}
func newDeps(cfg config.Provider) *deps.Deps {
+ cfg.Set("contentDir", "content")
+ cfg.Set("i18nDir", "i18n")
+
l := langs.NewLanguage("en", cfg)
- l.Set("i18nDir", "i18n")
+
cs, err := helpers.NewContentSpec(l)
if err != nil {
panic(err)
DIR diff --git a/tpl/transform/unmarshal.go b/tpl/transform/unmarshal.go
@@ -0,0 +1,98 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package transform
+
+import (
+ "io/ioutil"
+
+ "github.com/gohugoio/hugo/common/hugio"
+
+ "github.com/gohugoio/hugo/helpers"
+ "github.com/gohugoio/hugo/parser/metadecoders"
+ "github.com/gohugoio/hugo/resource"
+ "github.com/pkg/errors"
+
+ "github.com/spf13/cast"
+)
+
+// Unmarshal unmarshals the data given, which can be either a string
+// or a Resource. Supported formats are JSON, TOML and YAML.
+func (ns *Namespace) Unmarshal(data interface{}) (interface{}, error) {
+
+ // All the relevant Resource types implements ReadSeekCloserResource,
+ // which should be the most effective way to get the content.
+ if r, ok := data.(resource.ReadSeekCloserResource); ok {
+ var key string
+ var reader hugio.ReadSeekCloser
+
+ if k, ok := r.(resource.Identifier); ok {
+ key = k.Key()
+ }
+
+ if key == "" {
+ reader, err := r.ReadSeekCloser()
+ if err != nil {
+ return nil, err
+ }
+ defer reader.Close()
+
+ key, err = helpers.MD5FromReader(reader)
+ if err != nil {
+ return nil, err
+ }
+
+ reader.Seek(0, 0)
+ }
+
+ return ns.cache.GetOrCreate(key, func() (interface{}, error) {
+ f := metadecoders.FormatFromMediaType(r.MediaType())
+ if f == "" {
+ return nil, errors.Errorf("MIME %q not supported", r.MediaType())
+ }
+
+ if reader == nil {
+ var err error
+ reader, err = r.ReadSeekCloser()
+ if err != nil {
+ return nil, err
+ }
+ defer reader.Close()
+ }
+
+ b, err := ioutil.ReadAll(reader)
+ if err != nil {
+ return nil, err
+ }
+
+ return metadecoders.Unmarshal(b, f)
+ })
+
+ }
+
+ dataStr, err := cast.ToStringE(data)
+ if err != nil {
+ return nil, errors.Errorf("type %T not supported", data)
+ }
+
+ key := helpers.MD5String(dataStr)
+
+ return ns.cache.GetOrCreate(key, func() (interface{}, error) {
+ f := metadecoders.FormatFromContentString(dataStr)
+ if f == "" {
+ return nil, errors.New("unknown format")
+ }
+
+ return metadecoders.Unmarshal([]byte(dataStr), f)
+ })
+}
DIR diff --git a/tpl/transform/unmarshal_test.go b/tpl/transform/unmarshal_test.go
@@ -0,0 +1,185 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package transform
+
+import (
+ "fmt"
+ "math/rand"
+ "strings"
+ "testing"
+
+ "github.com/gohugoio/hugo/common/hugio"
+
+ "github.com/gohugoio/hugo/media"
+
+ "github.com/gohugoio/hugo/resource"
+ "github.com/spf13/viper"
+ "github.com/stretchr/testify/require"
+)
+
+const (
+ testJSON = `
+
+{
+ "ROOT_KEY": {
+ "title": "example glossary",
+ "GlossDiv": {
+ "title": "S",
+ "GlossList": {
+ "GlossEntry": {
+ "ID": "SGML",
+ "SortAs": "SGML",
+ "GlossTerm": "Standard Generalized Markup Language",
+ "Acronym": "SGML",
+ "Abbrev": "ISO 8879:1986",
+ "GlossDef": {
+ "para": "A meta-markup language, used to create markup languages such as DocBook.",
+ "GlossSeeAlso": ["GML", "XML"]
+ },
+ "GlossSee": "markup"
+ }
+ }
+ }
+ }
+}
+
+ `
+)
+
+var _ resource.ReadSeekCloserResource = (*testContentResource)(nil)
+
+type testContentResource struct {
+ content string
+ mime media.Type
+
+ key string
+}
+
+func (t testContentResource) ReadSeekCloser() (hugio.ReadSeekCloser, error) {
+ return hugio.NewReadSeekerNoOpCloserFromString(t.content), nil
+}
+
+func (t testContentResource) MediaType() media.Type {
+ return t.mime
+}
+
+func (t testContentResource) Key() string {
+ return t.key
+}
+
+func TestUnmarshal(t *testing.T) {
+
+ v := viper.New()
+ ns := New(newDeps(v))
+ assert := require.New(t)
+
+ assertSlogan := func(m map[string]interface{}) {
+ assert.Equal("Hugo Rocks!", m["slogan"])
+ }
+
+ for i, test := range []struct {
+ data interface{}
+ expect interface{}
+ }{
+ {`{ "slogan": "Hugo Rocks!" }`, func(m map[string]interface{}) {
+ assertSlogan(m)
+ }},
+ {`slogan: "Hugo Rocks!"`, func(m map[string]interface{}) {
+ assertSlogan(m)
+ }},
+ {`slogan = "Hugo Rocks!"`, func(m map[string]interface{}) {
+ assertSlogan(m)
+ }},
+ {testContentResource{content: `slogan: "Hugo Rocks!"`, mime: media.YAMLType}, func(m map[string]interface{}) {
+ assertSlogan(m)
+ }},
+ {testContentResource{content: `{ "slogan": "Hugo Rocks!" }`, mime: media.JSONType}, func(m map[string]interface{}) {
+ assertSlogan(m)
+ }},
+ {testContentResource{content: `slogan = "Hugo Rocks!"`, mime: media.TOMLType}, func(m map[string]interface{}) {
+ assertSlogan(m)
+ }},
+ // errors
+ {"thisisnotavaliddataformat", false},
+ {testContentResource{content: `invalid&toml"`, mime: media.TOMLType}, false},
+ {testContentResource{content: `unsupported: MIME"`, mime: media.CalendarType}, false},
+ {"thisisnotavaliddataformat", false},
+ {`{ notjson }`, false},
+ {tstNoStringer{}, false},
+ } {
+ errMsg := fmt.Sprintf("[%d]", i)
+
+ result, err := ns.Unmarshal(test.data)
+
+ if b, ok := test.expect.(bool); ok && !b {
+ assert.Error(err, errMsg)
+ } else if fn, ok := test.expect.(func(m map[string]interface{})); ok {
+ assert.NoError(err, errMsg)
+ m, ok := result.(map[string]interface{})
+ assert.True(ok, errMsg)
+ fn(m)
+ } else {
+ assert.NoError(err, errMsg)
+ assert.Equal(test.expect, result, errMsg)
+ }
+
+ }
+}
+
+func BenchmarkUnmarshalString(b *testing.B) {
+ v := viper.New()
+ ns := New(newDeps(v))
+
+ const numJsons = 100
+
+ var jsons [numJsons]string
+ for i := 0; i < numJsons; i++ {
+ jsons[i] = strings.Replace(testJSON, "ROOT_KEY", fmt.Sprintf("root%d", i), 1)
+ }
+
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ result, err := ns.Unmarshal(jsons[rand.Intn(numJsons)])
+ if err != nil {
+ b.Fatal(err)
+ }
+ if result == nil {
+ b.Fatal("no result")
+ }
+ }
+}
+
+func BenchmarkUnmarshalResource(b *testing.B) {
+ v := viper.New()
+ ns := New(newDeps(v))
+
+ const numJsons = 100
+
+ var jsons [numJsons]testContentResource
+ for i := 0; i < numJsons; i++ {
+ key := fmt.Sprintf("root%d", i)
+ jsons[i] = testContentResource{key: key, content: strings.Replace(testJSON, "ROOT_KEY", key, 1), mime: media.JSONType}
+ }
+
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ result, err := ns.Unmarshal(jsons[rand.Intn(numJsons)])
+ if err != nil {
+ b.Fatal(err)
+ }
+ if result == nil {
+ b.Fatal("no result")
+ }
+ }
+}