Image resource refactor - hugo - [fork] hugo port for 9front
HTML git clone git@git.drkhsh.at/hugo.git
DIR Log
DIR Files
DIR Refs
DIR Submodules
DIR README
DIR LICENSE
---
DIR commit f9978ed16476ca6d233a89669c62c798cdf9db9d
DIR parent 58d4c0a8be8beefbd7437b17bf7a9a381164d09b
HTML Author: Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>
Date: Sun, 18 Aug 2019 11:21:27 +0200
Image resource refactor
This commit pulls most of the image related logic into its own package, to make it easier to reason about and extend.
This is also a rewrite of the transformation logic used in Hugo Pipes, mostly to allow constructs like the one below:
{{ ($myimg | fingerprint ).Width }}
Fixes #5903
Fixes #6234
Fixes #6266
Diffstat:
M common/herrors/errors.go | 1 +
M htesting/test_helpers.go | 19 +++++++++++++++++++
A hugolib/assets/images/sunset.jpg | 0
M hugolib/pagebundler_test.go | 10 ++++------
M hugolib/resource_chain_test.go | 67 +++++++++++++++++++++++++++++--
M hugolib/testhelpers_test.go | 1 +
M resources/image.go | 506 +++++--------------------------
M resources/image_cache.go | 63 ++++++++++++++-----------------
M resources/image_test.go | 189 ++++++++++---------------------
A resources/images/config.go | 276 ++++++++++++++++++++++++++++++
A resources/images/config_test.go | 125 +++++++++++++++++++++++++++++++
A resources/images/image.go | 170 +++++++++++++++++++++++++++++++
A resources/images/smartcrop.go | 75 +++++++++++++++++++++++++++++++
A resources/internal/key.go | 61 +++++++++++++++++++++++++++++++
A resources/internal/key_test.go | 36 +++++++++++++++++++++++++++++++
M resources/resource.go | 827 +++++++++++++------------------
M resources/resource/resourcetypes.go | 20 +++++++++++++++++++-
M resources/resource_cache.go | 2 +-
M resources/resource_metadata.go | 20 ++++++++++++++++----
M resources/resource_metadata_test.go | 2 +-
A resources/resource_spec.go | 304 +++++++++++++++++++++++++++++++
M resources/resource_test.go | 25 ++++++++++---------------
A resources/resource_transformers/ht… | 80 +++++++++++++++++++++++++++++++
M resources/resource_transformers/in… | 25 +++++++++++++++++--------
M resources/resource_transformers/in… | 24 ++++++++++++++++++++++++
M resources/resource_transformers/mi… | 18 +++++++++---------
A resources/resource_transformers/mi… | 43 ++++++++++++++++++++++++++++++
M resources/resource_transformers/po… | 12 +++++-------
M resources/resource_transformers/te… | 22 ++++++++++------------
M resources/resource_transformers/to… | 12 +++++-------
D resources/smartcrop.go | 77 -------------------------------
M resources/testhelpers_test.go | 44 ++++++++++++++++++++++---------
M resources/transform.go | 598 ++++++++++++++++---------------
M resources/transform_test.go | 428 ++++++++++++++++++++++++++++++-
M tpl/resources/resources.go | 22 ++++++++++++----------
35 files changed, 2661 insertions(+), 1543 deletions(-)
---
DIR diff --git a/common/herrors/errors.go b/common/herrors/errors.go
@@ -52,6 +52,7 @@ func FprintStackTrace(w io.Writer, err error) {
// defer herrors.Recover()
func Recover(args ...interface{}) {
if r := recover(); r != nil {
+ fmt.Println("ERR:", r)
args = append(args, "stacktrace from panic: \n"+string(debug.Stack()), "\n")
fmt.Println(args...)
}
DIR diff --git a/htesting/test_helpers.go b/htesting/test_helpers.go
@@ -14,8 +14,10 @@
package htesting
import (
+ "math/rand"
"runtime"
"strings"
+ "time"
"github.com/spf13/afero"
)
@@ -37,3 +39,20 @@ func CreateTempDir(fs afero.Fs, prefix string) (string, func(), error) {
}
return tempDir, func() { fs.RemoveAll(tempDir) }, nil
}
+
+// BailOut panics with a stack trace after the given duration. Useful for
+// hanging tests.
+func BailOut(after time.Duration) {
+ time.AfterFunc(after, func() {
+ buf := make([]byte, 1<<16)
+ runtime.Stack(buf, true)
+ panic(string(buf))
+ })
+
+}
+
+var rnd = rand.New(rand.NewSource(time.Now().UnixNano()))
+
+func RandIntn(n int) int {
+ return rnd.Intn(n)
+}
DIR diff --git a/hugolib/assets/images/sunset.jpg b/hugolib/assets/images/sunset.jpg
Binary files differ.
DIR diff --git a/hugolib/pagebundler_test.go b/hugolib/pagebundler_test.go
@@ -42,8 +42,7 @@ import (
)
func TestPageBundlerSiteRegular(t *testing.T) {
- t.Parallel()
-
+ c := qt.New(t)
baseBaseURL := "https://example.com"
for _, baseURLPath := range []string{"", "/hugo"} {
@@ -55,15 +54,14 @@ func TestPageBundlerSiteRegular(t *testing.T) {
}
ugly := ugly
canonify := canonify
- t.Run(fmt.Sprintf("ugly=%t,canonify=%t,path=%s", ugly, canonify, baseURLPathId),
- func(t *testing.T) {
- t.Parallel()
+ c.Run(fmt.Sprintf("ugly=%t,canonify=%t,path=%s", ugly, canonify, baseURLPathId),
+ func(c *qt.C) {
+ c.Parallel()
baseURL := baseBaseURL + baseURLPath
relURLBase := baseURLPath
if canonify {
relURLBase = ""
}
- c := qt.New(t)
fs, cfg := newTestBundleSources(t)
cfg.Set("baseURL", baseURL)
cfg.Set("canonifyURLs", canonify)
DIR diff --git a/hugolib/resource_chain_test.go b/hugolib/resource_chain_test.go
@@ -14,6 +14,7 @@
package hugolib
import (
+ "io"
"os"
"path/filepath"
"testing"
@@ -167,6 +168,64 @@ T1: {{ $r.Content }}
}
+func TestResourceChainBasic(t *testing.T) {
+ t.Parallel()
+
+ b := newTestSitesBuilder(t)
+ b.WithTemplatesAdded("index.html", `
+{{ $hello := "<h1> Hello World! </h1>" | resources.FromString "hello.html" | fingerprint "sha512" | minify | fingerprint }}
+
+HELLO: {{ $hello.Name }}|{{ $hello.RelPermalink }}|{{ $hello.Content | safeHTML }}
+
+{{ $img := resources.Get "images/sunset.jpg" }}
+{{ $fit := $img.Fit "200x200" }}
+{{ $fit2 := $fit.Fit "100x200" }}
+{{ $img = $img | fingerprint }}
+SUNSET: {{ $img.Name }}|{{ $img.RelPermalink }}|{{ $img.Width }}|{{ len $img.Content }}
+FIT: {{ $fit.Name }}|{{ $fit.RelPermalink }}|{{ $fit.Width }}
+`)
+
+ fs := b.Fs.Source
+
+ imageDir := filepath.Join("assets", "images")
+ b.Assert(os.MkdirAll(imageDir, 0777), qt.IsNil)
+ src, err := os.Open("testdata/sunset.jpg")
+ b.Assert(err, qt.IsNil)
+ out, err := fs.Create(filepath.Join(imageDir, "sunset.jpg"))
+ b.Assert(err, qt.IsNil)
+ _, err = io.Copy(out, src)
+ b.Assert(err, qt.IsNil)
+ out.Close()
+
+ b.Running()
+
+ for i := 0; i < 2; i++ {
+
+ b.Build(BuildCfg{})
+
+ b.AssertFileContent("public/index.html",
+ `
+SUNSET: images/sunset.jpg|/images/sunset.a9bf1d944e19c0f382e0d8f51de690f7d0bc8fa97390c4242a86c3e5c0737e71.jpg|900|90587
+FIT: images/sunset.jpg|/images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_200x200_fit_q75_box.jpg|200
+
+`)
+
+ b.EditFiles("page1.md", `
+---
+title: "Page 1 edit"
+summary: "Edited summary"
+---
+
+Edited content.
+
+`)
+
+ b.Assert(b.Fs.Destination.Remove("public"), qt.IsNil)
+ b.H.ResourceSpec.ClearCaches()
+
+ }
+}
+
func TestResourceChain(t *testing.T) {
t.Parallel()
@@ -353,9 +412,11 @@ Publish 2: {{ $cssPublish2.Permalink }}
"Publish 1: body{color:blue} /external1.min.css",
"Publish 2: http://example.com/external2.min.css",
)
- c.Assert(b.CheckExists("public/external2.min.css"), qt.Equals, true)
- c.Assert(b.CheckExists("public/external1.min.css"), qt.Equals, true)
- c.Assert(b.CheckExists("public/inline.min.css"), qt.Equals, false)
+ b.Assert(b.CheckExists("public/external2.css"), qt.Equals, false)
+ b.Assert(b.CheckExists("public/external1.css"), qt.Equals, false)
+ b.Assert(b.CheckExists("public/external2.min.css"), qt.Equals, true)
+ b.Assert(b.CheckExists("public/external1.min.css"), qt.Equals, true)
+ b.Assert(b.CheckExists("public/inline.min.css"), qt.Equals, false)
}},
{"unmarshal", func() bool { return true }, func(b *sitesBuilder) {
DIR diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go
@@ -536,6 +536,7 @@ func (s *sitesBuilder) changeEvents() []fsnotify.Event {
}
func (s *sitesBuilder) build(cfg BuildCfg, shouldFail bool) *sitesBuilder {
+ s.Helper()
defer func() {
s.changedFiles = nil
}()
DIR diff --git a/resources/image.go b/resources/image.go
@@ -14,198 +14,98 @@
package resources
import (
- "errors"
"fmt"
"image"
"image/color"
"image/draw"
- "image/jpeg"
- "io"
+ _ "image/gif"
+ _ "image/png"
"os"
- "strconv"
"strings"
- "sync"
"github.com/gohugoio/hugo/resources/resource"
_errors "github.com/pkg/errors"
"github.com/disintegration/imaging"
- "github.com/gohugoio/hugo/common/hugio"
"github.com/gohugoio/hugo/helpers"
- "github.com/mitchellh/mapstructure"
+ "github.com/gohugoio/hugo/resources/images"
// Blind import for image.Decode
- _ "image/gif"
- _ "image/png"
// Blind import for image.Decode
_ "golang.org/x/image/webp"
)
var (
- _ resource.Resource = (*Image)(nil)
- _ resource.Source = (*Image)(nil)
- _ resource.Cloner = (*Image)(nil)
+ _ resource.Image = (*imageResource)(nil)
+ _ resource.Source = (*imageResource)(nil)
+ _ resource.Cloner = (*imageResource)(nil)
)
-// Imaging contains default image processing configuration. This will be fetched
-// from site (or language) config.
-type Imaging struct {
- // Default image quality setting (1-100). Only used for JPEG images.
- Quality int
-
- // Resample filter used. See https://github.com/disintegration/imaging
- ResampleFilter string
+// ImageResource represents an image resource.
+type imageResource struct {
+ *images.Image
- // The anchor used in Fill. Default is "smart", i.e. Smart Crop.
- Anchor string
+ baseResource
}
-const (
- defaultJPEGQuality = 75
- defaultResampleFilter = "box"
-)
-
-var (
- imageFormats = map[string]imaging.Format{
- ".jpg": imaging.JPEG,
- ".jpeg": imaging.JPEG,
- ".png": imaging.PNG,
- ".tif": imaging.TIFF,
- ".tiff": imaging.TIFF,
- ".bmp": imaging.BMP,
- ".gif": imaging.GIF,
+func (i *imageResource) Clone() resource.Resource {
+ gr := i.baseResource.Clone().(baseResource)
+ return &imageResource{
+ Image: i.WithSpec(gr),
+ baseResource: gr,
}
-
- // Add or increment if changes to an image format's processing requires
- // re-generation.
- imageFormatsVersions = map[imaging.Format]int{
- imaging.PNG: 2, // Floyd Steinberg dithering
- }
-
- // Increment to mark all processed images as stale. Only use when absolutely needed.
- // See the finer grained smartCropVersionNumber and imageFormatsVersions.
- mainImageVersionNumber = 0
-)
-
-var anchorPositions = map[string]imaging.Anchor{
- strings.ToLower("Center"): imaging.Center,
- strings.ToLower("TopLeft"): imaging.TopLeft,
- strings.ToLower("Top"): imaging.Top,
- strings.ToLower("TopRight"): imaging.TopRight,
- strings.ToLower("Left"): imaging.Left,
- strings.ToLower("Right"): imaging.Right,
- strings.ToLower("BottomLeft"): imaging.BottomLeft,
- strings.ToLower("Bottom"): imaging.Bottom,
- strings.ToLower("BottomRight"): imaging.BottomRight,
}
-var imageFilters = map[string]imaging.ResampleFilter{
- strings.ToLower("NearestNeighbor"): imaging.NearestNeighbor,
- strings.ToLower("Box"): imaging.Box,
- strings.ToLower("Linear"): imaging.Linear,
- strings.ToLower("Hermite"): imaging.Hermite,
- strings.ToLower("MitchellNetravali"): imaging.MitchellNetravali,
- strings.ToLower("CatmullRom"): imaging.CatmullRom,
- strings.ToLower("BSpline"): imaging.BSpline,
- strings.ToLower("Gaussian"): imaging.Gaussian,
- strings.ToLower("Lanczos"): imaging.Lanczos,
- strings.ToLower("Hann"): imaging.Hann,
- strings.ToLower("Hamming"): imaging.Hamming,
- strings.ToLower("Blackman"): imaging.Blackman,
- strings.ToLower("Bartlett"): imaging.Bartlett,
- strings.ToLower("Welch"): imaging.Welch,
- strings.ToLower("Cosine"): imaging.Cosine,
-}
-
-// Image represents an image resource.
-type Image struct {
- config image.Config
- configInit sync.Once
- configLoaded bool
-
- imaging *Imaging
-
- format imaging.Format
-
- *genericResource
-}
+func (i *imageResource) cloneWithUpdates(u *transformationUpdate) (baseResource, error) {
+ base, err := i.baseResource.cloneWithUpdates(u)
+ if err != nil {
+ return nil, err
+ }
-// Width returns i's width.
-func (i *Image) Width() int {
- i.initConfig()
- return i.config.Width
-}
+ var img *images.Image
-// Height returns i's height.
-func (i *Image) Height() int {
- i.initConfig()
- return i.config.Height
-}
+ if u.isContenChanged() {
+ img = i.WithSpec(base)
+ } else {
+ img = i.Image
+ }
-// WithNewBase implements the Cloner interface.
-func (i *Image) WithNewBase(base string) resource.Resource {
- return &Image{
- imaging: i.imaging,
- format: i.format,
- genericResource: i.genericResource.WithNewBase(base).(*genericResource)}
+ return &imageResource{
+ Image: img,
+ baseResource: base,
+ }, nil
}
// Resize resizes the image to the specified width and height using the specified resampling
// filter and returns the transformed image. If one of width or height is 0, the image aspect
// ratio is preserved.
-func (i *Image) Resize(spec string) (*Image, error) {
- return i.doWithImageConfig("resize", spec, func(src image.Image, conf imageConfig) (image.Image, error) {
- return imaging.Resize(src, conf.Width, conf.Height, conf.Filter), nil
+func (i *imageResource) Resize(spec string) (resource.Image, error) {
+ return i.doWithImageConfig("resize", spec, func(src image.Image, conf images.ImageConfig) (image.Image, error) {
+ return i.Proc.Resize(src, conf)
})
}
// Fit scales down the image using the specified resample filter to fit the specified
// maximum width and height.
-func (i *Image) Fit(spec string) (*Image, error) {
- return i.doWithImageConfig("fit", spec, func(src image.Image, conf imageConfig) (image.Image, error) {
- return imaging.Fit(src, conf.Width, conf.Height, conf.Filter), nil
+func (i *imageResource) Fit(spec string) (resource.Image, error) {
+ return i.doWithImageConfig("fit", spec, func(src image.Image, conf images.ImageConfig) (image.Image, error) {
+ return i.Proc.Fit(src, conf)
})
}
// Fill scales the image to the smallest possible size that will cover the specified dimensions,
// crops the resized image to the specified dimensions using the given anchor point.
// Space delimited config: 200x300 TopLeft
-func (i *Image) Fill(spec string) (*Image, error) {
- return i.doWithImageConfig("fill", spec, func(src image.Image, conf imageConfig) (image.Image, error) {
- if conf.AnchorStr == smartCropIdentifier {
- return smartCrop(src, conf.Width, conf.Height, conf.Anchor, conf.Filter)
- }
- return imaging.Fill(src, conf.Width, conf.Height, conf.Anchor, conf.Filter), nil
+func (i *imageResource) Fill(spec string) (resource.Image, error) {
+ return i.doWithImageConfig("fill", spec, func(src image.Image, conf images.ImageConfig) (image.Image, error) {
+ return i.Proc.Fill(src, conf)
})
}
-// Holds configuration to create a new image from an existing one, resize etc.
-type imageConfig struct {
- Action string
-
- // Quality ranges from 1 to 100 inclusive, higher is better.
- // This is only relevant for JPEG images.
- // Default is 75.
- Quality int
-
- // Rotate rotates an image by the given angle counter-clockwise.
- // The rotation will be performed first.
- Rotate int
-
- Width int
- Height int
-
- Filter imaging.ResampleFilter
- FilterStr string
-
- Anchor imaging.Anchor
- AnchorStr string
-}
-
-func (i *Image) isJPEG() bool {
- name := strings.ToLower(i.relTargetDirFile.file)
+func (i *imageResource) isJPEG() bool {
+ name := strings.ToLower(i.getResourcePaths().relTargetDirFile.file)
return strings.HasSuffix(name, ".jpg") || strings.HasSuffix(name, ".jpeg")
}
@@ -218,42 +118,20 @@ const imageProcWorkers = 1
var imageProcSem = make(chan bool, imageProcWorkers)
-func (i *Image) doWithImageConfig(action, spec string, f func(src image.Image, conf imageConfig) (image.Image, error)) (*Image, error) {
- conf, err := parseImageConfig(spec)
+func (i *imageResource) doWithImageConfig(action, spec string, f func(src image.Image, conf images.ImageConfig) (image.Image, error)) (resource.Image, error) {
+ conf, err := i.decodeImageConfig(action, spec)
if err != nil {
return nil, err
}
- conf.Action = action
-
- if conf.Quality <= 0 && i.isJPEG() {
- // We need a quality setting for all JPEGs
- conf.Quality = i.imaging.Quality
- }
-
- if conf.FilterStr == "" {
- conf.FilterStr = i.imaging.ResampleFilter
- conf.Filter = imageFilters[conf.FilterStr]
- }
-
- if conf.AnchorStr == "" {
- conf.AnchorStr = i.imaging.Anchor
- if !strings.EqualFold(conf.AnchorStr, smartCropIdentifier) {
- conf.Anchor = anchorPositions[conf.AnchorStr]
- }
- }
- return i.spec.imageCache.getOrCreate(i, conf, func() (*Image, image.Image, error) {
+ return i.getSpec().imageCache.getOrCreate(i, conf, func() (*imageResource, image.Image, error) {
imageProcSem <- true
defer func() {
<-imageProcSem
}()
- ci := i.clone()
-
errOp := action
- errPath := i.sourceFilename
-
- ci.setBasePath(conf)
+ errPath := i.getSourceFilename()
src, err := i.decodeSource()
if err != nil {
@@ -267,10 +145,10 @@ func (i *Image) doWithImageConfig(action, spec string, f func(src image.Image, c
converted, err := f(src, conf)
if err != nil {
- return ci, nil, &os.PathError{Op: errOp, Path: errPath, Err: err}
+ return nil, nil, &os.PathError{Op: errOp, Path: errPath, Err: err}
}
- if i.format == imaging.PNG {
+ if i.Format == imaging.PNG {
// Apply the colour palette from the source
if paletted, ok := src.(*image.Paletted); ok {
tmp := image.NewPaletted(converted.Bounds(), paletted.Palette)
@@ -279,177 +157,30 @@ func (i *Image) doWithImageConfig(action, spec string, f func(src image.Image, c
}
}
- b := converted.Bounds()
- ci.config = image.Config{Width: b.Max.X, Height: b.Max.Y}
- ci.configLoaded = true
+ ci := i.clone(converted)
+ ci.setBasePath(conf)
return ci, converted, nil
})
-
-}
-
-func (i imageConfig) key(format imaging.Format) string {
- k := strconv.Itoa(i.Width) + "x" + strconv.Itoa(i.Height)
- if i.Action != "" {
- k += "_" + i.Action
- }
- if i.Quality > 0 {
- k += "_q" + strconv.Itoa(i.Quality)
- }
- if i.Rotate != 0 {
- k += "_r" + strconv.Itoa(i.Rotate)
- }
- anchor := i.AnchorStr
- if anchor == smartCropIdentifier {
- anchor = anchor + strconv.Itoa(smartCropVersionNumber)
- }
-
- k += "_" + i.FilterStr
-
- if strings.EqualFold(i.Action, "fill") {
- k += "_" + anchor
- }
-
- if v, ok := imageFormatsVersions[format]; ok {
- k += "_" + strconv.Itoa(v)
- }
-
- if mainImageVersionNumber > 0 {
- k += "_" + strconv.Itoa(mainImageVersionNumber)
- }
-
- return k
-}
-
-func newImageConfig(width, height, quality, rotate int, filter, anchor string) imageConfig {
- var c imageConfig
-
- c.Width = width
- c.Height = height
- c.Quality = quality
- c.Rotate = rotate
-
- if filter != "" {
- filter = strings.ToLower(filter)
- if v, ok := imageFilters[filter]; ok {
- c.Filter = v
- c.FilterStr = filter
- }
- }
-
- if anchor != "" {
- anchor = strings.ToLower(anchor)
- if v, ok := anchorPositions[anchor]; ok {
- c.Anchor = v
- c.AnchorStr = anchor
- }
- }
-
- return c
}
-func parseImageConfig(config string) (imageConfig, error) {
- var (
- c imageConfig
- err error
- )
-
- if config == "" {
- return c, errors.New("image config cannot be empty")
- }
-
- parts := strings.Fields(config)
- for _, part := range parts {
- part = strings.ToLower(part)
-
- if part == smartCropIdentifier {
- c.AnchorStr = smartCropIdentifier
- } else if pos, ok := anchorPositions[part]; ok {
- c.Anchor = pos
- c.AnchorStr = part
- } else if filter, ok := imageFilters[part]; ok {
- c.Filter = filter
- c.FilterStr = part
- } else if part[0] == 'q' {
- c.Quality, err = strconv.Atoi(part[1:])
- if err != nil {
- return c, err
- }
- if c.Quality < 1 || c.Quality > 100 {
- return c, errors.New("quality ranges from 1 to 100 inclusive")
- }
- } else if part[0] == 'r' {
- c.Rotate, err = strconv.Atoi(part[1:])
- if err != nil {
- return c, err
- }
- } else if strings.Contains(part, "x") {
- widthHeight := strings.Split(part, "x")
- if len(widthHeight) <= 2 {
- first := widthHeight[0]
- if first != "" {
- c.Width, err = strconv.Atoi(first)
- if err != nil {
- return c, err
- }
- }
-
- if len(widthHeight) == 2 {
- second := widthHeight[1]
- if second != "" {
- c.Height, err = strconv.Atoi(second)
- if err != nil {
- return c, err
- }
- }
- }
- } else {
- return c, errors.New("invalid image dimensions")
- }
-
- }
- }
-
- if c.Width == 0 && c.Height == 0 {
- return c, errors.New("must provide Width or Height")
+func (i *imageResource) decodeImageConfig(action, spec string) (images.ImageConfig, error) {
+ conf, err := images.DecodeImageConfig(action, spec, i.Proc.Cfg)
+ if err != nil {
+ return conf, err
}
- return c, nil
-}
-
-func (i *Image) initConfig() error {
- var err error
- i.configInit.Do(func() {
- if i.configLoaded {
- return
- }
-
- var (
- f hugio.ReadSeekCloser
- config image.Config
- )
-
- f, err = i.ReadSeekCloser()
- if err != nil {
- return
- }
- defer f.Close()
-
- config, _, err = image.DecodeConfig(f)
- if err != nil {
- return
- }
- i.config = config
- })
+ iconf := i.Proc.Cfg
- if err != nil {
- return _errors.Wrap(err, "failed to load image config")
+ if conf.Quality <= 0 && i.isJPEG() {
+ // We need a quality setting for all JPEGs
+ conf.Quality = iconf.Quality
}
- return nil
+ return conf, nil
}
-func (i *Image) decodeSource() (image.Image, error) {
+func (i *imageResource) decodeSource() (image.Image, error) {
f, err := i.ReadSeekCloser()
if err != nil {
return nil, _errors.Wrap(err, "failed to open image for decode")
@@ -459,80 +190,39 @@ func (i *Image) decodeSource() (image.Image, error) {
return img, err
}
-// returns an opened file or nil if nothing to write.
-func (i *Image) openDestinationsForWriting() (io.WriteCloser, error) {
- targetFilenames := i.targetFilenames()
- var changedFilenames []string
-
- // Fast path:
- // This is a processed version of the original;
- // check if it already existis at the destination.
- for _, targetFilename := range targetFilenames {
- if _, err := i.spec.BaseFs.PublishFs.Stat(targetFilename); err == nil {
- continue
- }
- changedFilenames = append(changedFilenames, targetFilename)
- }
-
- if len(changedFilenames) == 0 {
- return nil, nil
- }
-
- return helpers.OpenFilesForWriting(i.spec.BaseFs.PublishFs, changedFilenames...)
-
-}
-
-func (i *Image) encodeTo(conf imageConfig, img image.Image, w io.Writer) error {
- switch i.format {
- case imaging.JPEG:
-
- var rgba *image.RGBA
- quality := conf.Quality
+func (i *imageResource) clone(img image.Image) *imageResource {
+ spec := i.baseResource.Clone().(baseResource)
- if nrgba, ok := img.(*image.NRGBA); ok {
- if nrgba.Opaque() {
- rgba = &image.RGBA{
- Pix: nrgba.Pix,
- Stride: nrgba.Stride,
- Rect: nrgba.Rect,
- }
- }
- }
- if rgba != nil {
- return jpeg.Encode(w, rgba, &jpeg.Options{Quality: quality})
- }
- return jpeg.Encode(w, img, &jpeg.Options{Quality: quality})
- default:
- return imaging.Encode(w, img, i.format)
+ var image *images.Image
+ if img != nil {
+ image = i.WithImage(img)
+ } else {
+ image = i.WithSpec(spec)
}
-}
-func (i *Image) clone() *Image {
- g := *i.genericResource
- g.resourceContent = &resourceContent{}
- if g.publishOnce != nil {
- g.publishOnce = &publishOnce{logger: g.publishOnce.logger}
+ return &imageResource{
+ Image: image,
+ baseResource: spec,
}
-
- return &Image{
- imaging: i.imaging,
- format: i.format,
- genericResource: &g}
}
-func (i *Image) setBasePath(conf imageConfig) {
- i.relTargetDirFile = i.relTargetPathFromConfig(conf)
+func (i *imageResource) setBasePath(conf images.ImageConfig) {
+ i.getResourcePaths().relTargetDirFile = i.relTargetPathFromConfig(conf)
}
-func (i *Image) relTargetPathFromConfig(conf imageConfig) dirFile {
- p1, p2 := helpers.FileAndExt(i.relTargetDirFile.file)
+func (i *imageResource) relTargetPathFromConfig(conf images.ImageConfig) dirFile {
+ p1, p2 := helpers.FileAndExt(i.getResourcePaths().relTargetDirFile.file)
+ if conf.Action == "trace" {
+ p2 = ".svg"
+ }
- idStr := fmt.Sprintf("_hu%s_%d", i.hash, i.osFileInfo.Size())
+ h, _ := i.hash()
+ idStr := fmt.Sprintf("_hu%s_%d", h, i.size())
// Do not change for no good reason.
const md5Threshold = 100
- key := conf.key(i.format)
+ key := conf.Key(i.Format)
// It is useful to have the key in clear text, but when nesting transforms, it
// can easily be too long to read, and maybe even too long
@@ -554,43 +244,7 @@ func (i *Image) relTargetPathFromConfig(conf imageConfig) dirFile {
}
return dirFile{
- dir: i.relTargetDirFile.dir,
+ dir: i.getResourcePaths().relTargetDirFile.dir,
file: fmt.Sprintf("%s%s_%s%s", p1, idStr, key, p2),
}
-
-}
-
-func decodeImaging(m map[string]interface{}) (Imaging, error) {
- var i Imaging
- if err := mapstructure.WeakDecode(m, &i); err != nil {
- return i, err
- }
-
- if i.Quality == 0 {
- i.Quality = defaultJPEGQuality
- } else if i.Quality < 0 || i.Quality > 100 {
- return i, errors.New("JPEG quality must be a number between 1 and 100")
- }
-
- if i.Anchor == "" || strings.EqualFold(i.Anchor, smartCropIdentifier) {
- i.Anchor = smartCropIdentifier
- } else {
- i.Anchor = strings.ToLower(i.Anchor)
- if _, found := anchorPositions[i.Anchor]; !found {
- return i, errors.New("invalid anchor value in imaging config")
- }
- }
-
- if i.ResampleFilter == "" {
- i.ResampleFilter = defaultResampleFilter
- } else {
- filter := strings.ToLower(i.ResampleFilter)
- _, found := imageFilters[filter]
- if !found {
- return i, fmt.Errorf("%q is not a valid resample filter", filter)
- }
- i.ResampleFilter = filter
- }
-
- return i, nil
}
DIR diff --git a/resources/image_cache.go b/resources/image_cache.go
@@ -20,7 +20,7 @@ import (
"strings"
"sync"
- "github.com/gohugoio/hugo/common/hugio"
+ "github.com/gohugoio/hugo/resources/images"
"github.com/gohugoio/hugo/cache/filecache"
"github.com/gohugoio/hugo/helpers"
@@ -32,7 +32,7 @@ type imageCache struct {
fileCache *filecache.Cache
mu sync.RWMutex
- store map[string]*Image
+ store map[string]*resourceAdapter
}
func (c *imageCache) isInCache(key string) bool {
@@ -66,33 +66,34 @@ func (c *imageCache) normalizeKey(key string) string {
func (c *imageCache) clear() {
c.mu.Lock()
defer c.mu.Unlock()
- c.store = make(map[string]*Image)
+ c.store = make(map[string]*resourceAdapter)
}
func (c *imageCache) getOrCreate(
- parent *Image, conf imageConfig, createImage func() (*Image, image.Image, error)) (*Image, error) {
-
+ parent *imageResource, conf images.ImageConfig,
+ createImage func() (*imageResource, image.Image, error)) (*resourceAdapter, error) {
relTarget := parent.relTargetPathFromConfig(conf)
key := parent.relTargetPathForRel(relTarget.path(), false, false, false)
// First check the in-memory store, then the disk.
c.mu.RLock()
- img, found := c.store[key]
+ cachedImage, found := c.store[key]
c.mu.RUnlock()
if found {
- return img, nil
+ return cachedImage, nil
}
+ var img *imageResource
+
// These funcs are protected by a named lock.
// read clones the parent to its new name and copies
// the content to the destinations.
read := func(info filecache.ItemInfo, r io.Reader) error {
- img = parent.clone()
- img.relTargetDirFile.file = relTarget.file
- img.sourceFilename = info.Name
- // Make sure it's always loaded by sourceFilename.
- img.openReadSeekerCloser = nil
+ img = parent.clone(nil)
+ rp := img.getResourcePaths()
+ rp.relTargetDirFile.file = relTarget.file
+ img.setSourceFilename(info.Name)
w, err := img.openDestinationsForWriting()
if err != nil {
@@ -109,29 +110,20 @@ func (c *imageCache) getOrCreate(
return err
}
- // create creates the image and encodes it to w (cache) and to its destinations.
+ // create creates the image and encodes it to the cache (w).
create := func(info filecache.ItemInfo, w io.WriteCloser) (err error) {
+ defer w.Close()
+
var conv image.Image
img, conv, err = createImage()
if err != nil {
- w.Close()
return
}
- img.relTargetDirFile.file = relTarget.file
- img.sourceFilename = info.Name
+ rp := img.getResourcePaths()
+ rp.relTargetDirFile.file = relTarget.file
+ img.setSourceFilename(info.Name)
- destinations, err := img.openDestinationsForWriting()
- if err != nil {
- w.Close()
- return err
- }
-
- if destinations != nil {
- w = hugio.NewMultiWriteCloser(w, destinations)
- }
- defer w.Close()
-
- return img.encodeTo(conf, conv, w)
+ return img.EncodeTo(conf, conv, w)
}
// Now look in the file cache.
@@ -147,20 +139,21 @@ func (c *imageCache) getOrCreate(
}
// The file is now stored in this cache.
- img.sourceFs = c.fileCache.Fs
+ img.setSourceFs(c.fileCache.Fs)
c.mu.Lock()
- if img2, found := c.store[key]; found {
+ if cachedImage, found = c.store[key]; found {
c.mu.Unlock()
- return img2, nil
+ return cachedImage, nil
}
- c.store[key] = img
- c.mu.Unlock()
- return img, nil
+ imgAdapter := newResourceAdapter(parent.getSpec(), true, img)
+ c.store[key] = imgAdapter
+ c.mu.Unlock()
+ return imgAdapter, nil
}
func newImageCache(fileCache *filecache.Cache, ps *helpers.PathSpec) *imageCache {
- return &imageCache{fileCache: fileCache, pathSpec: ps, store: make(map[string]*Image)}
+ return &imageCache{fileCache: fileCache, pathSpec: ps, store: make(map[string]*resourceAdapter)}
}
DIR diff --git a/resources/image_test.go b/resources/image_test.go
@@ -18,121 +18,101 @@ import (
"math/rand"
"path/filepath"
"strconv"
+ "sync"
"testing"
- "github.com/gohugoio/hugo/htesting/hqt"
+ "github.com/gohugoio/hugo/media"
+ "github.com/gohugoio/hugo/resources/resource"
- "github.com/disintegration/imaging"
+ "github.com/google/go-cmp/cmp"
- "sync"
+ "github.com/gohugoio/hugo/htesting/hqt"
qt "github.com/frankban/quicktest"
)
-func TestParseImageConfig(t *testing.T) {
- for i, this := range []struct {
- in string
- expect interface{}
- }{
- {"300x400", newImageConfig(300, 400, 0, 0, "", "")},
- {"100x200 bottomRight", newImageConfig(100, 200, 0, 0, "", "BottomRight")},
- {"10x20 topleft Lanczos", newImageConfig(10, 20, 0, 0, "Lanczos", "topleft")},
- {"linear left 10x r180", newImageConfig(10, 0, 0, 180, "linear", "left")},
- {"x20 riGht Cosine q95", newImageConfig(0, 20, 95, 0, "cosine", "right")},
-
- {"", false},
- {"foo", false},
- } {
- result, err := parseImageConfig(this.in)
- if b, ok := this.expect.(bool); ok && !b {
- if err == nil {
- t.Errorf("[%d] parseImageConfig didn't return an expected error", i)
- }
- } else {
- if err != nil {
- t.Fatalf("[%d] err: %s", i, err)
- }
- if fmt.Sprint(result) != fmt.Sprint(this.expect) {
- t.Fatalf("[%d] got\n%v\n but expected\n%v", i, result, this.expect)
- }
- }
- }
-}
+var eq = qt.CmpEquals(
+ cmp.Comparer(func(p1, p2 *resourceAdapter) bool {
+ return p1.resourceAdapterInner == p2.resourceAdapterInner
+ }),
+ cmp.Comparer(func(p1, p2 *genericResource) bool { return p1 == p2 }),
+ cmp.Comparer(func(m1, m2 media.Type) bool {
+ return m1.Type() == m2.Type()
+ }),
+)
func TestImageTransformBasic(t *testing.T) {
-
c := qt.New(t)
image := fetchSunset(c)
- fileCache := image.spec.FileCaches.ImageCache().Fs
+
+ fileCache := image.(specProvider).getSpec().FileCaches.ImageCache().Fs
+
+ assertWidthHeight := func(img resource.Image, w, h int) {
+ c.Helper()
+ c.Assert(img, qt.Not(qt.IsNil))
+ c.Assert(img.Width(), qt.Equals, w)
+ c.Assert(img.Height(), qt.Equals, h)
+ }
c.Assert(image.RelPermalink(), qt.Equals, "/a/sunset.jpg")
c.Assert(image.ResourceType(), qt.Equals, "image")
+ assertWidthHeight(image, 900, 562)
resized, err := image.Resize("300x200")
c.Assert(err, qt.IsNil)
c.Assert(image != resized, qt.Equals, true)
- c.Assert(image.genericResource != resized.genericResource, qt.Equals, true)
- c.Assert(image.sourceFilename != resized.sourceFilename, qt.Equals, true)
+ c.Assert(image, qt.Not(eq), resized)
+ assertWidthHeight(resized, 300, 200)
+ assertWidthHeight(image, 900, 562)
resized0x, err := image.Resize("x200")
c.Assert(err, qt.IsNil)
- c.Assert(resized0x.Width(), qt.Equals, 320)
- c.Assert(resized0x.Height(), qt.Equals, 200)
-
+ assertWidthHeight(resized0x, 320, 200)
assertFileCache(c, fileCache, resized0x.RelPermalink(), 320, 200)
resizedx0, err := image.Resize("200x")
c.Assert(err, qt.IsNil)
- c.Assert(resizedx0.Width(), qt.Equals, 200)
- c.Assert(resizedx0.Height(), qt.Equals, 125)
+ assertWidthHeight(resizedx0, 200, 125)
assertFileCache(c, fileCache, resizedx0.RelPermalink(), 200, 125)
resizedAndRotated, err := image.Resize("x200 r90")
c.Assert(err, qt.IsNil)
- c.Assert(resizedAndRotated.Width(), qt.Equals, 125)
- c.Assert(resizedAndRotated.Height(), qt.Equals, 200)
+ assertWidthHeight(resizedAndRotated, 125, 200)
assertFileCache(c, fileCache, resizedAndRotated.RelPermalink(), 125, 200)
+ assertWidthHeight(resized, 300, 200)
c.Assert(resized.RelPermalink(), qt.Equals, "/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_300x200_resize_q68_linear.jpg")
- c.Assert(resized.Width(), qt.Equals, 300)
- c.Assert(resized.Height(), qt.Equals, 200)
fitted, err := resized.Fit("50x50")
c.Assert(err, qt.IsNil)
c.Assert(fitted.RelPermalink(), qt.Equals, "/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_625708021e2bb281c9f1002f88e4753f.jpg")
- c.Assert(fitted.Width(), qt.Equals, 50)
- c.Assert(fitted.Height(), qt.Equals, 33)
+ assertWidthHeight(fitted, 50, 33)
// Check the MD5 key threshold
fittedAgain, _ := fitted.Fit("10x20")
fittedAgain, err = fittedAgain.Fit("10x20")
c.Assert(err, qt.IsNil)
c.Assert(fittedAgain.RelPermalink(), qt.Equals, "/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_3f65ba24dc2b7fba0f56d7f104519157.jpg")
- c.Assert(fittedAgain.Width(), qt.Equals, 10)
- c.Assert(fittedAgain.Height(), qt.Equals, 6)
+ assertWidthHeight(fittedAgain, 10, 6)
filled, err := image.Fill("200x100 bottomLeft")
c.Assert(err, qt.IsNil)
c.Assert(filled.RelPermalink(), qt.Equals, "/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_200x100_fill_q68_linear_bottomleft.jpg")
- c.Assert(filled.Width(), qt.Equals, 200)
- c.Assert(filled.Height(), qt.Equals, 100)
+ assertWidthHeight(filled, 200, 100)
assertFileCache(c, fileCache, filled.RelPermalink(), 200, 100)
smart, err := image.Fill("200x100 smart")
c.Assert(err, qt.IsNil)
- c.Assert(smart.RelPermalink(), qt.Equals, fmt.Sprintf("/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_200x100_fill_q68_linear_smart%d.jpg", smartCropVersionNumber))
- c.Assert(smart.Width(), qt.Equals, 200)
- c.Assert(smart.Height(), qt.Equals, 100)
+ c.Assert(smart.RelPermalink(), qt.Equals, fmt.Sprintf("/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_200x100_fill_q68_linear_smart%d.jpg", 1))
+ assertWidthHeight(smart, 200, 100)
assertFileCache(c, fileCache, smart.RelPermalink(), 200, 100)
// Check cache
filledAgain, err := image.Fill("200x100 bottomLeft")
c.Assert(err, qt.IsNil)
- c.Assert(filled == filledAgain, qt.Equals, true)
- c.Assert(filled.sourceFilename == filledAgain.sourceFilename, qt.Equals, true)
+ c.Assert(filled, eq, filledAgain)
assertFileCache(c, fileCache, filledAgain.RelPermalink(), 200, 100)
-
}
// https://github.com/gohugoio/hugo/issues/4261
@@ -158,6 +138,7 @@ func TestImageTransformLongFilename(t *testing.T) {
func TestImageTransformUppercaseExt(t *testing.T) {
c := qt.New(t)
image := fetchImage(c, "sunrise.JPG")
+
resized, err := image.Resize("200x")
c.Assert(err, qt.IsNil)
c.Assert(resized, qt.Not(qt.IsNil))
@@ -173,17 +154,16 @@ func TestImagePermalinkPublishOrder(t *testing.T) {
}
t.Run(name, func(t *testing.T) {
-
c := qt.New(t)
spec := newTestResourceOsFs(c)
- check1 := func(img *Image) {
+ check1 := func(img resource.Image) {
resizedLink := "/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_100x50_resize_q75_box.jpg"
c.Assert(img.RelPermalink(), qt.Equals, resizedLink)
assertImageFile(c, spec.PublishFs, resizedLink, 100, 50)
}
- check2 := func(img *Image) {
+ check2 := func(img resource.Image) {
c.Assert(img.RelPermalink(), qt.Equals, "/a/sunset.jpg")
assertImageFile(c, spec.PublishFs, "a/sunset.jpg", 900, 562)
}
@@ -198,18 +178,16 @@ func TestImagePermalinkPublishOrder(t *testing.T) {
resized, err := orignal.Resize("100x50")
c.Assert(err, qt.IsNil)
- check1(resized)
+ check1(resized.(resource.Image))
if !checkOriginalFirst {
check2(orignal)
}
})
}
-
}
func TestImageTransformConcurrent(t *testing.T) {
-
var wg sync.WaitGroup
c := qt.New(t)
@@ -239,12 +217,7 @@ func TestImageTransformConcurrent(t *testing.T) {
t.Error(err)
}
- _, err = r2.decodeSource()
- if err != nil {
- t.Error("Err decode:", err)
- }
-
- img = r1
+ img = r2
}
}
}(i + 20)
@@ -253,58 +226,12 @@ func TestImageTransformConcurrent(t *testing.T) {
wg.Wait()
}
-func TestDecodeImaging(t *testing.T) {
- c := qt.New(t)
- m := map[string]interface{}{
- "quality": 42,
- "resampleFilter": "NearestNeighbor",
- "anchor": "topLeft",
- }
-
- imaging, err := decodeImaging(m)
-
- c.Assert(err, qt.IsNil)
- c.Assert(imaging.Quality, qt.Equals, 42)
- c.Assert(imaging.ResampleFilter, qt.Equals, "nearestneighbor")
- c.Assert(imaging.Anchor, qt.Equals, "topleft")
-
- m = map[string]interface{}{}
-
- imaging, err = decodeImaging(m)
- c.Assert(err, qt.IsNil)
- c.Assert(imaging.Quality, qt.Equals, defaultJPEGQuality)
- c.Assert(imaging.ResampleFilter, qt.Equals, "box")
- c.Assert(imaging.Anchor, qt.Equals, "smart")
-
- _, err = decodeImaging(map[string]interface{}{
- "quality": 123,
- })
- c.Assert(err, qt.Not(qt.IsNil))
-
- _, err = decodeImaging(map[string]interface{}{
- "resampleFilter": "asdf",
- })
- c.Assert(err, qt.Not(qt.IsNil))
-
- _, err = decodeImaging(map[string]interface{}{
- "anchor": "asdf",
- })
- c.Assert(err, qt.Not(qt.IsNil))
-
- imaging, err = decodeImaging(map[string]interface{}{
- "anchor": "Smart",
- })
- c.Assert(err, qt.IsNil)
- c.Assert(imaging.Anchor, qt.Equals, "smart")
-
-}
-
func TestImageWithMetadata(t *testing.T) {
c := qt.New(t)
image := fetchSunset(c)
- var meta = []map[string]interface{}{
+ meta := []map[string]interface{}{
{
"title": "My Sunset",
"name": "Sunset #:counter",
@@ -318,71 +245,69 @@ func TestImageWithMetadata(t *testing.T) {
resized, err := image.Resize("200x")
c.Assert(err, qt.IsNil)
c.Assert(resized.Name(), qt.Equals, "Sunset #1")
-
}
func TestImageResize8BitPNG(t *testing.T) {
-
c := qt.New(t)
image := fetchImage(c, "gohugoio.png")
- c.Assert(image.format, qt.Equals, imaging.PNG)
+ c.Assert(image.MediaType().Type(), qt.Equals, "image/png")
c.Assert(image.RelPermalink(), qt.Equals, "/a/gohugoio.png")
c.Assert(image.ResourceType(), qt.Equals, "image")
resized, err := image.Resize("800x")
c.Assert(err, qt.IsNil)
- c.Assert(resized.format, qt.Equals, imaging.PNG)
+ c.Assert(resized.MediaType().Type(), qt.Equals, "image/png")
c.Assert(resized.RelPermalink(), qt.Equals, "/a/gohugoio_hu0e1b9e4a4be4d6f86c7b37b9ccce3fbc_73886_800x0_resize_linear_2.png")
c.Assert(resized.Width(), qt.Equals, 800)
-
}
func TestImageResizeInSubPath(t *testing.T) {
-
c := qt.New(t)
image := fetchImage(c, "sub/gohugoio2.png")
- fileCache := image.spec.FileCaches.ImageCache().Fs
+ fileCache := image.(specProvider).getSpec().FileCaches.ImageCache().Fs
- c.Assert(image.format, qt.Equals, imaging.PNG)
+ c.Assert(image.MediaType(), eq, media.PNGType)
c.Assert(image.RelPermalink(), qt.Equals, "/a/sub/gohugoio2.png")
c.Assert(image.ResourceType(), qt.Equals, "image")
resized, err := image.Resize("101x101")
c.Assert(err, qt.IsNil)
- c.Assert(resized.format, qt.Equals, imaging.PNG)
+ c.Assert(resized.MediaType().Type(), qt.Equals, "image/png")
c.Assert(resized.RelPermalink(), qt.Equals, "/a/sub/gohugoio2_hu0e1b9e4a4be4d6f86c7b37b9ccce3fbc_73886_101x101_resize_linear_2.png")
c.Assert(resized.Width(), qt.Equals, 101)
assertFileCache(c, fileCache, resized.RelPermalink(), 101, 101)
publishedImageFilename := filepath.Clean(resized.RelPermalink())
- assertImageFile(c, image.spec.BaseFs.PublishFs, publishedImageFilename, 101, 101)
- c.Assert(image.spec.BaseFs.PublishFs.Remove(publishedImageFilename), qt.IsNil)
+
+ spec := image.(specProvider).getSpec()
+
+ assertImageFile(c, spec.BaseFs.PublishFs, publishedImageFilename, 101, 101)
+ c.Assert(spec.BaseFs.PublishFs.Remove(publishedImageFilename), qt.IsNil)
// Cleare mem cache to simulate reading from the file cache.
- resized.spec.imageCache.clear()
+ spec.imageCache.clear()
resizedAgain, err := image.Resize("101x101")
c.Assert(err, qt.IsNil)
c.Assert(resizedAgain.RelPermalink(), qt.Equals, "/a/sub/gohugoio2_hu0e1b9e4a4be4d6f86c7b37b9ccce3fbc_73886_101x101_resize_linear_2.png")
c.Assert(resizedAgain.Width(), qt.Equals, 101)
assertFileCache(c, fileCache, resizedAgain.RelPermalink(), 101, 101)
- assertImageFile(c, image.spec.BaseFs.PublishFs, publishedImageFilename, 101, 101)
-
+ assertImageFile(c, image.(specProvider).getSpec().BaseFs.PublishFs, publishedImageFilename, 101, 101)
}
func TestSVGImage(t *testing.T) {
c := qt.New(t)
- spec := newTestResourceSpec(c)
+ spec := newTestResourceSpec(specDescriptor{c: c})
svg := fetchResourceForSpec(spec, c, "circle.svg")
c.Assert(svg, qt.Not(qt.IsNil))
}
func TestSVGImageContent(t *testing.T) {
c := qt.New(t)
- spec := newTestResourceSpec(c)
+ spec := newTestResourceSpec(specDescriptor{c: c})
svg := fetchResourceForSpec(spec, c, "circle.svg")
c.Assert(svg, qt.Not(qt.IsNil))
DIR diff --git a/resources/images/config.go b/resources/images/config.go
@@ -0,0 +1,276 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package images
+
+import (
+ "errors"
+ "fmt"
+ "strconv"
+ "strings"
+
+ "github.com/disintegration/imaging"
+ "github.com/mitchellh/mapstructure"
+)
+
+const (
+ defaultJPEGQuality = 75
+ defaultResampleFilter = "box"
+)
+
+var (
+ imageFormats = map[string]imaging.Format{
+ ".jpg": imaging.JPEG,
+ ".jpeg": imaging.JPEG,
+ ".png": imaging.PNG,
+ ".tif": imaging.TIFF,
+ ".tiff": imaging.TIFF,
+ ".bmp": imaging.BMP,
+ ".gif": imaging.GIF,
+ }
+
+ // Add or increment if changes to an image format's processing requires
+ // re-generation.
+ imageFormatsVersions = map[imaging.Format]int{
+ imaging.PNG: 2, // Floyd Steinberg dithering
+ }
+
+ // Increment to mark all processed images as stale. Only use when absolutely needed.
+ // See the finer grained smartCropVersionNumber and imageFormatsVersions.
+ mainImageVersionNumber = 0
+
+ // Increment to mark all traced SVGs as stale.
+ traceVersionNumber = 0
+)
+
+var anchorPositions = map[string]imaging.Anchor{
+ strings.ToLower("Center"): imaging.Center,
+ strings.ToLower("TopLeft"): imaging.TopLeft,
+ strings.ToLower("Top"): imaging.Top,
+ strings.ToLower("TopRight"): imaging.TopRight,
+ strings.ToLower("Left"): imaging.Left,
+ strings.ToLower("Right"): imaging.Right,
+ strings.ToLower("BottomLeft"): imaging.BottomLeft,
+ strings.ToLower("Bottom"): imaging.Bottom,
+ strings.ToLower("BottomRight"): imaging.BottomRight,
+}
+
+var imageFilters = map[string]imaging.ResampleFilter{
+ strings.ToLower("NearestNeighbor"): imaging.NearestNeighbor,
+ strings.ToLower("Box"): imaging.Box,
+ strings.ToLower("Linear"): imaging.Linear,
+ strings.ToLower("Hermite"): imaging.Hermite,
+ strings.ToLower("MitchellNetravali"): imaging.MitchellNetravali,
+ strings.ToLower("CatmullRom"): imaging.CatmullRom,
+ strings.ToLower("BSpline"): imaging.BSpline,
+ strings.ToLower("Gaussian"): imaging.Gaussian,
+ strings.ToLower("Lanczos"): imaging.Lanczos,
+ strings.ToLower("Hann"): imaging.Hann,
+ strings.ToLower("Hamming"): imaging.Hamming,
+ strings.ToLower("Blackman"): imaging.Blackman,
+ strings.ToLower("Bartlett"): imaging.Bartlett,
+ strings.ToLower("Welch"): imaging.Welch,
+ strings.ToLower("Cosine"): imaging.Cosine,
+}
+
+func ImageFormatFromExt(ext string) (imaging.Format, bool) {
+ f, found := imageFormats[ext]
+ return f, found
+}
+
+func DecodeConfig(m map[string]interface{}) (Imaging, error) {
+ var i Imaging
+ if err := mapstructure.WeakDecode(m, &i); err != nil {
+ return i, err
+ }
+
+ if i.Quality == 0 {
+ i.Quality = defaultJPEGQuality
+ } else if i.Quality < 0 || i.Quality > 100 {
+ return i, errors.New("JPEG quality must be a number between 1 and 100")
+ }
+
+ if i.Anchor == "" || strings.EqualFold(i.Anchor, SmartCropIdentifier) {
+ i.Anchor = SmartCropIdentifier
+ } else {
+ i.Anchor = strings.ToLower(i.Anchor)
+ if _, found := anchorPositions[i.Anchor]; !found {
+ return i, errors.New("invalid anchor value in imaging config")
+ }
+ }
+
+ if i.ResampleFilter == "" {
+ i.ResampleFilter = defaultResampleFilter
+ } else {
+ filter := strings.ToLower(i.ResampleFilter)
+ _, found := imageFilters[filter]
+ if !found {
+ return i, fmt.Errorf("%q is not a valid resample filter", filter)
+ }
+ i.ResampleFilter = filter
+ }
+
+ return i, nil
+}
+
+func DecodeImageConfig(action, config string, defaults Imaging) (ImageConfig, error) {
+ var (
+ c ImageConfig
+ err error
+ )
+
+ c.Action = action
+
+ if config == "" {
+ return c, errors.New("image config cannot be empty")
+ }
+
+ parts := strings.Fields(config)
+ for _, part := range parts {
+ part = strings.ToLower(part)
+
+ if part == SmartCropIdentifier {
+ c.AnchorStr = SmartCropIdentifier
+ } else if pos, ok := anchorPositions[part]; ok {
+ c.Anchor = pos
+ c.AnchorStr = part
+ } else if filter, ok := imageFilters[part]; ok {
+ c.Filter = filter
+ c.FilterStr = part
+ } else if part[0] == 'q' {
+ c.Quality, err = strconv.Atoi(part[1:])
+ if err != nil {
+ return c, err
+ }
+ if c.Quality < 1 || c.Quality > 100 {
+ return c, errors.New("quality ranges from 1 to 100 inclusive")
+ }
+ } else if part[0] == 'r' {
+ c.Rotate, err = strconv.Atoi(part[1:])
+ if err != nil {
+ return c, err
+ }
+ } else if strings.Contains(part, "x") {
+ widthHeight := strings.Split(part, "x")
+ if len(widthHeight) <= 2 {
+ first := widthHeight[0]
+ if first != "" {
+ c.Width, err = strconv.Atoi(first)
+ if err != nil {
+ return c, err
+ }
+ }
+
+ if len(widthHeight) == 2 {
+ second := widthHeight[1]
+ if second != "" {
+ c.Height, err = strconv.Atoi(second)
+ if err != nil {
+ return c, err
+ }
+ }
+ }
+ } else {
+ return c, errors.New("invalid image dimensions")
+ }
+
+ }
+ }
+
+ if c.Width == 0 && c.Height == 0 {
+ return c, errors.New("must provide Width or Height")
+ }
+
+ if c.FilterStr == "" {
+ c.FilterStr = defaults.ResampleFilter
+ c.Filter = imageFilters[c.FilterStr]
+ }
+
+ if c.AnchorStr == "" {
+ c.AnchorStr = defaults.Anchor
+ if !strings.EqualFold(c.AnchorStr, SmartCropIdentifier) {
+ c.Anchor = anchorPositions[c.AnchorStr]
+ }
+ }
+
+ return c, nil
+}
+
+// ImageConfig holds configuration to create a new image from an existing one, resize etc.
+type ImageConfig struct {
+ Action string
+
+ // Quality ranges from 1 to 100 inclusive, higher is better.
+ // This is only relevant for JPEG images.
+ // Default is 75.
+ Quality int
+
+ // Rotate rotates an image by the given angle counter-clockwise.
+ // The rotation will be performed first.
+ Rotate int
+
+ Width int
+ Height int
+
+ Filter imaging.ResampleFilter
+ FilterStr string
+
+ Anchor imaging.Anchor
+ AnchorStr string
+}
+
+func (i ImageConfig) Key(format imaging.Format) string {
+ k := strconv.Itoa(i.Width) + "x" + strconv.Itoa(i.Height)
+ if i.Action != "" {
+ k += "_" + i.Action
+ }
+ if i.Quality > 0 {
+ k += "_q" + strconv.Itoa(i.Quality)
+ }
+ if i.Rotate != 0 {
+ k += "_r" + strconv.Itoa(i.Rotate)
+ }
+ anchor := i.AnchorStr
+ if anchor == SmartCropIdentifier {
+ anchor = anchor + strconv.Itoa(smartCropVersionNumber)
+ }
+
+ k += "_" + i.FilterStr
+
+ if strings.EqualFold(i.Action, "fill") {
+ k += "_" + anchor
+ }
+
+ if v, ok := imageFormatsVersions[format]; ok {
+ k += "_" + strconv.Itoa(v)
+ }
+
+ if mainImageVersionNumber > 0 {
+ k += "_" + strconv.Itoa(mainImageVersionNumber)
+ }
+
+ return k
+}
+
+// Imaging contains default image processing configuration. This will be fetched
+// from site (or language) config.
+type Imaging struct {
+ // Default image quality setting (1-100). Only used for JPEG images.
+ Quality int
+
+ // Resample filter used. See https://github.com/disintegration/imaging
+ ResampleFilter string
+
+ // The anchor used in Fill. Default is "smart", i.e. Smart Crop.
+ Anchor string
+}
DIR diff --git a/resources/images/config_test.go b/resources/images/config_test.go
@@ -0,0 +1,125 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package images
+
+import (
+ "fmt"
+ "strings"
+ "testing"
+
+ qt "github.com/frankban/quicktest"
+)
+
+func TestDecodeConfig(t *testing.T) {
+ c := qt.New(t)
+ m := map[string]interface{}{
+ "quality": 42,
+ "resampleFilter": "NearestNeighbor",
+ "anchor": "topLeft",
+ }
+
+ imaging, err := DecodeConfig(m)
+
+ c.Assert(err, qt.IsNil)
+ c.Assert(imaging.Quality, qt.Equals, 42)
+ c.Assert(imaging.ResampleFilter, qt.Equals, "nearestneighbor")
+ c.Assert(imaging.Anchor, qt.Equals, "topleft")
+
+ m = map[string]interface{}{}
+
+ imaging, err = DecodeConfig(m)
+ c.Assert(err, qt.IsNil)
+ c.Assert(imaging.Quality, qt.Equals, defaultJPEGQuality)
+ c.Assert(imaging.ResampleFilter, qt.Equals, "box")
+ c.Assert(imaging.Anchor, qt.Equals, "smart")
+
+ _, err = DecodeConfig(map[string]interface{}{
+ "quality": 123,
+ })
+ c.Assert(err, qt.Not(qt.IsNil))
+
+ _, err = DecodeConfig(map[string]interface{}{
+ "resampleFilter": "asdf",
+ })
+ c.Assert(err, qt.Not(qt.IsNil))
+
+ _, err = DecodeConfig(map[string]interface{}{
+ "anchor": "asdf",
+ })
+ c.Assert(err, qt.Not(qt.IsNil))
+
+ imaging, err = DecodeConfig(map[string]interface{}{
+ "anchor": "Smart",
+ })
+ c.Assert(err, qt.IsNil)
+ c.Assert(imaging.Anchor, qt.Equals, "smart")
+}
+
+func TestDecodeImageConfig(t *testing.T) {
+ for i, this := range []struct {
+ in string
+ expect interface{}
+ }{
+ {"300x400", newImageConfig(300, 400, 0, 0, "", "")},
+ {"100x200 bottomRight", newImageConfig(100, 200, 0, 0, "", "BottomRight")},
+ {"10x20 topleft Lanczos", newImageConfig(10, 20, 0, 0, "Lanczos", "topleft")},
+ {"linear left 10x r180", newImageConfig(10, 0, 0, 180, "linear", "left")},
+ {"x20 riGht Cosine q95", newImageConfig(0, 20, 95, 0, "cosine", "right")},
+
+ {"", false},
+ {"foo", false},
+ } {
+
+ result, err := DecodeImageConfig("resize", this.in, Imaging{})
+ if b, ok := this.expect.(bool); ok && !b {
+ if err == nil {
+ t.Errorf("[%d] parseImageConfig didn't return an expected error", i)
+ }
+ } else {
+ if err != nil {
+ t.Fatalf("[%d] err: %s", i, err)
+ }
+ if fmt.Sprint(result) != fmt.Sprint(this.expect) {
+ t.Fatalf("[%d] got\n%v\n but expected\n%v", i, result, this.expect)
+ }
+ }
+ }
+}
+
+func newImageConfig(width, height, quality, rotate int, filter, anchor string) ImageConfig {
+ var c ImageConfig
+ c.Action = "resize"
+ c.Width = width
+ c.Height = height
+ c.Quality = quality
+ c.Rotate = rotate
+
+ if filter != "" {
+ filter = strings.ToLower(filter)
+ if v, ok := imageFilters[filter]; ok {
+ c.Filter = v
+ c.FilterStr = filter
+ }
+ }
+
+ if anchor != "" {
+ anchor = strings.ToLower(anchor)
+ if v, ok := anchorPositions[anchor]; ok {
+ c.Anchor = v
+ c.AnchorStr = anchor
+ }
+ }
+
+ return c
+}
DIR diff --git a/resources/images/image.go b/resources/images/image.go
@@ -0,0 +1,170 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package images
+
+import (
+ "image"
+ "image/jpeg"
+ "io"
+ "sync"
+
+ "github.com/disintegration/imaging"
+ "github.com/gohugoio/hugo/common/hugio"
+ "github.com/pkg/errors"
+)
+
+func NewImage(f imaging.Format, proc *ImageProcessor, img image.Image, s Spec) *Image {
+ if img != nil {
+ return &Image{
+ Format: f,
+ Proc: proc,
+ Spec: s,
+ imageConfig: &imageConfig{
+ config: imageConfigFromImage(img),
+ configLoaded: true,
+ },
+ }
+ }
+ return &Image{Format: f, Proc: proc, Spec: s, imageConfig: &imageConfig{}}
+}
+
+type Image struct {
+ Format imaging.Format
+
+ Proc *ImageProcessor
+
+ Spec Spec
+
+ *imageConfig
+}
+
+func (i *Image) EncodeTo(conf ImageConfig, img image.Image, w io.Writer) error {
+ switch i.Format {
+ case imaging.JPEG:
+
+ var rgba *image.RGBA
+ quality := conf.Quality
+
+ if nrgba, ok := img.(*image.NRGBA); ok {
+ if nrgba.Opaque() {
+ rgba = &image.RGBA{
+ Pix: nrgba.Pix,
+ Stride: nrgba.Stride,
+ Rect: nrgba.Rect,
+ }
+ }
+ }
+ if rgba != nil {
+ return jpeg.Encode(w, rgba, &jpeg.Options{Quality: quality})
+ }
+ return jpeg.Encode(w, img, &jpeg.Options{Quality: quality})
+ default:
+ return imaging.Encode(w, img, i.Format)
+ }
+}
+
+// Height returns i's height.
+func (i *Image) Height() int {
+ i.initConfig()
+ return i.config.Height
+}
+
+// Width returns i's width.
+func (i *Image) Width() int {
+ i.initConfig()
+ return i.config.Width
+}
+
+func (i Image) WithImage(img image.Image) *Image {
+ i.Spec = nil
+ i.imageConfig = &imageConfig{
+ config: imageConfigFromImage(img),
+ configLoaded: true,
+ }
+
+ return &i
+}
+
+func (i Image) WithSpec(s Spec) *Image {
+ i.Spec = s
+ i.imageConfig = &imageConfig{}
+ return &i
+}
+
+func (i *Image) initConfig() error {
+ var err error
+ i.configInit.Do(func() {
+ if i.configLoaded {
+ return
+ }
+
+ var (
+ f hugio.ReadSeekCloser
+ config image.Config
+ )
+
+ f, err = i.Spec.ReadSeekCloser()
+ if err != nil {
+ return
+ }
+ defer f.Close()
+
+ config, _, err = image.DecodeConfig(f)
+ if err != nil {
+ return
+ }
+ i.config = config
+ })
+
+ if err != nil {
+ return errors.Wrap(err, "failed to load image config")
+ }
+
+ return nil
+}
+
+type ImageProcessor struct {
+ Cfg Imaging
+}
+
+func (p *ImageProcessor) Fill(src image.Image, conf ImageConfig) (image.Image, error) {
+ if conf.AnchorStr == SmartCropIdentifier {
+ return smartCrop(src, conf.Width, conf.Height, conf.Anchor, conf.Filter)
+ }
+ return imaging.Fill(src, conf.Width, conf.Height, conf.Anchor, conf.Filter), nil
+}
+
+func (p *ImageProcessor) Fit(src image.Image, conf ImageConfig) (image.Image, error) {
+ return imaging.Fit(src, conf.Width, conf.Height, conf.Filter), nil
+}
+
+func (p *ImageProcessor) Resize(src image.Image, conf ImageConfig) (image.Image, error) {
+ return imaging.Resize(src, conf.Width, conf.Height, conf.Filter), nil
+}
+
+type Spec interface {
+ // Loads the image source.
+ ReadSeekCloser() (hugio.ReadSeekCloser, error)
+}
+
+type imageConfig struct {
+ config image.Config
+ configInit sync.Once
+ configLoaded bool
+}
+
+func imageConfigFromImage(img image.Image) image.Config {
+ b := img.Bounds()
+ return image.Config{Width: b.Max.X, Height: b.Max.Y}
+}
DIR diff --git a/resources/images/smartcrop.go b/resources/images/smartcrop.go
@@ -0,0 +1,75 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package images
+
+import (
+ "image"
+
+ "github.com/disintegration/imaging"
+ "github.com/muesli/smartcrop"
+)
+
+const (
+ // Do not change.
+ // TODO(bep) image unexport
+ SmartCropIdentifier = "smart"
+
+ // This is just a increment, starting on 1. If Smart Crop improves its cropping, we
+ // need a way to trigger a re-generation of the crops in the wild, so increment this.
+ smartCropVersionNumber = 1
+)
+
+func newSmartCropAnalyzer(filter imaging.ResampleFilter) smartcrop.Analyzer {
+ return smartcrop.NewAnalyzer(imagingResizer{filter: filter})
+}
+
+// Needed by smartcrop
+type imagingResizer struct {
+ filter imaging.ResampleFilter
+}
+
+func (r imagingResizer) Resize(img image.Image, width, height uint) image.Image {
+ return imaging.Resize(img, int(width), int(height), r.filter)
+}
+
+func smartCrop(img image.Image, width, height int, anchor imaging.Anchor, filter imaging.ResampleFilter) (*image.NRGBA, error) {
+ if width <= 0 || height <= 0 {
+ return &image.NRGBA{}, nil
+ }
+
+ srcBounds := img.Bounds()
+ srcW := srcBounds.Dx()
+ srcH := srcBounds.Dy()
+
+ if srcW <= 0 || srcH <= 0 {
+ return &image.NRGBA{}, nil
+ }
+
+ if srcW == width && srcH == height {
+ return imaging.Clone(img), nil
+ }
+
+ smart := newSmartCropAnalyzer(filter)
+
+ rect, err := smart.FindBestCrop(img, width, height)
+ if err != nil {
+ return nil, err
+ }
+
+ b := img.Bounds().Intersect(rect)
+
+ cropped := imaging.Crop(img, b)
+
+ return imaging.Resize(cropped, width, height, filter), nil
+}
DIR diff --git a/resources/internal/key.go b/resources/internal/key.go
@@ -0,0 +1,61 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package internal
+
+import (
+ "strconv"
+
+ bp "github.com/gohugoio/hugo/bufferpool"
+
+ "github.com/mitchellh/hashstructure"
+)
+
+// ResourceTransformationKey are provided by the different transformation implementations.
+// It identifies the transformation (name) and its configuration (elements).
+// We combine this in a chain with the rest of the transformations
+// with the target filename and a content hash of the origin to use as cache key.
+type ResourceTransformationKey struct {
+ Name string
+ elements []interface{}
+}
+
+// NewResourceTransformationKey creates a new ResourceTransformationKey from the transformation
+// name and elements. We will create a 64 bit FNV hash from the elements, which when combined
+// with the other key elements should be unique for all practical applications.
+func NewResourceTransformationKey(name string, elements ...interface{}) ResourceTransformationKey {
+ return ResourceTransformationKey{Name: name, elements: elements}
+}
+
+// Value returns the Key as a string.
+// Do not change this without good reasons.
+func (k ResourceTransformationKey) Value() string {
+ if len(k.elements) == 0 {
+ return k.Name
+ }
+
+ sb := bp.GetBuffer()
+ defer bp.PutBuffer(sb)
+
+ sb.WriteString(k.Name)
+ for _, element := range k.elements {
+ hash, err := hashstructure.Hash(element, nil)
+ if err != nil {
+ panic(err)
+ }
+ sb.WriteString("_")
+ sb.WriteString(strconv.FormatUint(hash, 10))
+ }
+
+ return sb.String()
+}
DIR diff --git a/resources/internal/key_test.go b/resources/internal/key_test.go
@@ -0,0 +1,36 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package internal
+
+import (
+ "testing"
+
+ qt "github.com/frankban/quicktest"
+)
+
+type testStruct struct {
+ Name string
+ V1 int64
+ V2 int32
+ V3 int
+ V4 uint64
+}
+
+func TestResourceTransformationKey(t *testing.T) {
+ // We really need this key to be portable across OSes.
+ key := NewResourceTransformationKey("testing",
+ testStruct{Name: "test", V1: int64(10), V2: int32(20), V3: 30, V4: uint64(40)})
+ c := qt.New(t)
+ c.Assert("testing_518996646957295636", qt.Equals, key.Value())
+}
DIR diff --git a/resources/resource.go b/resources/resource.go
@@ -17,30 +17,23 @@ import (
"fmt"
"io"
"io/ioutil"
- "mime"
"os"
"path"
"path/filepath"
- "strings"
"sync"
"github.com/gohugoio/hugo/media"
+ "github.com/gohugoio/hugo/source"
- "github.com/gohugoio/hugo/output"
- "github.com/gohugoio/hugo/tpl"
"github.com/pkg/errors"
- "github.com/gohugoio/hugo/cache/filecache"
- "github.com/gohugoio/hugo/common/collections"
"github.com/gohugoio/hugo/common/hugio"
- "github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/resources/page"
"github.com/gohugoio/hugo/resources/resource"
"github.com/spf13/afero"
"github.com/gohugoio/hugo/helpers"
- "github.com/gohugoio/hugo/source"
)
var (
@@ -51,80 +44,10 @@ var (
_ resource.Cloner = (*genericResource)(nil)
_ resource.ResourcesLanguageMerger = (*resource.Resources)(nil)
_ permalinker = (*genericResource)(nil)
- _ collections.Slicer = (*genericResource)(nil)
_ resource.Identifier = (*genericResource)(nil)
+ _ fileInfo = (*genericResource)(nil)
)
-var noData = make(map[string]interface{})
-
-type permalinker interface {
- relPermalinkFor(target string) string
- permalinkFor(target string) string
- relTargetPathsFor(target string) []string
- relTargetPaths() []string
- TargetPath() string
-}
-
-type Spec struct {
- *helpers.PathSpec
-
- MediaTypes media.Types
- OutputFormats output.Formats
-
- Logger *loggers.Logger
-
- TextTemplates tpl.TemplateParseFinder
-
- Permalinks page.PermalinkExpander
-
- // Holds default filter settings etc.
- imaging *Imaging
-
- imageCache *imageCache
- ResourceCache *ResourceCache
- FileCaches filecache.Caches
-}
-
-func NewSpec(
- s *helpers.PathSpec,
- fileCaches filecache.Caches,
- logger *loggers.Logger,
- outputFormats output.Formats,
- mimeTypes media.Types) (*Spec, error) {
-
- imaging, err := decodeImaging(s.Cfg.GetStringMap("imaging"))
- if err != nil {
- return nil, err
- }
-
- if logger == nil {
- logger = loggers.NewErrorLogger()
- }
-
- permalinks, err := page.NewPermalinkExpander(s)
- if err != nil {
- return nil, err
- }
-
- rs := &Spec{PathSpec: s,
- Logger: logger,
- imaging: &imaging,
- MediaTypes: mimeTypes,
- OutputFormats: outputFormats,
- Permalinks: permalinks,
- FileCaches: fileCaches,
- imageCache: newImageCache(
- fileCaches.ImageCache(),
-
- s,
- )}
-
- rs.ResourceCache = newResourceCache(rs)
-
- return rs, nil
-
-}
-
type ResourceSourceDescriptor struct {
// TargetPaths is a callback to fetch paths's relative to its owner.
TargetPaths func() page.TargetPaths
@@ -161,136 +84,77 @@ func (r ResourceSourceDescriptor) Filename() string {
return r.SourceFilename
}
-func (r *Spec) New(fd ResourceSourceDescriptor) (resource.Resource, error) {
- return r.newResourceFor(fd)
+type ResourceTransformer interface {
+ resource.Resource
+ Transformer
}
-func (r *Spec) newResourceFor(fd ResourceSourceDescriptor) (resource.Resource, error) {
- if fd.OpenReadSeekCloser == nil {
- if fd.SourceFile != nil && fd.SourceFilename != "" {
- return nil, errors.New("both SourceFile and AbsSourceFilename provided")
- } else if fd.SourceFile == nil && fd.SourceFilename == "" {
- return nil, errors.New("either SourceFile or AbsSourceFilename must be provided")
- }
- }
-
- if fd.RelTargetFilename == "" {
- fd.RelTargetFilename = fd.Filename()
- }
-
- if len(fd.TargetBasePaths) == 0 {
- // If not set, we publish the same resource to all hosts.
- fd.TargetBasePaths = r.MultihostTargetBasePaths
- }
-
- return r.newResource(fd.Fs, fd)
+type Transformer interface {
+ Transform(...ResourceTransformation) (ResourceTransformer, error)
}
-func (r *Spec) newResource(sourceFs afero.Fs, fd ResourceSourceDescriptor) (resource.Resource, error) {
- fi := fd.FileInfo
- var sourceFilename string
-
- if fd.OpenReadSeekCloser != nil {
- } else if fd.SourceFilename != "" {
- var err error
- fi, err = sourceFs.Stat(fd.SourceFilename)
- if err != nil {
- if os.IsNotExist(err) {
- return nil, nil
- }
- return nil, err
- }
- sourceFilename = fd.SourceFilename
- } else {
- sourceFilename = fd.SourceFile.Filename()
- }
-
- if fd.RelTargetFilename == "" {
- fd.RelTargetFilename = sourceFilename
- }
+type baseResourceResource interface {
+ resource.Cloner
+ resource.ContentProvider
+ resource.Resource
+ resource.Identifier
+}
- ext := strings.ToLower(filepath.Ext(fd.RelTargetFilename))
- mimeType, found := r.MediaTypes.GetFirstBySuffix(strings.TrimPrefix(ext, "."))
- // TODO(bep) we need to handle these ambigous types better, but in this context
- // we most likely want the application/xml type.
- if mimeType.Suffix() == "xml" && mimeType.SubType == "rss" {
- mimeType, found = r.MediaTypes.GetByType("application/xml")
- }
+type baseResourceInternal interface {
+ resource.Source
- if !found {
- // A fallback. Note that mime.TypeByExtension is slow by Hugo standards,
- // so we should configure media types to avoid this lookup for most
- // situations.
- mimeStr := mime.TypeByExtension(ext)
- if mimeStr != "" {
- mimeType, _ = media.FromStringAndExt(mimeStr, ext)
- }
- }
+ fileInfo
+ metaAssigner
+ targetPather
- gr := r.newGenericResourceWithBase(
- sourceFs,
- fd.LazyPublish,
- fd.OpenReadSeekCloser,
- fd.TargetBasePaths,
- fd.TargetPaths,
- fi,
- sourceFilename,
- fd.RelTargetFilename,
- mimeType)
-
- if mimeType.MainType == "image" {
- imgFormat, ok := imageFormats[ext]
- if !ok {
- // This allows SVG etc. to be used as resources. They will not have the methods of the Image, but
- // that would not (currently) have worked.
- return gr, nil
- }
+ ReadSeekCloser() (hugio.ReadSeekCloser, error)
- if err := gr.initHash(); err != nil {
- return nil, err
- }
+ // Internal
+ cloneWithUpdates(*transformationUpdate) (baseResource, error)
+ tryTransformedFileCache(key string, u *transformationUpdate) io.ReadCloser
- return &Image{
- format: imgFormat,
- imaging: r.imaging,
- genericResource: gr}, nil
- }
- return gr, nil
+ specProvider
+ getResourcePaths() *resourcePathDescriptor
+ getTargetFilenames() []string
+ openDestinationsForWriting() (io.WriteCloser, error)
+ openPublishFileForWriting(relTargetPath string) (io.WriteCloser, error)
+ relTargetPathForRel(rel string, addBaseTargetPath, isAbs, isURL bool) string
}
-// TODO(bep) unify
-func (r *Spec) IsInImageCache(key string) bool {
- // This is used for cache pruning. We currently only have images, but we could
- // imagine expanding on this.
- return r.imageCache.isInCache(key)
+type specProvider interface {
+ getSpec() *Spec
}
-func (r *Spec) DeleteCacheByPrefix(prefix string) {
- r.imageCache.deleteByPrefix(prefix)
+type baseResource interface {
+ baseResourceResource
+ baseResourceInternal
}
-func (r *Spec) ClearCaches() {
- r.imageCache.clear()
- r.ResourceCache.clear()
+type commonResource struct {
}
-func (r *Spec) CacheStats() string {
- r.imageCache.mu.RLock()
- defer r.imageCache.mu.RUnlock()
-
- s := fmt.Sprintf("Cache entries: %d", len(r.imageCache.store))
-
- count := 0
- for k := range r.imageCache.store {
- if count > 5 {
- break
+// Slice is not meant to be used externally. It's a bridge function
+// for the template functions. See collections.Slice.
+func (commonResource) Slice(in interface{}) (interface{}, error) {
+ switch items := in.(type) {
+ case resource.Resources:
+ return items, nil
+ case []interface{}:
+ groups := make(resource.Resources, len(items))
+ for i, v := range items {
+ g, ok := v.(resource.Resource)
+ if !ok {
+ return nil, fmt.Errorf("type %T is not a Resource", v)
+ }
+ groups[i] = g
+ {
+ }
}
- s += "\n" + k
- count++
+ return groups, nil
+ default:
+ return nil, fmt.Errorf("invalid slice type %T", items)
}
-
- return s
}
type dirFile struct {
@@ -304,91 +168,33 @@ func (d dirFile) path() string {
return path.Join(d.dir, d.file)
}
-type resourcePathDescriptor struct {
- // The relative target directory and filename.
- relTargetDirFile dirFile
-
- // Callback used to construct a target path relative to its owner.
- targetPathBuilder func() page.TargetPaths
-
- // This will normally be the same as above, but this will only apply to publishing
- // of resources. It may be mulltiple values when in multihost mode.
- baseTargetPathDirs []string
-
- // baseOffset is set when the output format's path has a offset, e.g. for AMP.
- baseOffset string
-}
-
-type resourceContent struct {
- content string
- contentInit sync.Once
-}
-
-type resourceHash struct {
- hash string
- hashInit sync.Once
-}
-
-type publishOnce struct {
- publisherInit sync.Once
- publisherErr error
- logger *loggers.Logger
-}
-
-func (l *publishOnce) publish(s resource.Source) error {
- l.publisherInit.Do(func() {
- l.publisherErr = s.Publish()
- if l.publisherErr != nil {
- l.logger.ERROR.Printf("failed to publish Resource: %s", l.publisherErr)
- }
- })
- return l.publisherErr
+type fileInfo interface {
+ getSourceFilename() string
+ setSourceFilename(string)
+ setSourceFs(afero.Fs)
+ hash() (string, error)
+ size() int
}
// genericResource represents a generic linkable resource.
type genericResource struct {
- commonResource
- resourcePathDescriptor
+ *resourcePathDescriptor
+ *resourceFileInfo
+ *resourceContent
+
+ spec *Spec
title string
name string
params map[string]interface{}
-
- // Absolute filename to the source, including any content folder path.
- // Note that this is absolute in relation to the filesystem it is stored in.
- // It can be a base path filesystem, and then this filename will not match
- // the path to the file on the real filesystem.
- sourceFilename string
-
- // Will be set if this resource is backed by something other than a file.
- openReadSeekerCloser resource.OpenReadSeekCloser
-
- // A hash of the source content. Is only calculated in caching situations.
- *resourceHash
-
- // This may be set to tell us to look in another filesystem for this resource.
- // We, by default, use the sourceFs filesystem in the spec below.
- sourceFs afero.Fs
-
- spec *Spec
+ data map[string]interface{}
resourceType string
mediaType media.Type
-
- osFileInfo os.FileInfo
-
- // We create copies of this struct, so this needs to be a pointer.
- *resourceContent
-
- // May be set to signal lazy/delayed publishing.
- *publishOnce
}
-type commonResource struct {
-}
-
-func (l *genericResource) Data() interface{} {
- return noData
+func (l *genericResource) Clone() resource.Resource {
+ return l.clone()
}
func (l *genericResource) Content() (interface{}, error) {
@@ -399,72 +205,80 @@ func (l *genericResource) Content() (interface{}, error) {
return l.content, nil
}
-func (l *genericResource) ReadSeekCloser() (hugio.ReadSeekCloser, error) {
- if l.openReadSeekerCloser != nil {
- return l.openReadSeekerCloser()
+func (l *genericResource) Data() interface{} {
+ return l.data
+}
+
+func (l *genericResource) Key() string {
+ return l.relTargetDirFile.path()
+}
+
+func (l *genericResource) MediaType() media.Type {
+ return l.mediaType
+}
+
+func (l *genericResource) Name() string {
+ return l.name
+}
+
+func (l *genericResource) Params() map[string]interface{} {
+ return l.params
+}
+
+func (l *genericResource) Permalink() string {
+ return l.spec.PermalinkForBaseURL(l.relPermalinkForRel(l.relTargetDirFile.path(), true), l.spec.BaseURL.HostURL())
+}
+
+func (l *genericResource) Publish() error {
+ fr, err := l.ReadSeekCloser()
+ if err != nil {
+ return err
}
+ defer fr.Close()
- f, err := l.getSourceFs().Open(l.sourceFilename)
+ fw, err := helpers.OpenFilesForWriting(l.spec.BaseFs.PublishFs, l.getTargetFilenames()...)
if err != nil {
- return nil, err
+ return err
}
- return f, nil
+ defer fw.Close()
+ _, err = io.Copy(fw, fr)
+ return err
}
-func (l *genericResource) MediaType() media.Type {
- return l.mediaType
+func (l *genericResource) RelPermalink() string {
+ return l.relPermalinkFor(l.relTargetDirFile.path())
}
-// Implement the Cloner interface.
-func (l genericResource) WithNewBase(base string) resource.Resource {
- l.baseOffset = base
- l.resourceContent = &resourceContent{}
- return &l
+func (l *genericResource) ResourceType() string {
+ return l.resourceType
}
-// Slice is not meant to be used externally. It's a bridge function
-// for the template functions. See collections.Slice.
-func (commonResource) Slice(in interface{}) (interface{}, error) {
- switch items := in.(type) {
- case resource.Resources:
- return items, nil
- case []interface{}:
- groups := make(resource.Resources, len(items))
- for i, v := range items {
- g, ok := v.(resource.Resource)
- if !ok {
- return nil, fmt.Errorf("type %T is not a Resource", v)
- }
- groups[i] = g
- }
- return groups, nil
- default:
- return nil, fmt.Errorf("invalid slice type %T", items)
- }
+func (l *genericResource) String() string {
+ return fmt.Sprintf("Resource(%s: %s)", l.resourceType, l.name)
}
-func (l *genericResource) initHash() error {
- var err error
- l.hashInit.Do(func() {
- var hash string
- var f hugio.ReadSeekCloser
- f, err = l.ReadSeekCloser()
- if err != nil {
- err = errors.Wrap(err, "failed to open source file")
- return
- }
- defer f.Close()
+// Path is stored with Unix style slashes.
+func (l *genericResource) TargetPath() string {
+ return l.relTargetDirFile.path()
+}
- hash, err = helpers.MD5FromFileFast(f)
- if err != nil {
- return
- }
- l.hash = hash
+func (l *genericResource) Title() string {
+ return l.title
+}
- })
+func (l *genericResource) createBasePath(rel string, isURL bool) string {
+ if l.targetPathBuilder == nil {
+ return rel
+ }
+ tp := l.targetPathBuilder()
- return err
+ if isURL {
+ return path.Join(tp.SubResourceBaseLink, rel)
+ }
+
+ // TODO(bep) path
+ return path.Join(filepath.ToSlash(tp.SubResourceBaseTarget), rel)
}
func (l *genericResource) initContent() error {
@@ -484,100 +298,141 @@ func (l *genericResource) initContent() error {
}
l.content = string(b)
-
})
return err
}
-func (l *genericResource) getSourceFs() afero.Fs {
- return l.sourceFs
+func (l *genericResource) setName(name string) {
+ l.name = name
}
-func (l *genericResource) publishIfNeeded() {
- if l.publishOnce != nil {
- l.publishOnce.publish(l)
- }
+func (l *genericResource) getResourcePaths() *resourcePathDescriptor {
+ return l.resourcePathDescriptor
}
-func (l *genericResource) Permalink() string {
- l.publishIfNeeded()
- return l.spec.PermalinkForBaseURL(l.relPermalinkForRel(l.relTargetDirFile.path(), true), l.spec.BaseURL.HostURL())
+func (l *genericResource) getSpec() *Spec {
+ return l.spec
}
-func (l *genericResource) RelPermalink() string {
- l.publishIfNeeded()
- return l.relPermalinkFor(l.relTargetDirFile.path())
+func (l *genericResource) getTargetFilenames() []string {
+ paths := l.relTargetPaths()
+ for i, p := range paths {
+ paths[i] = filepath.Clean(p)
+ }
+ return paths
}
-func (l *genericResource) Key() string {
- return l.relTargetDirFile.path()
+func (l *genericResource) setTitle(title string) {
+ l.title = title
}
-func (l *genericResource) relPermalinkFor(target string) string {
- return l.relPermalinkForRel(target, false)
-
+func (r *genericResource) tryTransformedFileCache(key string, u *transformationUpdate) io.ReadCloser {
+ fi, f, meta, found := r.spec.ResourceCache.getFromFile(key)
+ if !found {
+ return nil
+ }
+ u.sourceFilename = &fi.Name
+ mt, _ := r.spec.MediaTypes.GetByType(meta.MediaTypeV)
+ u.mediaType = mt
+ u.data = meta.MetaData
+ u.targetPath = meta.Target
+ return f
}
-func (l *genericResource) permalinkFor(target string) string {
- return l.spec.PermalinkForBaseURL(l.relPermalinkForRel(target, true), l.spec.BaseURL.HostURL())
-}
-func (l *genericResource) relTargetPathsFor(target string) []string {
- return l.relTargetPathsForRel(target)
+func (r *genericResource) mergeData(in map[string]interface{}) {
+ if len(in) == 0 {
+ return
+ }
+ if r.data == nil {
+ r.data = make(map[string]interface{})
+ }
+ for k, v := range in {
+ if _, found := r.data[k]; !found {
+ r.data[k] = v
+ }
+ }
}
-func (l *genericResource) relTargetPaths() []string {
- return l.relTargetPathsForRel(l.TargetPath())
-}
+func (rc *genericResource) cloneWithUpdates(u *transformationUpdate) (baseResource, error) {
+ r := rc.clone()
-func (l *genericResource) Name() string {
- return l.name
-}
+ if u.content != nil {
+ r.contentInit.Do(func() {
+ r.content = *u.content
+ r.openReadSeekerCloser = func() (hugio.ReadSeekCloser, error) {
+ return hugio.NewReadSeekerNoOpCloserFromString(r.content), nil
+ }
+ })
+ }
-func (l *genericResource) Title() string {
- return l.title
-}
+ r.mediaType = u.mediaType
-func (l *genericResource) Params() map[string]interface{} {
- return l.params
-}
+ if u.sourceFilename != nil {
+ r.setSourceFilename(*u.sourceFilename)
+ }
-func (l *genericResource) setTitle(title string) {
- l.title = title
+ if u.sourceFs != nil {
+ r.setSourceFs(u.sourceFs)
+ }
+
+ if u.targetPath == "" {
+ return nil, errors.New("missing targetPath")
+ }
+
+ fpath, fname := path.Split(u.targetPath)
+ r.resourcePathDescriptor.relTargetDirFile = dirFile{dir: fpath, file: fname}
+
+ r.mergeData(u.data)
+
+ return r, nil
}
-func (l *genericResource) setName(name string) {
- l.name = name
+func (l genericResource) clone() *genericResource {
+ gi := *l.resourceFileInfo
+ rp := *l.resourcePathDescriptor
+ l.resourceFileInfo = &gi
+ l.resourcePathDescriptor = &rp
+ l.resourceContent = &resourceContent{}
+ return &l
}
-func (l *genericResource) updateParams(params map[string]interface{}) {
- if l.params == nil {
- l.params = params
- return
- }
+// returns an opened file or nil if nothing to write.
+func (l *genericResource) openDestinationsForWriting() (io.WriteCloser, error) {
+ targetFilenames := l.getTargetFilenames()
+ var changedFilenames []string
- // Sets the params not already set
- for k, v := range params {
- if _, found := l.params[k]; !found {
- l.params[k] = v
+ // Fast path:
+ // This is a processed version of the original;
+ // check if it already existis at the destination.
+ for _, targetFilename := range targetFilenames {
+ if _, err := l.getSpec().BaseFs.PublishFs.Stat(targetFilename); err == nil {
+ continue
}
+ changedFilenames = append(changedFilenames, targetFilename)
}
+
+ if len(changedFilenames) == 0 {
+ return nil, nil
+ }
+
+ return helpers.OpenFilesForWriting(l.getSpec().BaseFs.PublishFs, changedFilenames...)
}
-func (l *genericResource) relPermalinkForRel(rel string, isAbs bool) string {
- return l.spec.PathSpec.URLizeFilename(l.relTargetPathForRel(rel, false, isAbs, true))
+func (r *genericResource) openPublishFileForWriting(relTargetPath string) (io.WriteCloser, error) {
+ return helpers.OpenFilesForWriting(r.spec.BaseFs.PublishFs, r.relTargetPathsFor(relTargetPath)...)
}
-func (l *genericResource) relTargetPathsForRel(rel string) []string {
- if len(l.baseTargetPathDirs) == 0 {
- return []string{l.relTargetPathForRelAndBasePath(rel, "", false, false)}
- }
+func (l *genericResource) permalinkFor(target string) string {
+ return l.spec.PermalinkForBaseURL(l.relPermalinkForRel(target, true), l.spec.BaseURL.HostURL())
+}
- var targetPaths = make([]string, len(l.baseTargetPathDirs))
- for i, dir := range l.baseTargetPathDirs {
- targetPaths[i] = l.relTargetPathForRelAndBasePath(rel, dir, false, false)
- }
- return targetPaths
+func (l *genericResource) relPermalinkFor(target string) string {
+ return l.relPermalinkForRel(target, false)
+}
+
+func (l *genericResource) relPermalinkForRel(rel string, isAbs bool) string {
+ return l.spec.PathSpec.URLizeFilename(l.relTargetPathForRel(rel, false, isAbs, true))
}
func (l *genericResource) relTargetPathForRel(rel string, addBaseTargetPath, isAbs, isURL bool) string {
@@ -592,20 +447,6 @@ func (l *genericResource) relTargetPathForRel(rel string, addBaseTargetPath, isA
return l.relTargetPathForRelAndBasePath(rel, basePath, isAbs, isURL)
}
-func (l *genericResource) createBasePath(rel string, isURL bool) string {
- if l.targetPathBuilder == nil {
- return rel
- }
- tp := l.targetPathBuilder()
-
- if isURL {
- return path.Join(tp.SubResourceBaseLink, rel)
- }
-
- // TODO(bep) path
- return path.Join(filepath.ToSlash(tp.SubResourceBaseTarget), rel)
-}
-
func (l *genericResource) relTargetPathForRelAndBasePath(rel, basePath string, isAbs, isURL bool) string {
rel = l.createBasePath(rel, isURL)
@@ -631,117 +472,153 @@ func (l *genericResource) relTargetPathForRelAndBasePath(rel, basePath string, i
return rel
}
-func (l *genericResource) ResourceType() string {
- return l.resourceType
+func (l *genericResource) relTargetPaths() []string {
+ return l.relTargetPathsForRel(l.TargetPath())
}
-func (l *genericResource) String() string {
- return fmt.Sprintf("Resource(%s: %s)", l.resourceType, l.name)
+func (l *genericResource) relTargetPathsFor(target string) []string {
+ return l.relTargetPathsForRel(target)
}
-func (l *genericResource) Publish() error {
- fr, err := l.ReadSeekCloser()
- if err != nil {
- return err
+func (l *genericResource) relTargetPathsForRel(rel string) []string {
+ if len(l.baseTargetPathDirs) == 0 {
+ return []string{l.relTargetPathForRelAndBasePath(rel, "", false, false)}
}
- defer fr.Close()
- fw, err := helpers.OpenFilesForWriting(l.spec.BaseFs.PublishFs, l.targetFilenames()...)
- if err != nil {
- return err
+ targetPaths := make([]string, len(l.baseTargetPathDirs))
+ for i, dir := range l.baseTargetPathDirs {
+ targetPaths[i] = l.relTargetPathForRelAndBasePath(rel, dir, false, false)
}
- defer fw.Close()
+ return targetPaths
+}
- _, err = io.Copy(fw, fr)
- return err
+func (l *genericResource) updateParams(params map[string]interface{}) {
+ if l.params == nil {
+ l.params = params
+ return
+ }
+
+ // Sets the params not already set
+ for k, v := range params {
+ if _, found := l.params[k]; !found {
+ l.params[k] = v
+ }
+ }
}
-// Path is stored with Unix style slashes.
-func (l *genericResource) TargetPath() string {
- return l.relTargetDirFile.path()
+type targetPather interface {
+ TargetPath() string
}
-func (l *genericResource) targetFilenames() []string {
- paths := l.relTargetPaths()
- for i, p := range paths {
- paths[i] = filepath.Clean(p)
- }
- return paths
+type permalinker interface {
+ targetPather
+ permalinkFor(target string) string
+ relPermalinkFor(target string) string
+ relTargetPaths() []string
+ relTargetPathsFor(target string) []string
}
-// TODO(bep) clean up below
-func (r *Spec) newGenericResource(sourceFs afero.Fs,
- targetPathBuilder func() page.TargetPaths,
- osFileInfo os.FileInfo,
- sourceFilename,
- baseFilename string,
- mediaType media.Type) *genericResource {
- return r.newGenericResourceWithBase(
- sourceFs,
- false,
- nil,
- nil,
- targetPathBuilder,
- osFileInfo,
- sourceFilename,
- baseFilename,
- mediaType,
- )
-
-}
-
-func (r *Spec) newGenericResourceWithBase(
- sourceFs afero.Fs,
- lazyPublish bool,
- openReadSeekerCloser resource.OpenReadSeekCloser,
- targetPathBaseDirs []string,
- targetPathBuilder func() page.TargetPaths,
- osFileInfo os.FileInfo,
- sourceFilename,
- baseFilename string,
- mediaType media.Type) *genericResource {
-
- if osFileInfo != nil && osFileInfo.IsDir() {
- panic(fmt.Sprintf("dirs not supported resource types: %v", osFileInfo))
- }
+type resourceContent struct {
+ content string
+ contentInit sync.Once
+}
- // This value is used both to construct URLs and file paths, but start
- // with a Unix-styled path.
- baseFilename = helpers.ToSlashTrimLeading(baseFilename)
- fpath, fname := path.Split(baseFilename)
+type resourceFileInfo struct {
+ // Will be set if this resource is backed by something other than a file.
+ openReadSeekerCloser resource.OpenReadSeekCloser
- var resourceType string
- if mediaType.MainType == "image" {
- resourceType = mediaType.MainType
- } else {
- resourceType = mediaType.SubType
- }
+ // This may be set to tell us to look in another filesystem for this resource.
+ // We, by default, use the sourceFs filesystem in the spec below.
+ sourceFs afero.Fs
- pathDescriptor := resourcePathDescriptor{
- baseTargetPathDirs: helpers.UniqueStringsReuse(targetPathBaseDirs),
- targetPathBuilder: targetPathBuilder,
- relTargetDirFile: dirFile{dir: fpath, file: fname},
+ // Absolute filename to the source, including any content folder path.
+ // Note that this is absolute in relation to the filesystem it is stored in.
+ // It can be a base path filesystem, and then this filename will not match
+ // the path to the file on the real filesystem.
+ sourceFilename string
+
+ fi os.FileInfo
+
+ // A hash of the source content. Is only calculated in caching situations.
+ h *resourceHash
+}
+
+func (fi *resourceFileInfo) ReadSeekCloser() (hugio.ReadSeekCloser, error) {
+ if fi.openReadSeekerCloser != nil {
+ return fi.openReadSeekerCloser()
}
- var po *publishOnce
- if lazyPublish {
- po = &publishOnce{logger: r.Logger}
+ f, err := fi.getSourceFs().Open(fi.getSourceFilename())
+ if err != nil {
+ return nil, err
}
+ return f, nil
+}
+
+func (fi *resourceFileInfo) getSourceFilename() string {
+ return fi.sourceFilename
+}
+
+func (fi *resourceFileInfo) setSourceFilename(s string) {
+ // Make sure it's always loaded by sourceFilename.
+ fi.openReadSeekerCloser = nil
+ fi.sourceFilename = s
+}
+
+func (fi *resourceFileInfo) getSourceFs() afero.Fs {
+ return fi.sourceFs
+}
+
+func (fi *resourceFileInfo) setSourceFs(fs afero.Fs) {
+ fi.sourceFs = fs
+}
+
+func (fi *resourceFileInfo) hash() (string, error) {
+ var err error
+ fi.h.init.Do(func() {
+ var hash string
+ var f hugio.ReadSeekCloser
+ f, err = fi.ReadSeekCloser()
+ if err != nil {
+ err = errors.Wrap(err, "failed to open source file")
+ return
+ }
+ defer f.Close()
+
+ hash, err = helpers.MD5FromFileFast(f)
+ if err != nil {
+ return
+ }
+ fi.h.value = hash
+ })
- return &genericResource{
- openReadSeekerCloser: openReadSeekerCloser,
- publishOnce: po,
- resourcePathDescriptor: pathDescriptor,
- sourceFs: sourceFs,
- osFileInfo: osFileInfo,
- sourceFilename: sourceFilename,
- mediaType: mediaType,
- resourceType: resourceType,
- spec: r,
- params: make(map[string]interface{}),
- name: baseFilename,
- title: baseFilename,
- resourceContent: &resourceContent{},
- resourceHash: &resourceHash{},
+ return fi.h.value, err
+}
+
+func (fi *resourceFileInfo) size() int {
+ if fi.fi == nil {
+ return 0
}
+
+ return int(fi.fi.Size())
+}
+
+type resourceHash struct {
+ value string
+ init sync.Once
+}
+
+type resourcePathDescriptor struct {
+ // The relative target directory and filename.
+ relTargetDirFile dirFile
+
+ // Callback used to construct a target path relative to its owner.
+ targetPathBuilder func() page.TargetPaths
+
+ // This will normally be the same as above, but this will only apply to publishing
+ // of resources. It may be mulltiple values when in multihost mode.
+ baseTargetPathDirs []string
+
+ // baseOffset is set when the output format's path has a offset, e.g. for AMP.
+ baseOffset string
}
DIR diff --git a/resources/resource/resourcetypes.go b/resources/resource/resourcetypes.go
@@ -23,7 +23,7 @@ import (
// Cloner is an internal template and not meant for use in the templates. It
// may change without notice.
type Cloner interface {
- WithNewBase(base string) Resource
+ Clone() Resource
}
// Resource represents a linkable resource, i.e. a content page, image etc.
@@ -35,6 +35,20 @@ type Resource interface {
ResourceDataProvider
}
+// Image represents an image resource.
+type Image interface {
+ Resource
+ ImageOps
+}
+
+type ImageOps interface {
+ Height() int
+ Width() int
+ Fill(spec string) (Image, error)
+ Fit(spec string) (Image, error)
+ Resize(spec string) (Image, error)
+}
+
type ResourceTypesProvider interface {
// MediaType is this resource's MIME type.
MediaType() media.Type
@@ -117,6 +131,10 @@ type OpenReadSeekCloser func() (hugio.ReadSeekCloser, error)
// ReadSeekCloserResource is a Resource that supports loading its content.
type ReadSeekCloserResource interface {
MediaType() media.Type
+ ReadSeekCloserProvider
+}
+
+type ReadSeekCloserProvider interface {
ReadSeekCloser() (hugio.ReadSeekCloser, error)
}
DIR diff --git a/resources/resource_cache.go b/resources/resource_cache.go
@@ -281,7 +281,7 @@ func (c *ResourceCache) DeletePartitions(partitions ...string) {
for k := range c.cache {
clear := false
- for p, _ := range partitionsSet {
+ for p := range partitionsSet {
if strings.Contains(k, p) {
// There will be some false positive, but that's fine.
clear = true
DIR diff --git a/resources/resource_metadata.go b/resources/resource_metadata.go
@@ -29,9 +29,15 @@ import (
)
var (
- _ metaAssigner = (*genericResource)(nil)
+ _ metaAssigner = (*genericResource)(nil)
+ _ metaAssigner = (*imageResource)(nil)
+ _ metaAssignerProvider = (*resourceAdapter)(nil)
)
+type metaAssignerProvider interface {
+ getMetaAssigner() metaAssigner
+}
+
// metaAssigner allows updating metadata in resources that supports it.
type metaAssigner interface {
setTitle(title string)
@@ -50,8 +56,15 @@ func AssignMetadata(metadata []map[string]interface{}, resources ...resource.Res
counters := make(map[string]int)
for _, r := range resources {
- if _, ok := r.(metaAssigner); !ok {
- continue
+ var ma metaAssigner
+ mp, ok := r.(metaAssignerProvider)
+ if ok {
+ ma = mp.getMetaAssigner()
+ } else {
+ ma, ok = r.(metaAssigner)
+ if !ok {
+ continue
+ }
}
var (
@@ -61,7 +74,6 @@ func AssignMetadata(metadata []map[string]interface{}, resources ...resource.Res
resourceSrcKey = strings.ToLower(r.Name())
)
- ma := r.(metaAssigner)
for _, meta := range metadata {
src, found := meta["src"]
if !found {
DIR diff --git a/resources/resource_metadata_test.go b/resources/resource_metadata_test.go
@@ -24,7 +24,7 @@ import (
func TestAssignMetadata(t *testing.T) {
c := qt.New(t)
- spec := newTestResourceSpec(c)
+ spec := newTestResourceSpec(specDescriptor{c: c})
var foo1, foo2, foo3, logo1, logo2, logo3 resource.Resource
var resources resource.Resources
DIR diff --git a/resources/resource_spec.go b/resources/resource_spec.go
@@ -0,0 +1,304 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package resources
+
+import (
+ "errors"
+ "fmt"
+ "mime"
+ "os"
+ "path"
+ "path/filepath"
+ "strings"
+
+ "github.com/gohugoio/hugo/helpers"
+
+ "github.com/gohugoio/hugo/cache/filecache"
+ "github.com/gohugoio/hugo/common/loggers"
+ "github.com/gohugoio/hugo/media"
+ "github.com/gohugoio/hugo/output"
+ "github.com/gohugoio/hugo/resources/images"
+ "github.com/gohugoio/hugo/resources/page"
+ "github.com/gohugoio/hugo/resources/resource"
+ "github.com/gohugoio/hugo/tpl"
+ "github.com/spf13/afero"
+)
+
+func NewSpec(
+ s *helpers.PathSpec,
+ fileCaches filecache.Caches,
+ logger *loggers.Logger,
+ outputFormats output.Formats,
+ mimeTypes media.Types) (*Spec, error) {
+
+ imgConfig, err := images.DecodeConfig(s.Cfg.GetStringMap("imaging"))
+ if err != nil {
+ return nil, err
+ }
+
+ imaging := &images.ImageProcessor{Cfg: imgConfig}
+
+ if logger == nil {
+ logger = loggers.NewErrorLogger()
+ }
+
+ permalinks, err := page.NewPermalinkExpander(s)
+ if err != nil {
+ return nil, err
+ }
+
+ rs := &Spec{PathSpec: s,
+ Logger: logger,
+ imaging: imaging,
+ MediaTypes: mimeTypes,
+ OutputFormats: outputFormats,
+ Permalinks: permalinks,
+ FileCaches: fileCaches,
+ imageCache: newImageCache(
+ fileCaches.ImageCache(),
+
+ s,
+ )}
+
+ rs.ResourceCache = newResourceCache(rs)
+
+ return rs, nil
+
+}
+
+type Spec struct {
+ *helpers.PathSpec
+
+ MediaTypes media.Types
+ OutputFormats output.Formats
+
+ Logger *loggers.Logger
+
+ TextTemplates tpl.TemplateParseFinder
+
+ Permalinks page.PermalinkExpander
+
+ // Holds default filter settings etc.
+ imaging *images.ImageProcessor
+
+ imageCache *imageCache
+ ResourceCache *ResourceCache
+ FileCaches filecache.Caches
+}
+
+func (r *Spec) New(fd ResourceSourceDescriptor) (resource.Resource, error) {
+ return r.newResourceFor(fd)
+}
+
+func (r *Spec) CacheStats() string {
+ r.imageCache.mu.RLock()
+ defer r.imageCache.mu.RUnlock()
+
+ s := fmt.Sprintf("Cache entries: %d", len(r.imageCache.store))
+
+ count := 0
+ for k := range r.imageCache.store {
+ if count > 5 {
+ break
+ }
+ s += "\n" + k
+ count++
+ }
+
+ return s
+}
+
+func (r *Spec) ClearCaches() {
+ r.imageCache.clear()
+ r.ResourceCache.clear()
+}
+
+func (r *Spec) DeleteCacheByPrefix(prefix string) {
+ r.imageCache.deleteByPrefix(prefix)
+}
+
+// TODO(bep) unify
+func (r *Spec) IsInImageCache(key string) bool {
+ // This is used for cache pruning. We currently only have images, but we could
+ // imagine expanding on this.
+ return r.imageCache.isInCache(key)
+}
+
+func (s *Spec) String() string {
+ return "spec"
+}
+
+// TODO(bep) clean up below
+func (r *Spec) newGenericResource(sourceFs afero.Fs,
+ targetPathBuilder func() page.TargetPaths,
+ osFileInfo os.FileInfo,
+ sourceFilename,
+ baseFilename string,
+ mediaType media.Type) *genericResource {
+ return r.newGenericResourceWithBase(
+ sourceFs,
+ nil,
+ nil,
+ targetPathBuilder,
+ osFileInfo,
+ sourceFilename,
+ baseFilename,
+ mediaType,
+ )
+
+}
+
+func (r *Spec) newGenericResourceWithBase(
+ sourceFs afero.Fs,
+ openReadSeekerCloser resource.OpenReadSeekCloser,
+ targetPathBaseDirs []string,
+ targetPathBuilder func() page.TargetPaths,
+ osFileInfo os.FileInfo,
+ sourceFilename,
+ baseFilename string,
+ mediaType media.Type) *genericResource {
+
+ if osFileInfo != nil && osFileInfo.IsDir() {
+ panic(fmt.Sprintf("dirs not supported resource types: %v", osFileInfo))
+ }
+
+ // This value is used both to construct URLs and file paths, but start
+ // with a Unix-styled path.
+ baseFilename = helpers.ToSlashTrimLeading(baseFilename)
+ fpath, fname := path.Split(baseFilename)
+
+ var resourceType string
+ if mediaType.MainType == "image" {
+ resourceType = mediaType.MainType
+ } else {
+ resourceType = mediaType.SubType
+ }
+
+ pathDescriptor := &resourcePathDescriptor{
+ baseTargetPathDirs: helpers.UniqueStringsReuse(targetPathBaseDirs),
+ targetPathBuilder: targetPathBuilder,
+ relTargetDirFile: dirFile{dir: fpath, file: fname},
+ }
+
+ gfi := &resourceFileInfo{
+ fi: osFileInfo,
+ openReadSeekerCloser: openReadSeekerCloser,
+ sourceFs: sourceFs,
+ sourceFilename: sourceFilename,
+ h: &resourceHash{},
+ }
+
+ g := &genericResource{
+ resourceFileInfo: gfi,
+ resourcePathDescriptor: pathDescriptor,
+ mediaType: mediaType,
+ resourceType: resourceType,
+ spec: r,
+ params: make(map[string]interface{}),
+ name: baseFilename,
+ title: baseFilename,
+ resourceContent: &resourceContent{},
+ }
+
+ return g
+
+}
+
+func (r *Spec) newResource(sourceFs afero.Fs, fd ResourceSourceDescriptor) (resource.Resource, error) {
+ fi := fd.FileInfo
+ var sourceFilename string
+
+ if fd.OpenReadSeekCloser != nil {
+ } else if fd.SourceFilename != "" {
+ var err error
+ fi, err = sourceFs.Stat(fd.SourceFilename)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil, nil
+ }
+ return nil, err
+ }
+ sourceFilename = fd.SourceFilename
+ } else {
+ sourceFilename = fd.SourceFile.Filename()
+ }
+
+ if fd.RelTargetFilename == "" {
+ fd.RelTargetFilename = sourceFilename
+ }
+
+ ext := strings.ToLower(filepath.Ext(fd.RelTargetFilename))
+ mimeType, found := r.MediaTypes.GetFirstBySuffix(strings.TrimPrefix(ext, "."))
+ // TODO(bep) we need to handle these ambigous types better, but in this context
+ // we most likely want the application/xml type.
+ if mimeType.Suffix() == "xml" && mimeType.SubType == "rss" {
+ mimeType, found = r.MediaTypes.GetByType("application/xml")
+ }
+
+ if !found {
+ // A fallback. Note that mime.TypeByExtension is slow by Hugo standards,
+ // so we should configure media types to avoid this lookup for most
+ // situations.
+ mimeStr := mime.TypeByExtension(ext)
+ if mimeStr != "" {
+ mimeType, _ = media.FromStringAndExt(mimeStr, ext)
+ }
+ }
+
+ gr := r.newGenericResourceWithBase(
+ sourceFs,
+ fd.OpenReadSeekCloser,
+ fd.TargetBasePaths,
+ fd.TargetPaths,
+ fi,
+ sourceFilename,
+ fd.RelTargetFilename,
+ mimeType)
+
+ if mimeType.MainType == "image" {
+ imgFormat, ok := images.ImageFormatFromExt(ext)
+ if ok {
+ ir := &imageResource{
+ Image: images.NewImage(imgFormat, r.imaging, nil, gr),
+ baseResource: gr,
+ }
+ return newResourceAdapter(gr.spec, fd.LazyPublish, ir), nil
+ }
+
+ }
+
+ return newResourceAdapter(gr.spec, fd.LazyPublish, gr), nil
+
+}
+
+func (r *Spec) newResourceFor(fd ResourceSourceDescriptor) (resource.Resource, error) {
+ if fd.OpenReadSeekCloser == nil {
+ if fd.SourceFile != nil && fd.SourceFilename != "" {
+ return nil, errors.New("both SourceFile and AbsSourceFilename provided")
+ } else if fd.SourceFile == nil && fd.SourceFilename == "" {
+ return nil, errors.New("either SourceFile or AbsSourceFilename must be provided")
+ }
+ }
+
+ if fd.RelTargetFilename == "" {
+ fd.RelTargetFilename = fd.Filename()
+ }
+
+ if len(fd.TargetBasePaths) == 0 {
+ // If not set, we publish the same resource to all hosts.
+ fd.TargetBasePaths = r.MultihostTargetBasePaths
+ }
+
+ return r.newResource(fd.Fs, fd)
+}
DIR diff --git a/resources/resource_test.go b/resources/resource_test.go
@@ -32,7 +32,7 @@ import (
func TestGenericResource(t *testing.T) {
c := qt.New(t)
- spec := newTestResourceSpec(c)
+ spec := newTestResourceSpec(specDescriptor{c: c})
r := spec.newGenericResource(nil, nil, nil, "/a/foo.css", "foo.css", media.CSSType)
@@ -44,7 +44,7 @@ func TestGenericResource(t *testing.T) {
func TestGenericResourceWithLinkFacory(t *testing.T) {
c := qt.New(t)
- spec := newTestResourceSpec(c)
+ spec := newTestResourceSpec(specDescriptor{c: c})
factory := newTargetPaths("/foo")
@@ -58,7 +58,7 @@ func TestGenericResourceWithLinkFacory(t *testing.T) {
func TestNewResourceFromFilename(t *testing.T) {
c := qt.New(t)
- spec := newTestResourceSpec(c)
+ spec := newTestResourceSpec(specDescriptor{c: c})
writeSource(t, spec.Fs, "content/a/b/logo.png", "image")
writeSource(t, spec.Fs, "content/a/b/data.json", "json")
@@ -79,14 +79,11 @@ func TestNewResourceFromFilename(t *testing.T) {
c.Assert(r, qt.Not(qt.IsNil))
c.Assert(r.ResourceType(), qt.Equals, "json")
- cloned := r.(resource.Cloner).WithNewBase("aceof")
- c.Assert(cloned.ResourceType(), qt.Equals, r.ResourceType())
- c.Assert(cloned.RelPermalink(), qt.Equals, "/aceof/a/b/data.json")
}
func TestNewResourceFromFilenameSubPathInBaseURL(t *testing.T) {
c := qt.New(t)
- spec := newTestResourceSpecForBaseURL(c, "https://example.com/docs")
+ spec := newTestResourceSpec(specDescriptor{c: c, baseURL: "https://example.com/docs"})
writeSource(t, spec.Fs, "content/a/b/logo.png", "image")
bfs := afero.NewBasePathFs(spec.Fs.Source, "content")
@@ -99,8 +96,6 @@ func TestNewResourceFromFilenameSubPathInBaseURL(t *testing.T) {
c.Assert(r.ResourceType(), qt.Equals, "image")
c.Assert(r.RelPermalink(), qt.Equals, "/docs/a/b/logo.png")
c.Assert(r.Permalink(), qt.Equals, "https://example.com/docs/a/b/logo.png")
- img := r.(*Image)
- c.Assert(img.targetFilenames()[0], qt.Equals, filepath.FromSlash("/a/b/logo.png"))
}
@@ -108,7 +103,7 @@ var pngType, _ = media.FromStringAndExt("image/png", "png")
func TestResourcesByType(t *testing.T) {
c := qt.New(t)
- spec := newTestResourceSpec(c)
+ spec := newTestResourceSpec(specDescriptor{c: c})
resources := resource.Resources{
spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType),
spec.newGenericResource(nil, nil, nil, "/a/logo.png", "logo.css", pngType),
@@ -122,7 +117,7 @@ func TestResourcesByType(t *testing.T) {
func TestResourcesGetByPrefix(t *testing.T) {
c := qt.New(t)
- spec := newTestResourceSpec(c)
+ spec := newTestResourceSpec(specDescriptor{c: c})
resources := resource.Resources{
spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType),
spec.newGenericResource(nil, nil, nil, "/a/logo1.png", "logo1.png", pngType),
@@ -151,7 +146,7 @@ func TestResourcesGetByPrefix(t *testing.T) {
func TestResourcesGetMatch(t *testing.T) {
c := qt.New(t)
- spec := newTestResourceSpec(c)
+ spec := newTestResourceSpec(specDescriptor{c: c})
resources := resource.Resources{
spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType),
spec.newGenericResource(nil, nil, nil, "/a/logo1.png", "logo1.png", pngType),
@@ -213,7 +208,7 @@ func BenchmarkResourcesMatch(b *testing.B) {
// my own curiosity.
func BenchmarkResourcesMatchA100(b *testing.B) {
c := qt.New(b)
- spec := newTestResourceSpec(c)
+ spec := newTestResourceSpec(specDescriptor{c: c})
a100 := strings.Repeat("a", 100)
pattern := "a*a*a*a*a*a*a*a*b"
@@ -228,7 +223,7 @@ func BenchmarkResourcesMatchA100(b *testing.B) {
func benchResources(b *testing.B) resource.Resources {
c := qt.New(b)
- spec := newTestResourceSpec(c)
+ spec := newTestResourceSpec(specDescriptor{c: c})
var resources resource.Resources
for i := 0; i < 30; i++ {
@@ -252,7 +247,7 @@ func benchResources(b *testing.B) resource.Resources {
func BenchmarkAssignMetadata(b *testing.B) {
c := qt.New(b)
- spec := newTestResourceSpec(c)
+ spec := newTestResourceSpec(specDescriptor{c: c})
for i := 0; i < b.N; i++ {
b.StopTimer()
DIR diff --git a/resources/resource_transformers/htesting/testhelpers.go b/resources/resource_transformers/htesting/testhelpers.go
@@ -0,0 +1,80 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package htesting
+
+import (
+ "path/filepath"
+
+ "github.com/gohugoio/hugo/cache/filecache"
+ "github.com/gohugoio/hugo/helpers"
+ "github.com/gohugoio/hugo/hugofs"
+ "github.com/gohugoio/hugo/media"
+ "github.com/gohugoio/hugo/output"
+ "github.com/gohugoio/hugo/resources"
+ "github.com/spf13/afero"
+ "github.com/spf13/viper"
+)
+
+func NewTestResourceSpec() (*resources.Spec, error) {
+ cfg := viper.New()
+ cfg.Set("baseURL", "https://example.org")
+ cfg.Set("publishDir", "public")
+
+ imagingCfg := map[string]interface{}{
+ "resampleFilter": "linear",
+ "quality": 68,
+ "anchor": "left",
+ }
+
+ cfg.Set("imaging", imagingCfg)
+
+ fs := hugofs.NewMem(cfg)
+
+ s, err := helpers.NewPathSpec(fs, cfg, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ filecaches, err := filecache.NewCaches(s)
+ if err != nil {
+ return nil, err
+ }
+
+ spec, err := resources.NewSpec(s, filecaches, nil, output.DefaultFormats, media.DefaultTypes)
+ return spec, err
+}
+
+func NewResourceTransformer(filename, content string) (resources.ResourceTransformer, error) {
+ spec, err := NewTestResourceSpec()
+ if err != nil {
+ return nil, err
+ }
+ return NewResourceTransformerForSpec(spec, filename, content)
+}
+
+func NewResourceTransformerForSpec(spec *resources.Spec, filename, content string) (resources.ResourceTransformer, error) {
+ filename = filepath.FromSlash(filename)
+
+ fs := spec.Fs.Source
+ if err := afero.WriteFile(fs, filename, []byte(content), 0777); err != nil {
+ return nil, err
+ }
+
+ r, err := spec.New(resources.ResourceSourceDescriptor{Fs: fs, SourceFilename: filename})
+ if err != nil {
+ return nil, err
+ }
+
+ return r.(resources.ResourceTransformer), nil
+}
DIR diff --git a/resources/resource_transformers/integrity/integrity.go b/resources/resource_transformers/integrity/integrity.go
@@ -23,6 +23,8 @@ import (
"html/template"
"io"
+ "github.com/gohugoio/hugo/resources/internal"
+
"github.com/pkg/errors"
"github.com/gohugoio/hugo/resources"
@@ -46,8 +48,8 @@ type fingerprintTransformation struct {
algo string
}
-func (t *fingerprintTransformation) Key() resources.ResourceTransformationKey {
- return resources.NewResourceTransformationKey("fingerprint", t.algo)
+func (t *fingerprintTransformation) Key() internal.ResourceTransformationKey {
+ return internal.NewResourceTransformationKey("fingerprint", t.algo)
}
// Transform creates a MD5 hash of the Resource content and inserts that hash before
@@ -59,7 +61,17 @@ func (t *fingerprintTransformation) Transform(ctx *resources.ResourceTransformat
return err
}
- io.Copy(io.MultiWriter(h, ctx.To), ctx.From)
+ var w io.Writer
+ if rc, ok := ctx.From.(io.ReadSeeker); ok {
+ // This transformation does not change the content, so try to
+ // avoid writing to To if we can.
+ defer rc.Seek(0, 0)
+ w = h
+ } else {
+ w = io.MultiWriter(h, ctx.To)
+ }
+
+ io.Copy(w, ctx.From)
d, err := digest(h)
if err != nil {
return err
@@ -91,15 +103,12 @@ func newHash(algo string) (hash.Hash, error) {
// the base64-encoded Subresource Integrity hash, so you will have to stay away from
// md5 if you plan to use both.
// See https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity
-func (c *Client) Fingerprint(res resource.Resource, algo string) (resource.Resource, error) {
+func (c *Client) Fingerprint(res resources.ResourceTransformer, algo string) (resource.Resource, error) {
if algo == "" {
algo = defaultHashAlgo
}
- return c.rs.Transform(
- res,
- &fingerprintTransformation{algo: algo},
- )
+ return res.Transform(&fingerprintTransformation{algo: algo})
}
func integrity(algo string, sum []byte) template.HTMLAttr {
DIR diff --git a/resources/resource_transformers/integrity/integrity_test.go b/resources/resource_transformers/integrity/integrity_test.go
@@ -14,9 +14,13 @@
package integrity
import (
+ "html/template"
"testing"
+ "github.com/gohugoio/hugo/resources/resource"
+
qt "github.com/frankban/quicktest"
+ "github.com/gohugoio/hugo/resources/resource_transformers/htesting"
)
func TestHashFromAlgo(t *testing.T) {
@@ -46,3 +50,23 @@ func TestHashFromAlgo(t *testing.T) {
})
}
}
+
+func TestTransform(t *testing.T) {
+ c := qt.New(t)
+
+ spec, err := htesting.NewTestResourceSpec()
+ c.Assert(err, qt.IsNil)
+ client := New(spec)
+
+ r, err := htesting.NewResourceTransformerForSpec(spec, "hugo.txt", "Hugo Rocks!")
+ c.Assert(err, qt.IsNil)
+
+ transformed, err := client.Fingerprint(r, "")
+
+ c.Assert(err, qt.IsNil)
+ c.Assert(transformed.RelPermalink(), qt.Equals, "/hugo.a5ad1c6961214a55de53c1ce6e60d27b6b761f54851fa65e33066460dfa6a0db.txt")
+ c.Assert(transformed.Data(), qt.DeepEquals, map[string]interface{}{"Integrity": template.HTMLAttr("sha256-pa0caWEhSlXeU8HObmDSe2t2H1SFH6ZeMwZkYN+moNs=")})
+ content, err := transformed.(resource.ContentProvider).Content()
+ c.Assert(err, qt.IsNil)
+ c.Assert(content, qt.Equals, "Hugo Rocks!")
+}
DIR diff --git a/resources/resource_transformers/minifier/minify.go b/resources/resource_transformers/minifier/minify.go
@@ -16,6 +16,7 @@ package minifier
import (
"github.com/gohugoio/hugo/minifiers"
"github.com/gohugoio/hugo/resources"
+ "github.com/gohugoio/hugo/resources/internal"
"github.com/gohugoio/hugo/resources/resource"
)
@@ -37,8 +38,8 @@ type minifyTransformation struct {
m minifiers.Client
}
-func (t *minifyTransformation) Key() resources.ResourceTransformationKey {
- return resources.NewResourceTransformationKey("minify")
+func (t *minifyTransformation) Key() internal.ResourceTransformationKey {
+ return internal.NewResourceTransformationKey("minify")
}
func (t *minifyTransformation) Transform(ctx *resources.ResourceTransformationCtx) error {
@@ -49,11 +50,10 @@ func (t *minifyTransformation) Transform(ctx *resources.ResourceTransformationCt
return nil
}
-func (c *Client) Minify(res resource.Resource) (resource.Resource, error) {
- return c.rs.Transform(
- res,
- &minifyTransformation{
- rs: c.rs,
- m: c.m},
- )
+func (c *Client) Minify(res resources.ResourceTransformer) (resource.Resource, error) {
+ return res.Transform(&minifyTransformation{
+ rs: c.rs,
+ m: c.m,
+ })
+
}
DIR diff --git a/resources/resource_transformers/minifier/minify_test.go b/resources/resource_transformers/minifier/minify_test.go
@@ -0,0 +1,43 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package minifier
+
+import (
+ "testing"
+
+ "github.com/gohugoio/hugo/resources/resource"
+
+ qt "github.com/frankban/quicktest"
+ "github.com/gohugoio/hugo/resources/resource_transformers/htesting"
+)
+
+func TestTransform(t *testing.T) {
+ c := qt.New(t)
+
+ spec, err := htesting.NewTestResourceSpec()
+ c.Assert(err, qt.IsNil)
+ client := New(spec)
+
+ r, err := htesting.NewResourceTransformerForSpec(spec, "hugo.html", "<h1> Hugo Rocks! </h1>")
+ c.Assert(err, qt.IsNil)
+
+ transformed, err := client.Minify(r)
+ c.Assert(err, qt.IsNil)
+
+ c.Assert(transformed.RelPermalink(), qt.Equals, "/hugo.min.html")
+ content, err := transformed.(resource.ContentProvider).Content()
+ c.Assert(err, qt.IsNil)
+ c.Assert(content, qt.Equals, "<h1>Hugo Rocks!</h1>")
+
+}
DIR diff --git a/resources/resource_transformers/postcss/postcss.go b/resources/resource_transformers/postcss/postcss.go
@@ -17,6 +17,7 @@ import (
"io"
"path/filepath"
+ "github.com/gohugoio/hugo/resources/internal"
"github.com/spf13/cast"
"github.com/gohugoio/hugo/hugofs"
@@ -98,8 +99,8 @@ type postcssTransformation struct {
rs *resources.Spec
}
-func (t *postcssTransformation) Key() resources.ResourceTransformationKey {
- return resources.NewResourceTransformationKey("postcss", t.options)
+func (t *postcssTransformation) Key() internal.ResourceTransformationKey {
+ return internal.NewResourceTransformationKey("postcss", t.options)
}
// Transform shells out to postcss-cli to do the heavy lifting.
@@ -187,9 +188,6 @@ func (t *postcssTransformation) Transform(ctx *resources.ResourceTransformationC
}
// Process transforms the given Resource with the PostCSS processor.
-func (c *Client) Process(res resource.Resource, options Options) (resource.Resource, error) {
- return c.rs.Transform(
- res,
- &postcssTransformation{rs: c.rs, options: options},
- )
+func (c *Client) Process(res resources.ResourceTransformer, options Options) (resource.Resource, error) {
+ return res.Transform(&postcssTransformation{rs: c.rs, options: options})
}
DIR diff --git a/resources/resource_transformers/templates/execute_as_template.go b/resources/resource_transformers/templates/execute_as_template.go
@@ -17,6 +17,7 @@ package templates
import (
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/resources"
+ "github.com/gohugoio/hugo/resources/internal"
"github.com/gohugoio/hugo/resources/resource"
"github.com/gohugoio/hugo/tpl"
"github.com/pkg/errors"
@@ -47,8 +48,8 @@ type executeAsTemplateTransform struct {
data interface{}
}
-func (t *executeAsTemplateTransform) Key() resources.ResourceTransformationKey {
- return resources.NewResourceTransformationKey("execute-as-template", t.targetPath)
+func (t *executeAsTemplateTransform) Key() internal.ResourceTransformationKey {
+ return internal.NewResourceTransformationKey("execute-as-template", t.targetPath)
}
func (t *executeAsTemplateTransform) Transform(ctx *resources.ResourceTransformationCtx) error {
@@ -63,14 +64,11 @@ func (t *executeAsTemplateTransform) Transform(ctx *resources.ResourceTransforma
return templ.Execute(ctx.To, t.data)
}
-func (c *Client) ExecuteAsTemplate(res resource.Resource, targetPath string, data interface{}) (resource.Resource, error) {
- return c.rs.Transform(
- res,
- &executeAsTemplateTransform{
- rs: c.rs,
- targetPath: helpers.ToSlashTrimLeading(targetPath),
- textTemplate: c.textTemplate,
- data: data,
- },
- )
+func (c *Client) ExecuteAsTemplate(res resources.ResourceTransformer, targetPath string, data interface{}) (resource.Resource, error) {
+ return res.Transform(&executeAsTemplateTransform{
+ rs: c.rs,
+ targetPath: helpers.ToSlashTrimLeading(targetPath),
+ textTemplate: c.textTemplate,
+ data: data,
+ })
}
DIR diff --git a/resources/resource_transformers/tocss/scss/client.go b/resources/resource_transformers/tocss/scss/client.go
@@ -18,6 +18,7 @@ import (
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/hugolib/filesystems"
"github.com/gohugoio/hugo/resources"
+ "github.com/gohugoio/hugo/resources/internal"
"github.com/gohugoio/hugo/resources/resource"
"github.com/spf13/afero"
@@ -68,7 +69,7 @@ type options struct {
to scss.Options
}
-func (c *Client) ToCSS(res resource.Resource, opts Options) (resource.Resource, error) {
+func (c *Client) ToCSS(res resources.ResourceTransformer, opts Options) (resource.Resource, error) {
internalOptions := options{
from: opts,
}
@@ -83,10 +84,7 @@ func (c *Client) ToCSS(res resource.Resource, opts Options) (resource.Resource,
internalOptions.to.Precision = 8
}
- return c.rs.Transform(
- res,
- &toCSSTransformation{c: c, options: internalOptions},
- )
+ return res.Transform(&toCSSTransformation{c: c, options: internalOptions})
}
type toCSSTransformation struct {
@@ -94,8 +92,8 @@ type toCSSTransformation struct {
options options
}
-func (t *toCSSTransformation) Key() resources.ResourceTransformationKey {
- return resources.NewResourceTransformationKey("tocss", t.options.from)
+func (t *toCSSTransformation) Key() internal.ResourceTransformationKey {
+ return internal.NewResourceTransformationKey("tocss", t.options.from)
}
func DecodeOptions(m map[string]interface{}) (opts Options, err error) {
DIR diff --git a/resources/smartcrop.go b/resources/smartcrop.go
@@ -1,77 +0,0 @@
-// Copyright 2019 The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package resources
-
-import (
- "image"
-
- "github.com/disintegration/imaging"
- "github.com/muesli/smartcrop"
-)
-
-const (
- // Do not change.
- smartCropIdentifier = "smart"
-
- // This is just a increment, starting on 1. If Smart Crop improves its cropping, we
- // need a way to trigger a re-generation of the crops in the wild, so increment this.
- smartCropVersionNumber = 1
-)
-
-// Needed by smartcrop
-type imagingResizer struct {
- filter imaging.ResampleFilter
-}
-
-func (r imagingResizer) Resize(img image.Image, width, height uint) image.Image {
- return imaging.Resize(img, int(width), int(height), r.filter)
-}
-
-func newSmartCropAnalyzer(filter imaging.ResampleFilter) smartcrop.Analyzer {
- return smartcrop.NewAnalyzer(imagingResizer{filter: filter})
-}
-
-func smartCrop(img image.Image, width, height int, anchor imaging.Anchor, filter imaging.ResampleFilter) (*image.NRGBA, error) {
-
- if width <= 0 || height <= 0 {
- return &image.NRGBA{}, nil
- }
-
- srcBounds := img.Bounds()
- srcW := srcBounds.Dx()
- srcH := srcBounds.Dy()
-
- if srcW <= 0 || srcH <= 0 {
- return &image.NRGBA{}, nil
- }
-
- if srcW == width && srcH == height {
- return imaging.Clone(img), nil
- }
-
- smart := newSmartCropAnalyzer(filter)
-
- rect, err := smart.FindBestCrop(img, width, height)
-
- if err != nil {
- return nil, err
- }
-
- b := img.Bounds().Intersect(rect)
-
- cropped := imaging.Crop(img, b)
-
- return imaging.Resize(cropped, width, height, filter), nil
-
-}
DIR diff --git a/resources/testhelpers_test.go b/resources/testhelpers_test.go
@@ -4,8 +4,6 @@ import (
"path/filepath"
"testing"
- "github.com/gohugoio/hugo/htesting/hqt"
-
"image"
"io"
"io/ioutil"
@@ -28,8 +26,10 @@ import (
"github.com/spf13/viper"
)
-func newTestResourceSpec(c *qt.C) *Spec {
- return newTestResourceSpecForBaseURL(c, "https://example.com/")
+type specDescriptor struct {
+ baseURL string
+ c *qt.C
+ fs afero.Fs
}
func createTestCfg() *viper.Viper {
@@ -54,7 +54,20 @@ func createTestCfg() *viper.Viper {
}
-func newTestResourceSpecForBaseURL(c *qt.C, baseURL string) *Spec {
+func newTestResourceSpec(desc specDescriptor) *Spec {
+
+ baseURL := desc.baseURL
+ if baseURL == "" {
+ baseURL = "https://example.com/"
+ }
+
+ afs := desc.fs
+ if afs == nil {
+ afs = afero.NewMemMapFs()
+ }
+
+ c := desc.c
+
cfg := createTestCfg()
cfg.Set("baseURL", baseURL)
@@ -66,7 +79,8 @@ func newTestResourceSpecForBaseURL(c *qt.C, baseURL string) *Spec {
cfg.Set("imaging", imagingCfg)
- fs := hugofs.NewMem(cfg)
+ fs := hugofs.NewFrom(afs, cfg)
+ fs.Destination = hugofs.NewCreateCountingFs(fs.Destination)
s, err := helpers.NewPathSpec(fs, cfg, nil)
c.Assert(err, qt.IsNil)
@@ -117,19 +131,23 @@ func newTestResourceOsFs(c *qt.C) *Spec {
}
-func fetchSunset(c *qt.C) *Image {
+func fetchSunset(c *qt.C) resource.Image {
return fetchImage(c, "sunset.jpg")
}
-func fetchImage(c *qt.C, name string) *Image {
- spec := newTestResourceSpec(c)
+func fetchImage(c *qt.C, name string) resource.Image {
+ spec := newTestResourceSpec(specDescriptor{c: c})
return fetchImageForSpec(spec, c, name)
}
-
-func fetchImageForSpec(spec *Spec, c *qt.C, name string) *Image {
+func fetchImageForSpec(spec *Spec, c *qt.C, name string) resource.Image {
r := fetchResourceForSpec(spec, c, name)
- c.Assert(r, hqt.IsSameType, &Image{})
- return r.(*Image)
+
+ img := r.(resource.Image)
+
+ c.Assert(img, qt.Not(qt.IsNil))
+ c.Assert(img.(specProvider).getSpec(), qt.Not(qt.IsNil))
+
+ return img
}
func fetchResourceForSpec(spec *Spec, c *qt.C, name string) resource.ContentResource {
DIR diff --git a/resources/transform.go b/resources/transform.go
@@ -15,45 +15,63 @@ package resources
import (
"bytes"
+ "fmt"
+ "io"
"path"
- "strconv"
"strings"
+ "sync"
+
+ "github.com/spf13/afero"
+
+ bp "github.com/gohugoio/hugo/bufferpool"
- "github.com/pkg/errors"
+ "github.com/gohugoio/hugo/resources/internal"
- "github.com/gohugoio/hugo/common/collections"
"github.com/gohugoio/hugo/common/herrors"
"github.com/gohugoio/hugo/common/hugio"
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/resources/resource"
- "github.com/mitchellh/hashstructure"
-
- "fmt"
- "io"
- "sync"
"github.com/gohugoio/hugo/media"
-
- bp "github.com/gohugoio/hugo/bufferpool"
)
var (
- _ resource.ContentResource = (*transformedResource)(nil)
- _ resource.ReadSeekCloserResource = (*transformedResource)(nil)
- _ collections.Slicer = (*transformedResource)(nil)
- _ resource.Identifier = (*transformedResource)(nil)
+ _ resource.ContentResource = (*resourceAdapter)(nil)
+ _ resource.ReadSeekCloserResource = (*resourceAdapter)(nil)
+ _ resource.Resource = (*resourceAdapter)(nil)
+ _ resource.Source = (*resourceAdapter)(nil)
+ _ resource.Identifier = (*resourceAdapter)(nil)
+ _ resource.ResourceMetaProvider = (*resourceAdapter)(nil)
)
-func (s *Spec) Transform(r resource.Resource, t ResourceTransformation) (resource.Resource, error) {
- if r == nil {
- return nil, errors.New("got nil Resource in transformation. Make sure you check with 'with' or 'if' when you get a resource, e.g. with resources.Get.")
+// These are transformations that need special support in Hugo that may not
+// be available when building the theme/site so we write the transformation
+// result to disk and reuse if needed for these,
+var transformationsToCacheOnDisk = map[string]bool{
+ "postcss": true,
+ "tocss": true,
+}
+
+func newResourceAdapter(spec *Spec, lazyPublish bool, target transformableResource) *resourceAdapter {
+ var po *publishOnce
+ if lazyPublish {
+ po = &publishOnce{}
+ }
+ return &resourceAdapter{
+ resourceTransformations: &resourceTransformations{},
+ resourceAdapterInner: &resourceAdapterInner{
+ spec: spec,
+ publishOnce: po,
+ target: target,
+ },
}
+}
- return &transformedResource{
- Resource: r,
- transformation: t,
- transformedResourceMetadata: transformedResourceMetadata{MetaData: make(map[string]interface{})},
- cache: s.ResourceCache}, nil
+// ResourceTransformation is the interface that a resource transformation step
+// needs to implement.
+type ResourceTransformation interface {
+ Key() internal.ResourceTransformationKey
+ Transform(ctx *ResourceTransformationCtx) error
}
type ResourceTransformationCtx struct {
@@ -95,20 +113,6 @@ func (ctx *ResourceTransformationCtx) AddOutPathIdentifier(identifier string) {
ctx.OutPath = ctx.addPathIdentifier(ctx.InPath, identifier)
}
-func (ctx *ResourceTransformationCtx) addPathIdentifier(inPath, identifier string) string {
- dir, file := path.Split(inPath)
- base, ext := helpers.PathAndExt(file)
- return path.Join(dir, (base + identifier + ext))
-}
-
-// ReplaceOutPathExtension transforming InPath to OutPath replacing the file
-// extension, e.g. ".scss"
-func (ctx *ResourceTransformationCtx) ReplaceOutPathExtension(newExt string) {
- dir, file := path.Split(ctx.InPath)
- base, _ := helpers.PathAndExt(file)
- ctx.OutPath = path.Join(dir, (base + newExt))
-}
-
// PublishSourceMap writes the content to the target folder of the main resource
// with the ".map" extension added.
func (ctx *ResourceTransformationCtx) PublishSourceMap(content string) error {
@@ -122,240 +126,198 @@ func (ctx *ResourceTransformationCtx) PublishSourceMap(content string) error {
return err
}
-// ResourceTransformationKey are provided by the different transformation implementations.
-// It identifies the transformation (name) and its configuration (elements).
-// We combine this in a chain with the rest of the transformations
-// with the target filename and a content hash of the origin to use as cache key.
-type ResourceTransformationKey struct {
- name string
- elements []interface{}
+// ReplaceOutPathExtension transforming InPath to OutPath replacing the file
+// extension, e.g. ".scss"
+func (ctx *ResourceTransformationCtx) ReplaceOutPathExtension(newExt string) {
+ dir, file := path.Split(ctx.InPath)
+ base, _ := helpers.PathAndExt(file)
+ ctx.OutPath = path.Join(dir, (base + newExt))
}
-// NewResourceTransformationKey creates a new ResourceTransformationKey from the transformation
-// name and elements. We will create a 64 bit FNV hash from the elements, which when combined
-// with the other key elements should be unique for all practical applications.
-func NewResourceTransformationKey(name string, elements ...interface{}) ResourceTransformationKey {
- return ResourceTransformationKey{name: name, elements: elements}
+func (ctx *ResourceTransformationCtx) addPathIdentifier(inPath, identifier string) string {
+ dir, file := path.Split(inPath)
+ base, ext := helpers.PathAndExt(file)
+ return path.Join(dir, (base + identifier + ext))
}
-// Do not change this without good reasons.
-func (k ResourceTransformationKey) key() string {
- if len(k.elements) == 0 {
- return k.name
- }
+type publishOnce struct {
+ publisherInit sync.Once
+ publisherErr error
+}
- sb := bp.GetBuffer()
- defer bp.PutBuffer(sb)
+type resourceAdapter struct {
+ commonResource
+ *resourceTransformations
+ *resourceAdapterInner
+}
- sb.WriteString(k.name)
- for _, element := range k.elements {
- hash, err := hashstructure.Hash(element, nil)
- if err != nil {
- panic(err)
- }
- sb.WriteString("_")
- sb.WriteString(strconv.FormatUint(hash, 10))
+func (r *resourceAdapter) Content() (interface{}, error) {
+ r.init(false, true)
+ if r.transformationsErr != nil {
+ return nil, r.transformationsErr
}
-
- return sb.String()
+ return r.target.Content()
}
-// ResourceTransformation is the interface that a resource transformation step
-// needs to implement.
-type ResourceTransformation interface {
- Key() ResourceTransformationKey
- Transform(ctx *ResourceTransformationCtx) error
+func (r *resourceAdapter) Data() interface{} {
+ r.init(false, false)
+ return r.target.Data()
}
-// We will persist this information to disk.
-type transformedResourceMetadata struct {
- Target string `json:"Target"`
- MediaTypeV string `json:"MediaType"`
- MetaData map[string]interface{} `json:"Data"`
+func (r *resourceAdapter) Fill(spec string) (resource.Image, error) {
+ return r.getImageOps().Fill(spec)
}
-type transformedResource struct {
- commonResource
+func (r *resourceAdapter) Fit(spec string) (resource.Image, error) {
+ return r.getImageOps().Fit(spec)
+}
- cache *ResourceCache
+func (r *resourceAdapter) Height() int {
+ return r.getImageOps().Height()
+}
- // This is the filename inside resources/_gen/assets
- sourceFilename string
+func (r *resourceAdapter) Key() string {
+ r.init(false, false)
+ return r.target.(resource.Identifier).Key()
+}
- linker permalinker
+func (r *resourceAdapter) MediaType() media.Type {
+ r.init(false, false)
+ return r.target.MediaType()
+}
- // The transformation to apply.
- transformation ResourceTransformation
+func (r *resourceAdapter) Name() string {
+ r.init(false, false)
+ return r.target.Name()
+}
- // We apply the tranformations lazily.
- transformInit sync.Once
- transformErr error
+func (r *resourceAdapter) Params() map[string]interface{} {
+ r.init(false, false)
+ return r.target.Params()
+}
- // We delay publishing until either .RelPermalink or .Permalink
- // is invoked.
- publishInit sync.Once
- published bool
+func (r *resourceAdapter) Permalink() string {
+ r.init(true, false)
+ return r.target.Permalink()
+}
- // The transformed values
- content string
- contentInit sync.Once
- transformedResourceMetadata
+func (r *resourceAdapter) Publish() error {
+ r.init(false, false)
- // The source
- resource.Resource
+ return r.target.Publish()
}
-func (r *transformedResource) ReadSeekCloser() (hugio.ReadSeekCloser, error) {
- if err := r.initContent(); err != nil {
- return nil, err
- }
- return hugio.NewReadSeekerNoOpCloserFromString(r.content), nil
+func (r *resourceAdapter) ReadSeekCloser() (hugio.ReadSeekCloser, error) {
+ r.init(false, false)
+ return r.target.ReadSeekCloser()
}
-func (r *transformedResource) transferTransformedValues(another *transformedResource) {
- if another.content != "" {
- r.contentInit.Do(func() {
- r.content = another.content
- })
- }
- r.transformedResourceMetadata = another.transformedResourceMetadata
+func (r *resourceAdapter) RelPermalink() string {
+ r.init(true, false)
+ return r.target.RelPermalink()
}
-func (r *transformedResource) tryTransformedFileCache(key string) io.ReadCloser {
- fi, f, meta, found := r.cache.getFromFile(key)
- if !found {
- return nil
- }
- r.transformedResourceMetadata = meta
- r.sourceFilename = fi.Name
+func (r *resourceAdapter) Resize(spec string) (resource.Image, error) {
+ return r.getImageOps().Resize(spec)
+}
- return f
+func (r *resourceAdapter) ResourceType() string {
+ r.init(false, false)
+ return r.target.ResourceType()
}
-func (r *transformedResource) Content() (interface{}, error) {
- if err := r.initTransform(true, false); err != nil {
- return nil, err
- }
- if err := r.initContent(); err != nil {
- return "", err
- }
- return r.content, nil
+func (r *resourceAdapter) String() string {
+ return r.Name()
}
-func (r *transformedResource) Data() interface{} {
- if err := r.initTransform(false, false); err != nil {
- return noData
- }
- return r.MetaData
+func (r *resourceAdapter) Title() string {
+ r.init(false, false)
+ return r.target.Title()
}
-func (r *transformedResource) MediaType() media.Type {
- if err := r.initTransform(false, false); err != nil {
- return media.Type{}
+func (r resourceAdapter) Transform(t ...ResourceTransformation) (ResourceTransformer, error) {
+ r.resourceTransformations = &resourceTransformations{
+ transformations: append(r.transformations, t...),
}
- m, _ := r.cache.rs.MediaTypes.GetByType(r.MediaTypeV)
- return m
-}
-func (r *transformedResource) Key() string {
- if err := r.initTransform(false, false); err != nil {
- return ""
+ r.resourceAdapterInner = &resourceAdapterInner{
+ spec: r.spec,
+ publishOnce: &publishOnce{},
+ target: r.target,
}
- return r.linker.relPermalinkFor(r.Target)
+
+ return &r, nil
}
-func (r *transformedResource) Permalink() string {
- if err := r.initTransform(false, true); err != nil {
- return ""
- }
- return r.linker.permalinkFor(r.Target)
+func (r *resourceAdapter) Width() int {
+ return r.getImageOps().Width()
}
-func (r *transformedResource) RelPermalink() string {
- if err := r.initTransform(false, true); err != nil {
- return ""
+func (r *resourceAdapter) getImageOps() resource.ImageOps {
+ img, ok := r.target.(resource.ImageOps)
+ if !ok {
+ panic(fmt.Sprintf("%T is not an image", r.target))
}
- return r.linker.relPermalinkFor(r.Target)
+ r.init(false, false)
+ return img
}
-func (r *transformedResource) initContent() error {
- var err error
- r.contentInit.Do(func() {
- var b []byte
- _, b, err = r.cache.fileCache.GetBytes(r.sourceFilename)
- if err != nil {
- return
- }
- r.content = string(b)
- })
- return err
+func (r *resourceAdapter) getMetaAssigner() metaAssigner {
+ return r.target
}
-func (r *transformedResource) openPublishFileForWriting(relTargetPath string) (io.WriteCloser, error) {
- return helpers.OpenFilesForWriting(r.cache.rs.PublishFs, r.linker.relTargetPathsFor(relTargetPath)...)
+func (r *resourceAdapter) getSpec() *Spec {
+ return r.spec
}
-func (r *transformedResource) transform(setContent, publish bool) (err error) {
+func (r *resourceAdapter) publish() {
+ if r.publishOnce == nil {
+ return
+ }
- // This can be the last resource in a chain.
- // Rewind and create a processing chain.
- var chain []resource.Resource
- current := r
- for {
- rr := current.Resource
- chain = append(chain[:0], append([]resource.Resource{rr}, chain[0:]...)...)
- if tr, ok := rr.(*transformedResource); ok {
- current = tr
- } else {
- break
+ r.publisherInit.Do(func() {
+ r.publisherErr = r.target.Publish()
+
+ if r.publisherErr != nil {
+ r.spec.Logger.ERROR.Printf("Failed to publish Resource: %s", r.publisherErr)
}
- }
+ })
- // Append the current transformer at the end
- chain = append(chain, r)
+}
- first := chain[0]
+func (r *resourceAdapter) transform(publish, setContent bool) error {
+ cache := r.spec.ResourceCache
// Files with a suffix will be stored in cache (both on disk and in memory)
- // partitioned by their suffix. There will be other files below /other.
- // This partition is also how we determine what to delete on server reloads.
- var key, base string
- for _, element := range chain {
- switch v := element.(type) {
- case *transformedResource:
- key = key + "_" + v.transformation.Key().key()
- case permalinker:
- r.linker = v
- p := v.TargetPath()
- if p == "" {
- panic("target path needed for key creation")
- }
- base = ResourceCacheKey(p)
- default:
- return fmt.Errorf("transformation not supported for type %T", element)
- }
+ // partitioned by their suffix.
+ var key string
+ for _, tr := range r.transformations {
+ key = key + "_" + tr.Key().Value()
}
- key = r.cache.cleanKey(base) + "_" + helpers.MD5String(key)
+ base := ResourceCacheKey(r.target.TargetPath())
+
+ key = cache.cleanKey(base) + "_" + helpers.MD5String(key)
+
+ cached, found := cache.get(key)
- cached, found := r.cache.get(key)
if found {
- r.transferTransformedValues(cached.(*transformedResource))
- return
+ r.resourceAdapterInner = cached.(*resourceAdapterInner)
+ return nil
}
// Acquire a write lock for the named transformation.
- r.cache.nlocker.Lock(key)
+ cache.nlocker.Lock(key)
// Check the cache again.
- cached, found = r.cache.get(key)
+ cached, found = cache.get(key)
if found {
- r.transferTransformedValues(cached.(*transformedResource))
- r.cache.nlocker.Unlock(key)
- return
+ r.resourceAdapterInner = cached.(*resourceAdapterInner)
+ cache.nlocker.Unlock(key)
+ return nil
}
- defer r.cache.nlocker.Unlock(key)
- defer r.cache.set(key, r)
+ defer cache.nlocker.Unlock(key)
+ defer cache.set(key, r.resourceAdapterInner)
b1 := bp.GetBuffer()
b2 := bp.GetBuffer()
@@ -363,68 +325,77 @@ func (r *transformedResource) transform(setContent, publish bool) (err error) {
defer bp.PutBuffer(b2)
tctx := &ResourceTransformationCtx{
- Data: r.transformedResourceMetadata.MetaData,
- OpenResourcePublisher: r.openPublishFileForWriting,
+ Data: make(map[string]interface{}),
+ OpenResourcePublisher: r.target.openPublishFileForWriting,
}
- tctx.InMediaType = first.MediaType()
- tctx.OutMediaType = first.MediaType()
+ tctx.InMediaType = r.target.MediaType()
+ tctx.OutMediaType = r.target.MediaType()
+
+ startCtx := *tctx
+ updates := &transformationUpdate{startCtx: startCtx}
+
+ var contentrc hugio.ReadSeekCloser
- contentrc, err := contentReadSeekerCloser(first)
+ contentrc, err := contentReadSeekerCloser(r.target)
if err != nil {
return err
}
+
defer contentrc.Close()
tctx.From = contentrc
tctx.To = b1
- if r.linker != nil {
- tctx.InPath = r.linker.TargetPath()
- tctx.SourcePath = tctx.InPath
- }
+ tctx.InPath = r.target.TargetPath()
+ tctx.SourcePath = tctx.InPath
counter := 0
+ writeToFileCache := false
var transformedContentr io.Reader
- for _, element := range chain {
- tr, ok := element.(*transformedResource)
- if !ok {
- continue
- }
- counter++
- if counter != 1 {
+ for i, tr := range r.transformations {
+ if i != 0 {
tctx.InMediaType = tctx.OutMediaType
}
- if counter%2 == 0 {
- tctx.From = b1
- b2.Reset()
- tctx.To = b2
- } else {
- if counter != 1 {
- // The first reader is the file.
- tctx.From = b2
- }
- b1.Reset()
- tctx.To = b1
+
+ if !writeToFileCache {
+ writeToFileCache = transformationsToCacheOnDisk[tr.Key().Name]
}
- if err := tr.transformation.Transform(tctx); err != nil {
+ if i > 0 {
+ hasWrites := tctx.To.(*bytes.Buffer).Len() > 0
+ if hasWrites {
+ counter++
+ // Switch the buffers
+ if counter%2 == 0 {
+ tctx.From = b2
+ b1.Reset()
+ tctx.To = b1
+ } else {
+ tctx.From = b1
+ b2.Reset()
+ tctx.To = b2
+ }
+ }
+ }
- if err == herrors.ErrFeatureNotAvailable {
+ if err = tr.Transform(tctx); err != nil {
+ if writeToFileCache && err == herrors.ErrFeatureNotAvailable {
// This transformation is not available in this
// Hugo installation (scss not compiled in, PostCSS not available etc.)
// If a prepared bundle for this transformation chain is available, use that.
- f := r.tryTransformedFileCache(key)
+ f := r.target.tryTransformedFileCache(key, updates)
if f == nil {
errMsg := err.Error()
- if tr.transformation.Key().name == "postcss" {
+ if tr.Key().Name == "postcss" {
errMsg = "PostCSS not found; install with \"npm install postcss-cli\". See https://gohugo.io/hugo-pipes/postcss/"
}
- return fmt.Errorf("%s: failed to transform %q (%s): %s", strings.ToUpper(tr.transformation.Key().name), tctx.InPath, tctx.InMediaType.Type(), errMsg)
+ return fmt.Errorf("%s: failed to transform %q (%s): %s", strings.ToUpper(tr.Key().Name), tctx.InPath, tctx.InMediaType.Type(), errMsg)
}
transformedContentr = f
+ updates.sourceFs = cache.fileCache.Fs
defer f.Close()
// The reader above is all we need.
@@ -442,34 +413,35 @@ func (r *transformedResource) transform(setContent, publish bool) (err error) {
}
if transformedContentr == nil {
- r.Target = tctx.InPath
- r.MediaTypeV = tctx.OutMediaType.Type()
+ updates.updateFromCtx(tctx)
}
var publishwriters []io.WriteCloser
if publish {
- publicw, err := r.openPublishFileForWriting(r.Target)
+ publicw, err := r.target.openPublishFileForWriting(updates.targetPath)
if err != nil {
- r.transformErr = err
return err
}
- defer publicw.Close()
-
publishwriters = append(publishwriters, publicw)
}
if transformedContentr == nil {
- // Also write it to the cache
- fi, metaw, err := r.cache.writeMeta(key, r.transformedResourceMetadata)
- if err != nil {
- return err
+ if writeToFileCache {
+ // Also write it to the cache
+ fi, metaw, err := cache.writeMeta(key, updates.toTransformedResourceMetadata())
+ if err != nil {
+ return err
+ }
+ updates.sourceFilename = &fi.Name
+ updates.sourceFs = cache.fileCache.Fs
+ publishwriters = append(publishwriters, metaw)
}
- r.sourceFilename = fi.Name
-
- publishwriters = append(publishwriters, metaw)
- if counter > 0 {
+ // Any transofrmations reading from From must also write to To.
+ // This means that if the target buffer is empty, we can just reuse
+ // the original reader.
+ if b, ok := tctx.To.(*bytes.Buffer); ok && b.Len() > 0 {
transformedContentr = tctx.To.(*bytes.Buffer)
} else {
transformedContentr = contentrc
@@ -479,6 +451,8 @@ func (r *transformedResource) transform(setContent, publish bool) (err error) {
// Also write it to memory
var contentmemw *bytes.Buffer
+ setContent = setContent || !writeToFileCache
+
if setContent {
contentmemw = bp.GetBuffer()
defer bp.PutBuffer(contentmemw)
@@ -486,65 +460,111 @@ func (r *transformedResource) transform(setContent, publish bool) (err error) {
}
publishw := hugio.NewMultiWriteCloser(publishwriters...)
- _, r.transformErr = io.Copy(publishw, transformedContentr)
+ _, err = io.Copy(publishw, transformedContentr)
+ if err != nil {
+ return err
+ }
publishw.Close()
if setContent {
- r.contentInit.Do(func() {
- r.content = contentmemw.String()
- })
+ s := contentmemw.String()
+ updates.content = &s
+ }
+
+ newTarget, err := r.target.cloneWithUpdates(updates)
+ if err != nil {
+ return err
}
+ r.target = newTarget
return nil
}
-func (r *transformedResource) initTransform(setContent, publish bool) error {
- r.transformInit.Do(func() {
- r.published = publish
- if err := r.transform(setContent, publish); err != nil {
- r.transformErr = err
- r.cache.rs.Logger.ERROR.Println("error: failed to transform resource:", err)
+func (r *resourceAdapter) init(publish, setContent bool) {
+ r.initTransform(publish, setContent)
+}
+
+func (r *resourceAdapter) initTransform(publish, setContent bool) {
+ r.transformationsInit.Do(func() {
+ if len(r.transformations) == 0 {
+ // Nothing to do.
+ return
+ }
+
+ if publish {
+ // The transformation will write the content directly to
+ // the destination.
+ r.publishOnce = nil
}
+ r.transformationsErr = r.transform(publish, setContent)
+ if r.transformationsErr != nil {
+ r.spec.Logger.ERROR.Printf("Transformation failed: %s", r.transformationsErr)
+ }
})
- if !publish {
- return r.transformErr
+ if publish && r.publishOnce != nil {
+ r.publish()
}
+}
- r.publishInit.Do(func() {
- if r.published {
- return
- }
+type resourceAdapterInner struct {
+ target transformableResource
- r.published = true
+ spec *Spec
- // Copy the file from cache to /public
- _, src, err := r.cache.fileCache.Get(r.sourceFilename)
- if src == nil {
- panic(fmt.Sprintf("[BUG] resource cache file not found: %q", r.sourceFilename))
- }
+ // Handles publishing (to /public) if needed.
+ *publishOnce
+}
- if err == nil {
- defer src.Close()
+type resourceTransformations struct {
+ transformationsInit sync.Once
+ transformationsErr error
+ transformations []ResourceTransformation
+}
- var dst io.WriteCloser
- dst, err = r.openPublishFileForWriting(r.Target)
- if err == nil {
- defer dst.Close()
- io.Copy(dst, src)
- }
- }
+type transformableResource interface {
+ baseResourceInternal
- if err != nil {
- r.transformErr = err
- r.cache.rs.Logger.ERROR.Println("error: failed to publish resource:", err)
- return
- }
+ resource.ContentProvider
+ resource.Resource
+}
- })
+type transformationUpdate struct {
+ content *string
+ sourceFilename *string
+ sourceFs afero.Fs
+ targetPath string
+ mediaType media.Type
+ data map[string]interface{}
- return r.transformErr
+ startCtx ResourceTransformationCtx
+}
+
+func (u *transformationUpdate) isContenChanged() bool {
+ return u.content != nil || u.sourceFilename != nil
+}
+
+func (u *transformationUpdate) toTransformedResourceMetadata() transformedResourceMetadata {
+ return transformedResourceMetadata{
+ MediaTypeV: u.mediaType.Type(),
+ Target: u.targetPath,
+ MetaData: u.data,
+ }
+}
+
+func (u *transformationUpdate) updateFromCtx(ctx *ResourceTransformationCtx) {
+ u.targetPath = ctx.OutPath
+ u.mediaType = ctx.OutMediaType
+ u.data = ctx.Data
+ u.targetPath = ctx.InPath
+}
+
+// We will persist this information to disk.
+type transformedResourceMetadata struct {
+ Target string `json:"Target"`
+ MediaTypeV string `json:"MediaType"`
+ MetaData map[string]interface{} `json:"Data"`
}
// contentReadSeekerCloser returns a ReadSeekerCloser if possible for a given Resource.
DIR diff --git a/resources/transform_test.go b/resources/transform_test.go
@@ -14,23 +14,427 @@
package resources
import (
+ "encoding/base64"
+ "fmt"
+ "io"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "sync"
"testing"
+ "github.com/gohugoio/hugo/htesting"
+
+ "github.com/gohugoio/hugo/common/herrors"
+ "github.com/gohugoio/hugo/hugofs"
+
+ "github.com/gohugoio/hugo/media"
+ "github.com/gohugoio/hugo/resources/internal"
+
+ "github.com/gohugoio/hugo/helpers"
+
+ "github.com/gohugoio/hugo/resources/resource"
+ "github.com/spf13/afero"
+
qt "github.com/frankban/quicktest"
)
-type testStruct struct {
- Name string
- V1 int64
- V2 int32
- V3 int
- V4 uint64
-}
drkhsh.at:70 /scm/hugo/commit/f9978ed16476ca6d233a89669c62c798cdf9db9d.gph:5028: line too long