Add build.UseResourceCacheWhen - hugo - [fork] hugo port for 9front
HTML git clone git@git.drkhsh.at/hugo.git
DIR Log
DIR Files
DIR Refs
DIR Submodules
DIR README
DIR LICENSE
---
DIR commit 3d3fa5c3fe5ee0c9df59d682ee0acaba71a06ae1
DIR parent ee3d02134d9b46b10e5a0403c9986ee1833ae6c1
HTML Author: Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>
Date: Tue, 3 Mar 2020 08:32:02 +0100
Add build.UseResourceCacheWhen
Fixes #6993
Diffstat:
A config/commonConfig.go | 90 +++++++++++++++++++++++++++++++
A config/commonConfig_test.go | 60 +++++++++++++++++++++++++++++++
D config/sitemap.go | 44 -------------------------------
M docs/content/en/getting-started/co… | 18 ++++++++++++++++++
M hugolib/resource_chain_test.go | 83 ++++++++++++++++++++++++-------
M hugolib/site_render.go | 1 -
M langs/language.go | 1 +
M resources/resource_spec.go | 6 +++++-
M resources/transform.go | 59 ++++++++++++++++++++-----------
9 files changed, 278 insertions(+), 84 deletions(-)
---
DIR diff --git a/config/commonConfig.go b/config/commonConfig.go
@@ -0,0 +1,90 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import (
+ "strings"
+
+ "github.com/gohugoio/hugo/common/herrors"
+ "github.com/mitchellh/mapstructure"
+ "github.com/spf13/cast"
+ jww "github.com/spf13/jwalterweatherman"
+)
+
+var DefaultBuild = Build{
+ UseResourceCacheWhen: "fallback",
+}
+
+// Build holds some build related condfiguration.
+type Build struct {
+ UseResourceCacheWhen string // never, fallback, always. Default is fallback
+}
+
+func (b Build) UseResourceCache(err error) bool {
+ if b.UseResourceCacheWhen == "never" {
+ return false
+ }
+
+ if b.UseResourceCacheWhen == "fallback" {
+ return err == herrors.ErrFeatureNotAvailable
+ }
+
+ return true
+}
+
+func DecodeBuild(cfg Provider) Build {
+ m := cfg.GetStringMap("build")
+ b := DefaultBuild
+ if m == nil {
+ return b
+ }
+
+ err := mapstructure.WeakDecode(m, &b)
+ if err != nil {
+ return DefaultBuild
+ }
+
+ b.UseResourceCacheWhen = strings.ToLower(b.UseResourceCacheWhen)
+ when := b.UseResourceCacheWhen
+ if when != "never" && when != "always" && when != "fallback" {
+ b.UseResourceCacheWhen = "fallback"
+ }
+
+ return b
+}
+
+// Sitemap configures the sitemap to be generated.
+type Sitemap struct {
+ ChangeFreq string
+ Priority float64
+ Filename string
+}
+
+func DecodeSitemap(prototype Sitemap, input map[string]interface{}) Sitemap {
+
+ for key, value := range input {
+ switch key {
+ case "changefreq":
+ prototype.ChangeFreq = cast.ToString(value)
+ case "priority":
+ prototype.Priority = cast.ToFloat64(value)
+ case "filename":
+ prototype.Filename = cast.ToString(value)
+ default:
+ jww.WARN.Printf("Unknown Sitemap field: %s\n", key)
+ }
+ }
+
+ return prototype
+}
DIR diff --git a/config/commonConfig_test.go b/config/commonConfig_test.go
@@ -0,0 +1,60 @@
+// Copyright 2020 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import (
+ "errors"
+ "testing"
+
+ "github.com/gohugoio/hugo/common/herrors"
+
+ qt "github.com/frankban/quicktest"
+
+ "github.com/spf13/viper"
+)
+
+func TestBuild(t *testing.T) {
+ c := qt.New(t)
+
+ v := viper.New()
+ v.Set("build", map[string]interface{}{
+ "useResourceCacheWhen": "always",
+ })
+
+ b := DecodeBuild(v)
+
+ c.Assert(b.UseResourceCacheWhen, qt.Equals, "always")
+
+ v.Set("build", map[string]interface{}{
+ "useResourceCacheWhen": "foo",
+ })
+
+ b = DecodeBuild(v)
+
+ c.Assert(b.UseResourceCacheWhen, qt.Equals, "fallback")
+
+ c.Assert(b.UseResourceCache(herrors.ErrFeatureNotAvailable), qt.Equals, true)
+ c.Assert(b.UseResourceCache(errors.New("err")), qt.Equals, false)
+
+ b.UseResourceCacheWhen = "always"
+ c.Assert(b.UseResourceCache(herrors.ErrFeatureNotAvailable), qt.Equals, true)
+ c.Assert(b.UseResourceCache(errors.New("err")), qt.Equals, true)
+ c.Assert(b.UseResourceCache(nil), qt.Equals, true)
+
+ b.UseResourceCacheWhen = "never"
+ c.Assert(b.UseResourceCache(herrors.ErrFeatureNotAvailable), qt.Equals, false)
+ c.Assert(b.UseResourceCache(errors.New("err")), qt.Equals, false)
+ c.Assert(b.UseResourceCache(nil), qt.Equals, false)
+
+}
DIR diff --git a/config/sitemap.go b/config/sitemap.go
@@ -1,44 +0,0 @@
-// Copyright 2019 The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package config
-
-import (
- "github.com/spf13/cast"
- jww "github.com/spf13/jwalterweatherman"
-)
-
-// Sitemap configures the sitemap to be generated.
-type Sitemap struct {
- ChangeFreq string
- Priority float64
- Filename string
-}
-
-func DecodeSitemap(prototype Sitemap, input map[string]interface{}) Sitemap {
-
- for key, value := range input {
- switch key {
- case "changefreq":
- prototype.ChangeFreq = cast.ToString(value)
- case "priority":
- prototype.Priority = cast.ToFloat64(value)
- case "filename":
- prototype.Filename = cast.ToString(value)
- default:
- jww.WARN.Printf("Unknown Sitemap field: %s\n", key)
- }
- }
-
- return prototype
-}
DIR diff --git a/docs/content/en/getting-started/configuration.md b/docs/content/en/getting-started/configuration.md
@@ -87,6 +87,9 @@ baseURL
blackfriday
: See [Configure Blackfriday](/getting-started/configuration-markup#blackfriday)
+build
+: See [Configure Build](#configure-build)
+
buildDrafts (false)
: Include drafts when building.
@@ -288,6 +291,21 @@ enableemoji: true
```
{{% /note %}}
+## Configure Build
+
+{{< new-in "0.66.0" >}}
+
+The `build` configuration section contains global build-realated configuration options.
+
+{{< code-toggle file="config">}}
+[build]
+useResourceCacheWhen="fallback"
+{{< /code-toggle >}}
+
+
+useResourceCacheWhen
+: When to use the cached resources in `/resources/_gen` for PostCSS and ToCSS. Valid values are `never`, `always` and `fallback`. The last value means that the cache will be tried if PostCSS/extended version is not available.
+
## Configure Title Case
Set `titleCaseStyle` to specify the title style used by the [title](/functions/title/) template function and the automatic section titles in Hugo. It defaults to [AP Stylebook](https://www.apstylebook.com/) for title casing, but you can also set it to `Chicago` or `Go` (every word starts with a capital letter).
DIR diff --git a/hugolib/resource_chain_test.go b/hugolib/resource_chain_test.go
@@ -753,20 +753,17 @@ h1 {
c.Assert(err, qt.IsNil)
defer clean()
- v := viper.New()
- v.Set("workingDir", workDir)
- v.Set("disableKinds", []string{"taxonomyTerm", "taxonomy", "page"})
- b := newTestSitesBuilder(t).WithLogger(loggers.NewWarningLogger())
- // Need to use OS fs for this.
- b.Fs = hugofs.NewDefault(v)
- b.WithWorkingDir(workDir)
- b.WithViper(v)
-
- cssDir := filepath.Join(workDir, "assets", "css", "components")
- b.Assert(os.MkdirAll(cssDir, 0777), qt.IsNil)
-
- b.WithContent("p1.md", "")
- b.WithTemplates("index.html", `
+ newTestBuilder := func(v *viper.Viper) *sitesBuilder {
+ v.Set("workingDir", workDir)
+ v.Set("disableKinds", []string{"taxonomyTerm", "taxonomy", "page"})
+ b := newTestSitesBuilder(t).WithLogger(loggers.NewWarningLogger())
+ // Need to use OS fs for this.
+ b.Fs = hugofs.NewDefault(v)
+ b.WithWorkingDir(workDir)
+ b.WithViper(v)
+
+ b.WithContent("p1.md", "")
+ b.WithTemplates("index.html", `
{{ $options := dict "inlineImports" true }}
{{ $styles := resources.Get "css/styles.css" | resources.PostCSS $options }}
Styles RelPermalink: {{ $styles.RelPermalink }}
@@ -774,6 +771,15 @@ Styles RelPermalink: {{ $styles.RelPermalink }}
Styles Content: Len: {{ len $styles.Content }}|
`)
+
+ return b
+ }
+
+ b := newTestBuilder(viper.New())
+
+ cssDir := filepath.Join(workDir, "assets", "css", "components")
+ b.Assert(os.MkdirAll(cssDir, 0777), qt.IsNil)
+
b.WithSourceFile("assets/css/styles.css", tailwindCss)
b.WithSourceFile("assets/css/components/all.css", `
@import "a.css";
@@ -810,9 +816,52 @@ Styles RelPermalink: /css/styles.css
Styles Content: Len: 770878|
`)
- content := b.FileContent("public/css/styles.css")
+ assertCss := func(b *sitesBuilder) {
+ content := b.FileContent("public/css/styles.css")
+
+ b.Assert(strings.Contains(content, "class-in-a"), qt.Equals, true)
+ b.Assert(strings.Contains(content, "class-in-b"), qt.Equals, true)
+
+ }
+
+ assertCss(b)
+
+ build := func(s string, shouldFail bool) {
+ b.Assert(os.RemoveAll(filepath.Join(workDir, "public")), qt.IsNil)
+
+ v := viper.New()
+ v.Set("build", map[string]interface{}{
+ "useResourceCacheWhen": s,
+ })
+
+ b = newTestBuilder(v)
+
+ b.Assert(os.RemoveAll(filepath.Join(workDir, "public")), qt.IsNil)
+ b.Assert(os.RemoveAll(filepath.Join(workDir, "node_modules")), qt.IsNil)
+
+ if shouldFail {
+ b.BuildFail(BuildCfg{})
+ } else {
+ b.Build(BuildCfg{})
+ assertCss(b)
+ }
+ }
+
+ build("always", false)
+ build("fallback", false)
+
+ // Remove PostCSS
+ b.Assert(os.RemoveAll(filepath.Join(workDir, "node_modules")), qt.IsNil)
+
+ build("always", false)
+ build("fallback", false)
+ build("never", true)
+
+ // Remove cache
+ b.Assert(os.RemoveAll(filepath.Join(workDir, "resources")), qt.IsNil)
- b.Assert(strings.Contains(content, "class-in-a"), qt.Equals, true)
- b.Assert(strings.Contains(content, "class-in-b"), qt.Equals, true)
+ build("always", true)
+ build("fallback", true)
+ build("never", true)
}
DIR diff --git a/hugolib/site_render.go b/hugolib/site_render.go
@@ -59,7 +59,6 @@ func (s siteRenderContext) renderSingletonPages() bool {
// renderPages renders pages each corresponding to a markdown file.
// TODO(bep np doc
func (s *Site) renderPages(ctx *siteRenderContext) error {
-
numWorkers := config.GetNumWorkerMultiplier()
results := make(chan error)
DIR diff --git a/langs/language.go b/langs/language.go
@@ -36,6 +36,7 @@ var globalOnlySettings = map[string]bool{
strings.ToLower("multilingual"): true,
strings.ToLower("assetDir"): true,
strings.ToLower("resourceDir"): true,
+ strings.ToLower("build"): true,
}
// Language manages specific-language configuration.
DIR diff --git a/resources/resource_spec.go b/resources/resource_spec.go
@@ -22,6 +22,8 @@ import (
"path/filepath"
"strings"
+ "github.com/gohugoio/hugo/config"
+
"github.com/gohugoio/hugo/hugofs"
"github.com/gohugoio/hugo/helpers"
@@ -69,6 +71,7 @@ func NewSpec(
MediaTypes: mimeTypes,
OutputFormats: outputFormats,
Permalinks: permalinks,
+ BuildConfig: config.DecodeBuild(s.Cfg),
FileCaches: fileCaches,
imageCache: newImageCache(
fileCaches.ImageCache(),
@@ -92,7 +95,8 @@ type Spec struct {
TextTemplates tpl.TemplateParseFinder
- Permalinks page.PermalinkExpander
+ Permalinks page.PermalinkExpander
+ BuildConfig config.Build
// Holds default filter settings etc.
imaging *images.ImageProcessor
DIR diff --git a/resources/transform.go b/resources/transform.go
@@ -21,12 +21,13 @@ import (
"strings"
"sync"
+ "github.com/pkg/errors"
+
"github.com/gohugoio/hugo/resources/images/exif"
"github.com/spf13/afero"
bp "github.com/gohugoio/hugo/bufferpool"
- "github.com/gohugoio/hugo/common/herrors"
"github.com/gohugoio/hugo/common/hugio"
"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/helpers"
@@ -369,8 +370,9 @@ func (r *resourceAdapter) transform(publish, setContent bool) error {
tctx.InMediaType = tctx.OutMediaType
}
+ mayBeCachedOnDisk := transformationsToCacheOnDisk[tr.Key().Name]
if !writeToFileCache {
- writeToFileCache = transformationsToCacheOnDisk[tr.Key().Name]
+ writeToFileCache = mayBeCachedOnDisk
}
if i > 0 {
@@ -390,29 +392,44 @@ func (r *resourceAdapter) transform(publish, setContent bool) error {
}
}
- if err = tr.Transform(tctx); err != nil {
- if writeToFileCache && err == herrors.ErrFeatureNotAvailable {
+ notAvailableErr := func(err error) error {
+ errMsg := err.Error()
+ if tr.Key().Name == "postcss" {
// This transformation is not available in this
- // Hugo installation (scss not compiled in, PostCSS not available etc.)
- // If a prepared bundle for this transformation chain is available, use that.
- f := r.target.tryTransformedFileCache(key, updates)
- if f == nil {
- errMsg := err.Error()
- if tr.Key().Name == "postcss" {
- errMsg = "PostCSS not found; install with \"npm install postcss-cli\". See https://gohugo.io/hugo-pipes/postcss/"
- }
- return fmt.Errorf("%s: failed to transform %q (%s): %s", strings.ToUpper(tr.Key().Name), tctx.InPath, tctx.InMediaType.Type(), errMsg)
- }
- transformedContentr = f
- updates.sourceFs = cache.fileCache.Fs
- defer f.Close()
+ // Most likely because PostCSS is not installed.
+ errMsg += ". Check your PostCSS installation; install with \"npm install postcss-cli\". See https://gohugo.io/hugo-pipes/postcss/"
+ } else if tr.Key().Name == "tocss" {
+ errMsg += ". Check your Hugo installation; you need the extended version to build SCSS/SASS."
+ }
+ return fmt.Errorf("%s: failed to transform %q (%s): %s", strings.ToUpper(tr.Key().Name), tctx.InPath, tctx.InMediaType.Type(), errMsg)
- // The reader above is all we need.
- break
+ }
+
+ var tryFileCache bool
+
+ if mayBeCachedOnDisk && r.spec.BuildConfig.UseResourceCache(nil) {
+ tryFileCache = true
+ } else {
+ err = tr.Transform(tctx)
+ if mayBeCachedOnDisk {
+ tryFileCache = r.spec.BuildConfig.UseResourceCache(err)
}
+ if err != nil && !tryFileCache {
+ return notAvailableErr(err)
+ }
+ }
- // Abort.
- return err
+ if tryFileCache {
+ f := r.target.tryTransformedFileCache(key, updates)
+ if f == nil {
+ return notAvailableErr(errors.Errorf("resource %q not found in file cache", key))
+ }
+ transformedContentr = f
+ updates.sourceFs = cache.fileCache.Fs
+ defer f.Close()
+
+ // The reader above is all we need.
+ break
}
if tctx.OutPath != "" {