Fix 0.62.1 server rebuild slowdown regression - hugo - [fork] hugo port for 9front
HTML git clone git@git.drkhsh.at/hugo.git
DIR Log
DIR Files
DIR Refs
DIR Submodules
DIR README
DIR LICENSE
---
DIR commit 17af79a03e249a731cf5634ffea23ca00774333d
DIR parent 2fefc01606fddb119f368c89fb2dedd452ad6547
HTML Author: Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>
Date: Wed, 22 Jan 2020 11:57:23 +0100
Fix 0.62.1 server rebuild slowdown regression
Fixes #6784
Diffstat:
M hugofs/fileinfo.go | 28 +++++++++-------------------
M hugofs/rootmapping_fs.go | 6 +-----
M hugofs/rootmapping_fs_test.go | 2 +-
M hugolib/content_render_hooks_test.… | 24 ++++++++++++++++++++++--
M hugolib/filesystems/basefs.go | 13 ++++---------
M hugolib/hugo_modules_test.go | 120 +++++++++++++++++++++++++++++++
M hugolib/hugo_sites.go | 16 ++++++++++++++++
M hugolib/hugo_sites_build.go | 2 ++
M hugolib/page__per_output.go | 2 ++
M hugolib/testhelpers_test.go | 9 +++++++++
10 files changed, 186 insertions(+), 36 deletions(-)
---
DIR diff --git a/hugofs/fileinfo.go b/hugofs/fileinfo.go
@@ -34,9 +34,9 @@ import (
)
const (
- metaKeyFilename = "filename"
- metaKeyPathFile = "pathFile" // Path of filename relative to a root.
- metaKeyIsFileMount = "isFileMount" // Whether the source mount was a file.
+ metaKeyFilename = "filename"
+
+ metaKeyBaseDir = "baseDir" // Abs base directory of source file.
metaKeyMountRoot = "mountRoot"
metaKeyOriginalFilename = "originalFilename"
metaKeyName = "name"
@@ -116,29 +116,19 @@ func (f FileMeta) Path() string {
return f.stringV(metaKeyPath)
}
-// PathFile returns the relative file path for the file source. This
-// will in most cases be the same as Path.
+// PathFile returns the relative file path for the file source.
func (f FileMeta) PathFile() string {
- pf := f.stringV(metaKeyPathFile)
- if f.isFileMount() {
- return pf
- }
- mountRoot := f.mountRoot()
- if mountRoot == pf {
- return f.Path()
+ base := f.stringV(metaKeyBaseDir)
+ if base == "" {
+ return ""
}
-
- return pf + (strings.TrimPrefix(f.Path(), mountRoot))
+ return strings.TrimPrefix(strings.TrimPrefix(f.Filename(), base), filepathSeparator)
}
-func (f FileMeta) mountRoot() string {
+func (f FileMeta) MountRoot() string {
return f.stringV(metaKeyMountRoot)
}
-func (f FileMeta) isFileMount() bool {
- return f.GetBool(metaKeyIsFileMount)
-}
-
func (f FileMeta) Weight() int {
return f.GetInt(metaKeyWeight)
}
DIR diff --git a/hugofs/rootmapping_fs.go b/hugofs/rootmapping_fs.go
@@ -57,12 +57,8 @@ func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) {
// Extract "blog" from "content/blog"
rm.path = strings.TrimPrefix(strings.TrimPrefix(rm.From, fromBase), filepathSeparator)
if rm.Meta != nil {
- rm.Meta[metaKeyIsFileMount] = !fi.IsDir()
+ rm.Meta[metaKeyBaseDir] = rm.ToBasedir
rm.Meta[metaKeyMountRoot] = rm.path
- if rm.ToBasedir != "" {
- pathFile := strings.TrimPrefix(strings.TrimPrefix(rm.To, rm.ToBasedir), filepathSeparator)
- rm.Meta[metaKeyPathFile] = pathFile
- }
}
meta := copyFileMeta(rm.Meta)
DIR diff --git a/hugofs/rootmapping_fs_test.go b/hugofs/rootmapping_fs_test.go
@@ -271,7 +271,7 @@ func TestRootMappingFsMount(t *testing.T) {
c.Assert(singles, qt.HasLen, 2)
for i, lang := range []string{"no", "sv"} {
fi := singles[i].(FileMetaInfo)
- c.Assert(fi.Meta().PathFile(), qt.Equals, lang+".txt")
+ c.Assert(fi.Meta().PathFile(), qt.Equals, filepath.FromSlash("themes/a/singlefiles/"+lang+".txt"))
c.Assert(fi.Meta().Lang(), qt.Equals, lang)
c.Assert(fi.Name(), qt.Equals, "p1.md")
}
DIR diff --git a/hugolib/content_render_hooks_test.go b/hugolib/content_render_hooks_test.go
@@ -14,7 +14,10 @@
package hugolib
import (
+ "fmt"
"testing"
+
+ qt "github.com/frankban/quicktest"
)
func TestRenderHooks(t *testing.T) {
@@ -118,7 +121,20 @@ title: With RenderString
{{< myshortcode5 >}}Inner Link: [Inner Link](https://www.gohugo.io "Hugo's Homepage"){{< /myshortcode5 >}}
`)
- b.Build(BuildCfg{})
+
+ for i := 1; i <= 30; i++ {
+ // Add some content with no shortcodes or links, i.e no templates needed.
+ b.WithContent(fmt.Sprintf("blog/notempl%d.md", i), `---
+title: No Template
+---
+
+## Content
+`)
+ }
+ counters := &testCounters{}
+ b.Build(BuildCfg{testCounters: counters})
+ b.Assert(int(counters.contentRenderCounter), qt.Equals, 50)
+
b.AssertFileContent("public/blog/p1/index.html", `
<p>Cool Page|https://www.google.com|Title: Google's Homepage|Text: First Link|END</p>
Text: Second
@@ -149,7 +165,11 @@ SHORT3|
"layouts/shortcodes/myshortcode3.html", `SHORT3_EDITED|`,
)
- b.Build(BuildCfg{})
+ counters = &testCounters{}
+ b.Build(BuildCfg{testCounters: counters})
+ // Make sure that only content using the changed templates are re-rendered.
+ b.Assert(int(counters.contentRenderCounter), qt.Equals, 7)
+
b.AssertFileContent("public/customview/p1/index.html", `.Render: myrender: Custom View|P4: PARTIAL4_EDITED`)
b.AssertFileContent("public/blog/p1/index.html", `<p>EDITED: https://www.google.com|</p>`, "SHORT3_EDITED|")
b.AssertFileContent("public/blog/p2/index.html", `PARTIAL1_EDITED`)
DIR diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go
@@ -295,21 +295,16 @@ func (d *SourceFilesystem) Contains(filename string) bool {
return false
}
-// Path returns the relative path to the given filename if it is a member of
+// Path returns the mount relative path to the given filename if it is a member of
// of the current filesystem, an empty string if not.
func (d *SourceFilesystem) Path(filename string) string {
for _, dir := range d.Dirs {
meta := dir.Meta()
- if !dir.IsDir() {
- if filename == meta.Filename() {
- return meta.PathFile()
- }
- continue
- }
-
if strings.HasPrefix(filename, meta.Filename()) {
p := strings.TrimPrefix(strings.TrimPrefix(filename, meta.Filename()), filePathSeparator)
- p = path.Join(meta.PathFile(), p)
+ if mountRoot := meta.MountRoot(); mountRoot != "" {
+ return filepath.Join(mountRoot, p)
+ }
return p
}
}
DIR diff --git a/hugolib/hugo_modules_test.go b/hugolib/hugo_modules_test.go
@@ -668,6 +668,126 @@ Readme Edit
}
+func TestMountsPaths(t *testing.T) {
+ c := qt.New(t)
+
+ type test struct {
+ b *sitesBuilder
+ clean func()
+ workingDir string
+ }
+
+ prepare := func(c *qt.C, mounts string) test {
+ workingDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-mounts-paths")
+ c.Assert(err, qt.IsNil)
+
+ configTemplate := `
+baseURL = "https://example.com"
+title = "My Modular Site"
+workingDir = %q
+
+%s
+
+`
+ config := fmt.Sprintf(configTemplate, workingDir, mounts)
+ config = strings.Replace(config, "WORKING_DIR", workingDir, -1)
+
+ b := newTestSitesBuilder(c).Running()
+
+ b.Fs = hugofs.NewDefault(viper.New())
+
+ os.MkdirAll(filepath.Join(workingDir, "content", "blog"), 0777)
+
+ b.WithWorkingDir(workingDir).WithConfigFile("toml", config)
+
+ return test{
+ b: b,
+ clean: clean,
+ workingDir: workingDir,
+ }
+
+ }
+
+ c.Run("Default", func(c *qt.C) {
+ mounts := ``
+
+ test := prepare(c, mounts)
+ b := test.b
+ defer test.clean()
+
+ b.WithContent("blog/p1.md", `---
+title: P1
+---`)
+
+ b.Build(BuildCfg{})
+
+ p := b.GetPage("blog/p1.md")
+ f := p.File().FileInfo().Meta()
+ b.Assert(filepath.ToSlash(f.Path()), qt.Equals, "blog/p1.md")
+ b.Assert(filepath.ToSlash(f.PathFile()), qt.Equals, "content/blog/p1.md")
+
+ b.Assert(b.H.BaseFs.Layouts.Path(filepath.Join(test.workingDir, "layouts", "_default", "single.html")), qt.Equals, filepath.FromSlash("_default/single.html"))
+
+ })
+
+ c.Run("Mounts", func(c *qt.C) {
+ absDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-mounts-paths-abs")
+ c.Assert(err, qt.IsNil)
+ defer clean()
+
+ mounts := `[module]
+ [[module.mounts]]
+ source = "README.md"
+ target = "content/_index.md"
+ [[module.mounts]]
+ source = "mycontent"
+ target = "content/blog"
+ [[module.mounts]]
+ source = "subdir/mypartials"
+ target = "layouts/partials"
+ [[module.mounts]]
+ source = %q
+ target = "layouts/shortcodes"
+`
+ mounts = fmt.Sprintf(mounts, filepath.Join(absDir, "/abs/myshortcodes"))
+
+ test := prepare(c, mounts)
+ b := test.b
+ defer test.clean()
+
+ subContentDir := filepath.Join(test.workingDir, "mycontent", "sub")
+ os.MkdirAll(subContentDir, 0777)
+ myPartialsDir := filepath.Join(test.workingDir, "subdir", "mypartials")
+ os.MkdirAll(myPartialsDir, 0777)
+
+ absShortcodesDir := filepath.Join(absDir, "abs", "myshortcodes")
+ os.MkdirAll(absShortcodesDir, 0777)
+
+ b.WithSourceFile("README.md", "---\ntitle: Readme\n---")
+ b.WithSourceFile("mycontent/sub/p1.md", "---\ntitle: P1\n---")
+
+ b.WithSourceFile(filepath.Join(absShortcodesDir, "myshort.html"), "MYSHORT")
+ b.WithSourceFile(filepath.Join(myPartialsDir, "mypartial.html"), "MYPARTIAL")
+
+ b.Build(BuildCfg{})
+
+ p1_1 := b.GetPage("/blog/sub/p1.md")
+ p1_2 := b.GetPage("/mycontent/sub/p1.md")
+ b.Assert(p1_1, qt.Not(qt.IsNil))
+ b.Assert(p1_2, qt.Equals, p1_1)
+
+ f := p1_1.File().FileInfo().Meta()
+ b.Assert(filepath.ToSlash(f.Path()), qt.Equals, "blog/sub/p1.md")
+ b.Assert(filepath.ToSlash(f.PathFile()), qt.Equals, "mycontent/sub/p1.md")
+ b.Assert(b.H.BaseFs.Layouts.Path(filepath.Join(myPartialsDir, "mypartial.html")), qt.Equals, filepath.FromSlash("partials/mypartial.html"))
+ b.Assert(b.H.BaseFs.Layouts.Path(filepath.Join(absShortcodesDir, "myshort.html")), qt.Equals, filepath.FromSlash("shortcodes/myshort.html"))
+ b.Assert(b.H.BaseFs.Content.Path(filepath.Join(subContentDir, "p1.md")), qt.Equals, filepath.FromSlash("blog/sub/p1.md"))
+ b.Assert(b.H.BaseFs.Content.Path(filepath.Join(test.workingDir, "README.md")), qt.Equals, filepath.FromSlash("_index.md"))
+
+ })
+
+}
+
// https://github.com/gohugoio/hugo/issues/6299
func TestSiteWithGoModButNoModules(t *testing.T) {
t.Parallel()
DIR diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go
@@ -19,6 +19,7 @@ import (
"sort"
"strings"
"sync"
+ "sync/atomic"
"github.com/gohugoio/hugo/identity"
@@ -82,6 +83,19 @@ type HugoSites struct {
init *hugoSitesInit
*fatalErrorHandler
+ *testCounters
+}
+
+// Only used in tests.
+type testCounters struct {
+ contentRenderCounter uint64
+}
+
+func (h *testCounters) IncrContentRender() {
+ if h == nil {
+ return
+ }
+ atomic.AddUint64(&h.contentRenderCounter, 1)
}
type fatalErrorHandler struct {
@@ -579,6 +593,8 @@ type BuildCfg struct {
// Recently visited URLs. This is used for partial re-rendering.
RecentlyVisited map[string]bool
+
+ testCounters *testCounters
}
// shouldRender is used in the Fast Render Mode to determine if we need to re-render
DIR diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go
@@ -66,6 +66,8 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error {
h.Metrics.Reset()
}
+ h.testCounters = config.testCounters
+
// Need a pointer as this may be modified.
conf := &config
DIR diff --git a/hugolib/page__per_output.go b/hugolib/page__per_output.go
@@ -80,6 +80,8 @@ func newPageContentOutput(p *pageState, po *pageOutput) (*pageContentOutput, err
}
initContent := func() (err error) {
+ p.s.h.IncrContentRender()
+
if p.cmap == nil {
// Nothing to do.
return nil
DIR diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go
@@ -224,6 +224,9 @@ func (s *sitesBuilder) WithSourceFile(filenameContent ...string) *sitesBuilder {
func (s *sitesBuilder) absFilename(filename string) string {
filename = filepath.FromSlash(filename)
+ if filepath.IsAbs(filename) {
+ return filename
+ }
if s.workingDir != "" && !strings.HasPrefix(filename, s.workingDir) {
filename = filepath.Join(s.workingDir, filename)
}
@@ -736,6 +739,12 @@ func (s *sitesBuilder) CheckExists(filename string) bool {
return destinationExists(s.Fs, filepath.Clean(filename))
}
+func (s *sitesBuilder) GetPage(ref string) page.Page {
+ p, err := s.H.Sites[0].getPageNew(nil, ref)
+ s.Assert(err, qt.IsNil)
+ return p
+}
+
func newTestHelper(cfg config.Provider, fs *hugofs.Fs, t testing.TB) testHelper {
return testHelper{
Cfg: cfg,