tpl: Add a partial lookup cache - hugo - [fork] hugo port for 9front
HTML git clone git@git.drkhsh.at/hugo.git
DIR Log
DIR Files
DIR Refs
DIR Submodules
DIR README
DIR LICENSE
---
DIR commit 208a0de6c31354df6f9463d49e90db9dec935169
DIR parent 18d2d2f98520b036a5138bc712a7e6843fa3380f
HTML Author: Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>
Date: Thu, 10 Apr 2025 09:22:29 +0200
tpl: Add a partial lookup cache
````
│ stash.bench │ perf-v146.bench │
│ sec/op │ sec/op vs base │
LookupPartial-10 248.00n ± 0% 14.75n ± 2% -94.05% (p=0.002 n=6)
│ stash.bench │ perf-v146.bench │
│ B/op │ B/op vs base │
LookupPartial-10 48.00 ± 0% 0.00 ± 0% -100.00% (p=0.002 n=6)
│ stash.bench │ perf-v146.bench │
│ allocs/op │ allocs/op vs base │
LookupPartial-10 3.000 ± 0% 0.000 ± 0% -100.00% (p=0.002 n=6)
```
THe speedup above assumes reuse of the same partials over and over again, which I think is not uncommon.
This commits also adds some more lookup benchmarks. The current output of these on my MacBook looks decent:
```
BenchmarkLookupPagesLayout/Single_root-10 3031562 395.5 ns/op 0 B/op 0 allocs/op
BenchmarkLookupPagesLayout/Single_sub_folder-10 2515915 480.9 ns/op 0 B/op 0 allocs/op
BenchmarkLookupPartial-10 84808112 14.13 ns/op 0 B/op 0 allocs/op
BenchmarkLookupShortcode/toplevelpage-10 8111779 148.2 ns/op 0 B/op 0 allocs/op
BenchmarkLookupShortcode/nestedpage-10 8088183 148.6 ns/op 0 B/op 0 allocs/op
```
Note that in the above the partial lookups are cahced, the others not (they are harder to cache because of the page path).
Closes #13571
Diffstat:
M common/maps/cache.go | 2 +-
M hugolib/alias.go | 2 +-
M hugolib/page.go | 5 +++--
M hugolib/page__common.go | 2 +-
M hugolib/page__new.go | 10 +++++-----
M hugolib/page__per_output.go | 2 +-
M hugolib/shortcode.go | 2 +-
M resources/page/page.go | 4 ++--
M resources/page/pages_related.go | 2 +-
M tpl/tplimpl/templatestore.go | 123 ++++++++++++++++---------------
M tpl/tplimpl/templatestore_integrat… | 81 ++++++++++++++++++++++++++++++-
11 files changed, 160 insertions(+), 75 deletions(-)
---
DIR diff --git a/common/maps/cache.go b/common/maps/cache.go
@@ -160,7 +160,7 @@ func (c *Cache[K, T]) Len() int {
func (c *Cache[K, T]) Reset() {
c.Lock()
- c.m = make(map[K]T)
+ clear(c.m)
c.hasBeenInitialized = false
c.Unlock()
}
DIR diff --git a/hugolib/alias.go b/hugolib/alias.go
@@ -51,7 +51,7 @@ func (a aliasHandler) renderAlias(permalink string, p page.Page) (io.Reader, err
var templateDesc tplimpl.TemplateDescriptor
var base string = ""
if ps, ok := p.(*pageState); ok {
- base, templateDesc = ps.getTemplateBasePathAndDescriptor()
+ base, templateDesc = ps.GetInternalTemplateBasePathAndDescriptor()
}
templateDesc.Layout = ""
templateDesc.Kind = ""
DIR diff --git a/hugolib/page.go b/hugolib/page.go
@@ -476,7 +476,8 @@ func (ps *pageState) initCommonProviders(pp pagePaths) error {
return nil
}
-func (po *pageOutput) getTemplateBasePathAndDescriptor() (string, tplimpl.TemplateDescriptor) {
+// Exported so it can be used in integration tests.
+func (po *pageOutput) GetInternalTemplateBasePathAndDescriptor() (string, tplimpl.TemplateDescriptor) {
p := po.p
f := po.f
base := p.PathInfo().BaseReTyped(p.m.pageConfig.Type)
@@ -491,7 +492,7 @@ func (po *pageOutput) getTemplateBasePathAndDescriptor() (string, tplimpl.Templa
}
func (p *pageState) resolveTemplate(layouts ...string) (*tplimpl.TemplInfo, bool, error) {
- dir, d := p.getTemplateBasePathAndDescriptor()
+ dir, d := p.GetInternalTemplateBasePathAndDescriptor()
if len(layouts) > 0 {
d.Layout = layouts[0]
DIR diff --git a/hugolib/page__common.go b/hugolib/page__common.go
@@ -97,7 +97,7 @@ type pageCommon struct {
pageMenus *pageMenus
// Internal use
- page.InternalDependencies
+ page.RelatedDocsHandlerProvider
contentConverterInit sync.Once
contentConverter converter.Converter
DIR diff --git a/hugolib/page__new.go b/hugolib/page__new.go
@@ -209,11 +209,11 @@ func (h *HugoSites) doNewPage(m *pageMeta) (*pageState, *paths.Path, error) {
ShortcodeInfoProvider: page.NopPage,
LanguageProvider: m.s,
- InternalDependencies: m.s,
- init: lazy.New(),
- m: m,
- s: m.s,
- sWrapped: page.WrapSite(m.s),
+ RelatedDocsHandlerProvider: m.s,
+ init: lazy.New(),
+ m: m,
+ s: m.s,
+ sWrapped: page.WrapSite(m.s),
},
}
DIR diff --git a/hugolib/page__per_output.go b/hugolib/page__per_output.go
@@ -275,7 +275,7 @@ func (pco *pageContentOutput) initRenderHooks() error {
// Inherit the descriptor from the page/current output format.
// This allows for fine-grained control of the template used for
// rendering of e.g. links.
- base, layoutDescriptor := pco.po.p.getTemplateBasePathAndDescriptor()
+ base, layoutDescriptor := pco.po.p.GetInternalTemplateBasePathAndDescriptor()
switch tp {
case hooks.LinkRendererType:
DIR diff --git a/hugolib/shortcode.go b/hugolib/shortcode.go
@@ -397,7 +397,7 @@ func doRenderShortcode(
ofCount[match.D.OutputFormat]++
return true
}
- base, layoutDescriptor := po.getTemplateBasePathAndDescriptor()
+ base, layoutDescriptor := po.GetInternalTemplateBasePathAndDescriptor()
q := tplimpl.TemplateQuery{
Path: base,
Name: sc.name,
DIR diff --git a/resources/page/page.go b/resources/page/page.go
@@ -148,8 +148,8 @@ type InSectionPositioner interface {
PrevInSection() Page
}
-// InternalDependencies is considered an internal interface.
-type InternalDependencies interface {
+// RelatedDocsHandlerProvider is considered an internal interface.
+type RelatedDocsHandlerProvider interface {
// GetInternalRelatedDocsHandler is for internal use only.
GetInternalRelatedDocsHandler() *RelatedDocsHandler
}
DIR diff --git a/resources/page/pages_related.go b/resources/page/pages_related.go
@@ -124,7 +124,7 @@ func (p Pages) withInvertedIndex(ctx context.Context, search func(idx *related.I
return nil, nil
}
- d, ok := p[0].(InternalDependencies)
+ d, ok := p[0].(RelatedDocsHandlerProvider)
if !ok {
return nil, fmt.Errorf("invalid type %T in related search", p[0])
}
DIR diff --git a/tpl/tplimpl/templatestore.go b/tpl/tplimpl/templatestore.go
@@ -97,16 +97,16 @@ func NewStore(opts StoreOptions, siteOpts SiteOptions) (*TemplateStore, error) {
panic("HTML output format not found")
}
s := &TemplateStore{
- opts: opts,
- siteOpts: siteOpts,
- optsOrig: opts,
- siteOptsOrig: siteOpts,
- htmlFormat: html,
- storeSite: configureSiteStorage(siteOpts, opts.Watching),
- treeMain: doctree.NewSimpleTree[map[nodeKey]*TemplInfo](),
- treeShortcodes: doctree.NewSimpleTree[map[string]map[TemplateDescriptor]*TemplInfo](),
- templatesByPath: maps.NewCache[string, *TemplInfo](),
- templateDescriptorByPath: maps.NewCache[string, PathTemplateDescriptor](),
+ opts: opts,
+ siteOpts: siteOpts,
+ optsOrig: opts,
+ siteOptsOrig: siteOpts,
+ htmlFormat: html,
+ storeSite: configureSiteStorage(siteOpts, opts.Watching),
+ treeMain: doctree.NewSimpleTree[map[nodeKey]*TemplInfo](),
+ treeShortcodes: doctree.NewSimpleTree[map[string]map[TemplateDescriptor]*TemplInfo](),
+ templatesByPath: maps.NewCache[string, *TemplInfo](),
+ cacheLookupPartials: maps.NewCache[string, *TemplInfo](),
// Note that the funcs passed below is just for name validation.
tns: newTemplateNamespace(siteOpts.TemplateFuncs),
@@ -400,10 +400,9 @@ type TemplateStore struct {
siteOpts SiteOptions
htmlFormat output.Format
- treeMain *doctree.SimpleTree[map[nodeKey]*TemplInfo]
- treeShortcodes *doctree.SimpleTree[map[string]map[TemplateDescriptor]*TemplInfo]
- templatesByPath *maps.Cache[string, *TemplInfo]
- templateDescriptorByPath *maps.Cache[string, PathTemplateDescriptor]
+ treeMain *doctree.SimpleTree[map[nodeKey]*TemplInfo]
+ treeShortcodes *doctree.SimpleTree[map[string]map[TemplateDescriptor]*TemplInfo]
+ templatesByPath *maps.Cache[string, *TemplInfo]
dh descriptorHandler
@@ -417,6 +416,9 @@ type TemplateStore struct {
// For testing benchmarking.
optsOrig StoreOptions
siteOptsOrig SiteOptions
+
+ // caches. These need to be refreshed when the templates are refreshed.
+ cacheLookupPartials *maps.Cache[string, *TemplInfo]
}
// NewFromOpts creates a new store with the same configuration as the original.
@@ -540,15 +542,19 @@ func (s *TemplateStore) LookupPagesLayout(q TemplateQuery) *TemplInfo {
}
func (s *TemplateStore) LookupPartial(pth string) *TemplInfo {
- d := s.templateDescriptorFromPath(pth)
- desc := d.Desc
- if desc.Layout != "" {
- panic("shortcode template descriptor must not have a layout")
- }
- best := s.getBest()
- defer s.putBest(best)
- s.findBestMatchGet(s.key(path.Join(containerPartials, d.Path)), CategoryPartial, nil, desc, best)
- return best.templ
+ ti, _ := s.cacheLookupPartials.GetOrCreate(pth, func() (*TemplInfo, error) {
+ d := s.templateDescriptorFromPath(pth)
+ desc := d.Desc
+ if desc.Layout != "" {
+ panic("shortcode template descriptor must not have a layout")
+ }
+ best := s.getBest()
+ defer s.putBest(best)
+ s.findBestMatchGet(s.key(path.Join(containerPartials, d.Path)), CategoryPartial, nil, desc, best)
+ return best.templ, nil
+ })
+
+ return ti
}
func (s *TemplateStore) LookupShortcode(q TemplateQuery) *TemplInfo {
@@ -619,8 +625,14 @@ func (s *TemplateStore) PrintDebug(prefix string, category Category, w io.Writer
})
}
+func (s *TemplateStore) clearCaches() {
+ s.cacheLookupPartials.Reset()
+}
+
// RefreshFiles refreshes this store for the files matching the given predicate.
func (s *TemplateStore) RefreshFiles(include func(fi hugofs.FileMetaInfo) bool) error {
+ s.clearCaches()
+
if err := s.tns.createPrototypesParse(); err != nil {
return err
}
@@ -1370,43 +1382,38 @@ type PathTemplateDescriptor struct {
// templateDescriptorFromPath returns a template descriptor from the given path.
// This is currently used in partial lookups only.
func (s *TemplateStore) templateDescriptorFromPath(pth string) PathTemplateDescriptor {
- // Check cache first.
- d, _ := s.templateDescriptorByPath.GetOrCreate(pth, func() (PathTemplateDescriptor, error) {
- var (
- mt media.Type
- of output.Format
- )
-
- // Common cases.
- dotCount := strings.Count(pth, ".")
- if dotCount <= 1 {
- if dotCount == 0 {
- // Asume HTML.
- of, mt = s.resolveOutputFormatAndOrMediaType("html", "")
- } else {
- pth = strings.TrimPrefix(pth, "/")
- ext := path.Ext(pth)
- pth = strings.TrimSuffix(pth, ext)
- ext = ext[1:]
- of, mt = s.resolveOutputFormatAndOrMediaType("", ext)
- }
+ var (
+ mt media.Type
+ of output.Format
+ )
+
+ // Common cases.
+ dotCount := strings.Count(pth, ".")
+ if dotCount <= 1 {
+ if dotCount == 0 {
+ // Asume HTML.
+ of, mt = s.resolveOutputFormatAndOrMediaType("html", "")
} else {
- path := s.opts.PathParser.Parse(files.ComponentFolderLayouts, pth)
- pth = path.PathNoIdentifier()
- of, mt = s.resolveOutputFormatAndOrMediaType(path.OutputFormat(), path.Ext())
- }
-
- return PathTemplateDescriptor{
- Path: pth,
- Desc: TemplateDescriptor{
- OutputFormat: of.Name,
- MediaType: mt.Type,
- IsPlainText: of.IsPlainText,
- },
- }, nil
- })
+ pth = strings.TrimPrefix(pth, "/")
+ ext := path.Ext(pth)
+ pth = strings.TrimSuffix(pth, ext)
+ ext = ext[1:]
+ of, mt = s.resolveOutputFormatAndOrMediaType("", ext)
+ }
+ } else {
+ path := s.opts.PathParser.Parse(files.ComponentFolderLayouts, pth)
+ pth = path.PathNoIdentifier()
+ of, mt = s.resolveOutputFormatAndOrMediaType(path.OutputFormat(), path.Ext())
+ }
- return d
+ return PathTemplateDescriptor{
+ Path: pth,
+ Desc: TemplateDescriptor{
+ OutputFormat: of.Name,
+ MediaType: mt.Type,
+ IsPlainText: of.IsPlainText,
+ },
+ }
}
// resolveOutputFormatAndOrMediaType resolves the output format and/or media type
DIR diff --git a/tpl/tplimpl/templatestore_integration_test.go b/tpl/tplimpl/templatestore_integration_test.go
@@ -8,6 +8,7 @@ import (
qt "github.com/frankban/quicktest"
"github.com/gohugoio/hugo/hugolib"
"github.com/gohugoio/hugo/resources/kinds"
+ "github.com/gohugoio/hugo/resources/page"
"github.com/gohugoio/hugo/tpl/tplimpl"
)
@@ -849,7 +850,7 @@ func BenchmarkExecuteWithContext(b *testing.B) {
disableKinds = ["taxonomy", "term", "home"]
-- layouts/all.html --
{{ .Title }}|
- {{ partial "p1.html" . }}
+{{ partial "p1.html" . }}
-- layouts/_partials/p1.html --
p1.
{{ partial "p2.html" . }}
@@ -878,6 +879,82 @@ p3
b.ResetTimer()
for i := 0; i < b.N; i++ {
err := store.ExecuteWithContext(context.Background(), ti, io.Discard, p)
- bb.Assert(err, qt.IsNil)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+}
+
+func BenchmarkLookupPartial(b *testing.B) {
+ files := `
+-- hugo.toml --
+disableKinds = ["taxonomy", "term", "home"]
+-- layouts/all.html --
+{{ .Title }}|
+-- layouts/_partials/p1.html --
+-- layouts/_partials/p2.html --
+-- layouts/_partials/p2.json --
+-- layouts/_partials/p3.html --
+`
+ bb := hugolib.Test(b, files)
+
+ store := bb.H.TemplateStore
+
+ for i := 0; i < b.N; i++ {
+ fi := store.LookupPartial("p3.html")
+ if fi == nil {
+ b.Fatal("not found")
+ }
}
}
+
+// Implemented by pageOutput.
+type getDescriptorProvider interface {
+ GetInternalTemplateBasePathAndDescriptor() (string, tplimpl.TemplateDescriptor)
+}
+
+func BenchmarkLookupShortcode(b *testing.B) {
+ files := `
+-- hugo.toml --
+disableKinds = ["taxonomy", "term", "home"]
+-- content/toplevelpage.md --
+-- content/a/b/c/nested.md --
+-- layouts/all.html --
+{{ .Title }}|
+-- layouts/_shortcodes/s.html --
+s1.
+-- layouts/_shortcodes/a/b/s.html --
+s2.
+
+`
+ bb := hugolib.Test(b, files)
+ store := bb.H.TemplateStore
+
+ runOne := func(p page.Page) {
+ pth, desc := p.(getDescriptorProvider).GetInternalTemplateBasePathAndDescriptor()
+ q := tplimpl.TemplateQuery{
+ Path: pth,
+ Name: "s",
+ Category: tplimpl.CategoryShortcode,
+ Desc: desc,
+ }
+ v := store.LookupShortcode(q)
+ if v == nil {
+ b.Fatal("not found")
+ }
+ }
+
+ b.Run("toplevelpage", func(b *testing.B) {
+ toplevelpage, _ := bb.H.Sites[0].GetPage("/toplevelpage")
+ for i := 0; i < b.N; i++ {
+ runOne(toplevelpage)
+ }
+ })
+
+ b.Run("nestedpage", func(b *testing.B) {
+ toplevelpage, _ := bb.H.Sites[0].GetPage("/a/b/c/nested")
+ for i := 0; i < b.N; i++ {
+ runOne(toplevelpage)
+ }
+ })
+}