hugo_sites.go - hugo - [fork] hugo port for 9front
HTML git clone https://git.drkhsh.at/hugo.git
DIR Log
DIR Files
DIR Refs
DIR Submodules
DIR README
DIR LICENSE
---
hugo_sites.go (15249B)
---
1 // Copyright 2024 The Hugo Authors. All rights reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 // http://www.apache.org/licenses/LICENSE-2.0
7 //
8 // Unless required by applicable law or agreed to in writing, software
9 // distributed under the License is distributed on an "AS IS" BASIS,
10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 // See the License for the specific language governing permissions and
12 // limitations under the License.
13
14 package hugolib
15
16 import (
17 "context"
18 "fmt"
19 "io"
20 "strings"
21 "sync"
22 "sync/atomic"
23
24 "github.com/bep/logg"
25 "github.com/gohugoio/hugo/cache/dynacache"
26 "github.com/gohugoio/hugo/config/allconfig"
27 "github.com/gohugoio/hugo/hugofs/glob"
28 "github.com/gohugoio/hugo/hugolib/doctree"
29 "github.com/gohugoio/hugo/resources"
30
31 "github.com/fsnotify/fsnotify"
32
33 "github.com/gohugoio/hugo/output"
34 "github.com/gohugoio/hugo/parser/metadecoders"
35
36 "github.com/gohugoio/hugo/common/hugo"
37 "github.com/gohugoio/hugo/common/maps"
38 "github.com/gohugoio/hugo/common/para"
39 "github.com/gohugoio/hugo/common/types"
40 "github.com/gohugoio/hugo/hugofs"
41
42 "github.com/gohugoio/hugo/source"
43
44 "github.com/gohugoio/hugo/common/herrors"
45 "github.com/gohugoio/hugo/deps"
46 "github.com/gohugoio/hugo/helpers"
47 "github.com/gohugoio/hugo/lazy"
48
49 "github.com/gohugoio/hugo/resources/page"
50 )
51
52 // HugoSites represents the sites to build. Each site represents a language.
53 type HugoSites struct {
54 Sites []*Site
55
56 Configs *allconfig.Configs
57
58 hugoInfo hugo.HugoInfo
59
60 // Render output formats for all sites.
61 renderFormats output.Formats
62
63 // The currently rendered Site.
64 currentSite *Site
65
66 *deps.Deps
67
68 gitInfo *gitInfo
69 codeownerInfo *codeownerInfo
70
71 // As loaded from the /data dirs
72 data map[string]any
73
74 // Cache for page listings.
75 cachePages *dynacache.Partition[string, page.Pages]
76 // Cache for content sources.
77 cacheContentSource *dynacache.Partition[string, *resources.StaleValue[[]byte]]
78
79 // Before Hugo 0.122.0 we managed all translations in a map using a translationKey
80 // that could be overridden in front matter.
81 // Now the different page dimensions (e.g. language) are built-in to the page trees above.
82 // But we sill need to support the overridden translationKey, but that should
83 // be relatively rare and low volume.
84 translationKeyPages *maps.SliceCache[page.Page]
85
86 pageTrees *pageTrees
87
88 printUnusedTemplatesInit sync.Once
89 printPathWarningsInit sync.Once
90
91 // File change events with filename stored in this map will be skipped.
92 skipRebuildForFilenamesMu sync.Mutex
93 skipRebuildForFilenames map[string]bool
94
95 init *hugoSitesInit
96
97 workersSite *para.Workers
98 numWorkersSites int
99 numWorkers int
100
101 *fatalErrorHandler
102 *buildCounters
103 // Tracks invocations of the Build method.
104 buildCounter atomic.Uint64
105 }
106
107 // ShouldSkipFileChangeEvent allows skipping filesystem event early before
108 // the build is started.
109 func (h *HugoSites) ShouldSkipFileChangeEvent(ev fsnotify.Event) bool {
110 h.skipRebuildForFilenamesMu.Lock()
111 defer h.skipRebuildForFilenamesMu.Unlock()
112 return h.skipRebuildForFilenames[ev.Name]
113 }
114
115 func (h *HugoSites) Close() error {
116 return h.Deps.Close()
117 }
118
119 func (h *HugoSites) isRebuild() bool {
120 return h.buildCounter.Load() > 0
121 }
122
123 func (h *HugoSites) resolveSite(lang string) *Site {
124 if lang == "" {
125 lang = h.Conf.DefaultContentLanguage()
126 }
127
128 for _, s := range h.Sites {
129 if s.Lang() == lang {
130 return s
131 }
132 }
133
134 return nil
135 }
136
137 type buildCounters struct {
138 contentRenderCounter atomic.Uint64
139 pageRenderCounter atomic.Uint64
140 }
141
142 func (c *buildCounters) loggFields() logg.Fields {
143 return logg.Fields{
144 {Name: "pages", Value: c.pageRenderCounter.Load()},
145 {Name: "content", Value: c.contentRenderCounter.Load()},
146 }
147 }
148
149 type fatalErrorHandler struct {
150 mu sync.Mutex
151
152 h *HugoSites
153
154 err error
155
156 done bool
157 donec chan bool // will be closed when done
158 }
159
160 // FatalError error is used in some rare situations where it does not make sense to
161 // continue processing, to abort as soon as possible and log the error.
162 func (f *fatalErrorHandler) FatalError(err error) {
163 f.mu.Lock()
164 defer f.mu.Unlock()
165 if !f.done {
166 f.done = true
167 close(f.donec)
168 }
169 f.err = err
170 }
171
172 func (f *fatalErrorHandler) getErr() error {
173 f.mu.Lock()
174 defer f.mu.Unlock()
175 return f.err
176 }
177
178 func (f *fatalErrorHandler) Done() <-chan bool {
179 return f.donec
180 }
181
182 type hugoSitesInit struct {
183 // Loads the data from all of the /data folders.
184 data *lazy.Init
185
186 // Loads the Git info and CODEOWNERS for all the pages if enabled.
187 gitInfo *lazy.Init
188 }
189
190 func (h *HugoSites) Data() map[string]any {
191 if _, err := h.init.data.Do(context.Background()); err != nil {
192 h.SendError(fmt.Errorf("failed to load data: %w", err))
193 return nil
194 }
195 return h.data
196 }
197
198 // Pages returns all pages for all sites.
199 func (h *HugoSites) Pages() page.Pages {
200 key := "pages"
201 v, err := h.cachePages.GetOrCreate(key, func(string) (page.Pages, error) {
202 var pages page.Pages
203 for _, s := range h.Sites {
204 pages = append(pages, s.Pages()...)
205 }
206 page.SortByDefault(pages)
207 return pages, nil
208 })
209 if err != nil {
210 panic(err)
211 }
212 return v
213 }
214
215 // Pages returns all regularpages for all sites.
216 func (h *HugoSites) RegularPages() page.Pages {
217 key := "regular-pages"
218 v, err := h.cachePages.GetOrCreate(key, func(string) (page.Pages, error) {
219 var pages page.Pages
220 for _, s := range h.Sites {
221 pages = append(pages, s.RegularPages()...)
222 }
223 page.SortByDefault(pages)
224
225 return pages, nil
226 })
227 if err != nil {
228 panic(err)
229 }
230 return v
231 }
232
233 func (h *HugoSites) gitInfoForPage(p page.Page) (*source.GitInfo, error) {
234 if _, err := h.init.gitInfo.Do(context.Background()); err != nil {
235 return nil, err
236 }
237
238 if h.gitInfo == nil {
239 return nil, nil
240 }
241
242 return h.gitInfo.forPage(p), nil
243 }
244
245 func (h *HugoSites) codeownersForPage(p page.Page) ([]string, error) {
246 if _, err := h.init.gitInfo.Do(context.Background()); err != nil {
247 return nil, err
248 }
249
250 if h.codeownerInfo == nil {
251 return nil, nil
252 }
253
254 return h.codeownerInfo.forPage(p), nil
255 }
256
257 func (h *HugoSites) pickOneAndLogTheRest(errors []error) error {
258 if len(errors) == 0 {
259 return nil
260 }
261
262 var i int
263
264 for j, err := range errors {
265 // If this is in server mode, we want to return an error to the client
266 // with a file context, if possible.
267 if herrors.UnwrapFileError(err) != nil {
268 i = j
269 break
270 }
271 }
272
273 // Log the rest, but add a threshold to avoid flooding the log.
274 const errLogThreshold = 5
275
276 for j, err := range errors {
277 if j == i || err == nil {
278 continue
279 }
280
281 if j >= errLogThreshold {
282 break
283 }
284
285 h.Log.Errorln(err)
286 }
287
288 return errors[i]
289 }
290
291 func (h *HugoSites) isMultilingual() bool {
292 return len(h.Sites) > 1
293 }
294
295 // TODO(bep) consolidate
296 func (h *HugoSites) LanguageSet() map[string]int {
297 set := make(map[string]int)
298 for i, s := range h.Sites {
299 set[s.language.Lang] = i
300 }
301 return set
302 }
303
304 func (h *HugoSites) NumLogErrors() int {
305 if h == nil {
306 return 0
307 }
308 return h.Log.LoggCount(logg.LevelError)
309 }
310
311 func (h *HugoSites) PrintProcessingStats(w io.Writer) {
312 stats := make([]*helpers.ProcessingStats, len(h.Sites))
313 for i := range h.Sites {
314 stats[i] = h.Sites[i].PathSpec.ProcessingStats
315 }
316 helpers.ProcessingStatsTable(w, stats...)
317 }
318
319 // GetContentPage finds a Page with content given the absolute filename.
320 // Returns nil if none found.
321 func (h *HugoSites) GetContentPage(filename string) page.Page {
322 var p page.Page
323
324 h.withPage(func(s string, p2 *pageState) bool {
325 if p2.File() == nil {
326 return false
327 }
328
329 if p2.File().FileInfo().Meta().Filename == filename {
330 p = p2
331 return true
332 }
333
334 for _, r := range p2.Resources().ByType(pageResourceType) {
335 p3 := r.(page.Page)
336 if p3.File() != nil && p3.File().FileInfo().Meta().Filename == filename {
337 p = p3
338 return true
339 }
340 }
341
342 return false
343 })
344
345 return p
346 }
347
348 func (h *HugoSites) loadGitInfo() error {
349 if h.Configs.Base.EnableGitInfo {
350 gi, err := newGitInfo(h.Deps)
351 if err != nil {
352 h.Log.Errorln("Failed to read Git log:", err)
353 } else {
354 h.gitInfo = gi
355 }
356
357 co, err := newCodeOwners(h.Configs.LoadingInfo.BaseConfig.WorkingDir)
358 if err != nil {
359 h.Log.Errorln("Failed to read CODEOWNERS:", err)
360 } else {
361 h.codeownerInfo = co
362 }
363 }
364 return nil
365 }
366
367 // Reset resets the sites and template caches etc., making it ready for a full rebuild.
368 func (h *HugoSites) reset(config *BuildCfg) {
369 h.fatalErrorHandler = &fatalErrorHandler{
370 h: h,
371 donec: make(chan bool),
372 }
373 }
374
375 // resetLogs resets the log counters etc. Used to do a new build on the same sites.
376 func (h *HugoSites) resetLogs() {
377 h.Log.Reset()
378 for _, s := range h.Sites {
379 s.Deps.Log.Reset()
380 }
381 }
382
383 func (h *HugoSites) withSite(fn func(s *Site) error) error {
384 for _, s := range h.Sites {
385 if err := fn(s); err != nil {
386 return err
387 }
388 }
389 return nil
390 }
391
392 func (h *HugoSites) withPage(fn func(s string, p *pageState) bool) {
393 h.withSite(func(s *Site) error {
394 w := &doctree.NodeShiftTreeWalker[contentNodeI]{
395 Tree: s.pageMap.treePages,
396 LockType: doctree.LockTypeRead,
397 Handle: func(s string, n contentNodeI, match doctree.DimensionFlag) (bool, error) {
398 return fn(s, n.(*pageState)), nil
399 },
400 }
401 return w.Walk(context.Background())
402 })
403 }
404
405 // BuildCfg holds build options used to, as an example, skip the render step.
406 type BuildCfg struct {
407 // Skip rendering. Useful for testing.
408 SkipRender bool
409
410 // Use this to indicate what changed (for rebuilds).
411 WhatChanged *WhatChanged
412
413 // This is a partial re-render of some selected pages.
414 PartialReRender bool
415
416 // Set in server mode when the last build failed for some reason.
417 ErrRecovery bool
418
419 // Recently visited or touched URLs. This is used for partial re-rendering.
420 RecentlyTouched *types.EvictingQueue[string]
421
422 // Can be set to build only with a sub set of the content source.
423 ContentInclusionFilter *glob.FilenameFilter
424
425 // Set when the buildlock is already acquired (e.g. the archetype content builder).
426 NoBuildLock bool
427
428 testCounters *buildCounters
429 }
430
431 // shouldRender returns whether this output format should be rendered or not.
432 func (cfg *BuildCfg) shouldRender(infol logg.LevelLogger, p *pageState) bool {
433 if p.skipRender() {
434 return false
435 }
436
437 if !p.renderOnce {
438 return true
439 }
440
441 // The render state is incremented on render and reset when a related change is detected.
442 // Note that this is set per output format.
443 shouldRender := p.renderState == 0
444
445 if !shouldRender {
446 return false
447 }
448
449 fastRenderMode := p.s.Conf.FastRenderMode()
450
451 if !fastRenderMode || p.s.h.buildCounter.Load() == 0 {
452 return shouldRender
453 }
454
455 if !p.render {
456 // Not be to rendered for this output format.
457 return false
458 }
459
460 if relURL := p.getRelURL(); relURL != "" {
461 if cfg.RecentlyTouched.Contains(relURL) {
462 infol.Logf("render recently touched URL %q (%s)", relURL, p.outputFormat().Name)
463 return true
464 }
465 }
466
467 // In fast render mode, we want to avoid re-rendering the sitemaps etc. and
468 // other big listings whenever we e.g. change a content file,
469 // but we want partial renders of the recently touched pages to also include
470 // alternative formats of the same HTML page (e.g. RSS, JSON).
471 for _, po := range p.pageOutputs {
472 if po.render && po.f.IsHTML && cfg.RecentlyTouched.Contains(po.getRelURL()) {
473 infol.Logf("render recently touched URL %q, %s version of %s", po.getRelURL(), po.f.Name, p.outputFormat().Name)
474 return true
475 }
476 }
477
478 return false
479 }
480
481 func (s *Site) preparePagesForRender(isRenderingSite bool, idx int) error {
482 var err error
483
484 initPage := func(p *pageState) error {
485 if err = p.shiftToOutputFormat(isRenderingSite, idx); err != nil {
486 return err
487 }
488 return nil
489 }
490
491 return s.pageMap.forEeachPageIncludingBundledPages(nil,
492 func(p *pageState) (bool, error) {
493 return false, initPage(p)
494 },
495 )
496 }
497
498 func (h *HugoSites) loadData() error {
499 h.data = make(map[string]any)
500 w := hugofs.NewWalkway(
501 hugofs.WalkwayConfig{
502 Fs: h.PathSpec.BaseFs.Data.Fs,
503 IgnoreFile: h.SourceSpec.IgnoreFile,
504 PathParser: h.Conf.PathParser(),
505 WalkFn: func(path string, fi hugofs.FileMetaInfo) error {
506 if fi.IsDir() {
507 return nil
508 }
509 pi := fi.Meta().PathInfo
510 if pi == nil {
511 panic("no path info")
512 }
513 return h.handleDataFile(source.NewFileInfo(fi))
514 },
515 })
516
517 if err := w.Walk(); err != nil {
518 return err
519 }
520 return nil
521 }
522
523 func (h *HugoSites) handleDataFile(r *source.File) error {
524 var current map[string]any
525
526 f, err := r.FileInfo().Meta().Open()
527 if err != nil {
528 return fmt.Errorf("data: failed to open %q: %w", r.LogicalName(), err)
529 }
530 defer f.Close()
531
532 // Crawl in data tree to insert data
533 current = h.data
534 dataPath := r.FileInfo().Meta().PathInfo.Unnormalized().Dir()[1:]
535 keyParts := strings.Split(dataPath, "/")
536
537 for _, key := range keyParts {
538 if key != "" {
539 if _, ok := current[key]; !ok {
540 current[key] = make(map[string]any)
541 }
542 current = current[key].(map[string]any)
543 }
544 }
545
546 data, err := h.readData(r)
547 if err != nil {
548 return h.errWithFileContext(err, r)
549 }
550
551 if data == nil {
552 return nil
553 }
554
555 // filepath.Walk walks the files in lexical order, '/' comes before '.'
556 higherPrecedentData := current[r.BaseFileName()]
557
558 switch data.(type) {
559 case map[string]any:
560
561 switch higherPrecedentData.(type) {
562 case nil:
563 current[r.BaseFileName()] = data
564 case map[string]any:
565 // merge maps: insert entries from data for keys that
566 // don't already exist in higherPrecedentData
567 higherPrecedentMap := higherPrecedentData.(map[string]any)
568 for key, value := range data.(map[string]any) {
569 if _, exists := higherPrecedentMap[key]; exists {
570 // this warning could happen if
571 // 1. A theme uses the same key; the main data folder wins
572 // 2. A sub folder uses the same key: the sub folder wins
573 // TODO(bep) figure out a way to detect 2) above and make that a WARN
574 h.Log.Infof("Data for key '%s' in path '%s' is overridden by higher precedence data already in the data tree", key, r.Path())
575 } else {
576 higherPrecedentMap[key] = value
577 }
578 }
579 default:
580 // can't merge: higherPrecedentData is not a map
581 h.Log.Warnf("The %T data from '%s' overridden by "+
582 "higher precedence %T data already in the data tree", data, r.Path(), higherPrecedentData)
583 }
584
585 case []any:
586 if higherPrecedentData == nil {
587 current[r.BaseFileName()] = data
588 } else {
589 // we don't merge array data
590 h.Log.Warnf("The %T data from '%s' overridden by "+
591 "higher precedence %T data already in the data tree", data, r.Path(), higherPrecedentData)
592 }
593
594 default:
595 h.Log.Errorf("unexpected data type %T in file %s", data, r.LogicalName())
596 }
597
598 return nil
599 }
600
601 func (h *HugoSites) errWithFileContext(err error, f *source.File) error {
602 realFilename := f.FileInfo().Meta().Filename
603 return herrors.NewFileErrorFromFile(err, realFilename, h.Fs.Source, nil)
604 }
605
606 func (h *HugoSites) readData(f *source.File) (any, error) {
607 file, err := f.FileInfo().Meta().Open()
608 if err != nil {
609 return nil, fmt.Errorf("readData: failed to open data file: %w", err)
610 }
611 defer file.Close()
612 content := helpers.ReaderToBytes(file)
613
614 format := metadecoders.FormatFromString(f.Ext())
615 return metadecoders.Default.Unmarshal(content, format)
616 }