hugo_sites_build.go - hugo - [fork] hugo port for 9front
HTML git clone https://git.drkhsh.at/hugo.git
DIR Log
DIR Files
DIR Refs
DIR Submodules
DIR README
DIR LICENSE
---
hugo_sites_build.go (34128B)
---
1 // Copyright 2024 The Hugo Authors. All rights reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 // http://www.apache.org/licenses/LICENSE-2.0
7 //
8 // Unless required by applicable law or agreed to in writing, software
9 // distributed under the License is distributed on an "AS IS" BASIS,
10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 // See the License for the specific language governing permissions and
12 // limitations under the License.
13
14 package hugolib
15
16 import (
17 "bytes"
18 "context"
19 "encoding/json"
20 "errors"
21 "fmt"
22 "os"
23 "path"
24 "path/filepath"
25 "strings"
26 "time"
27
28 "github.com/bep/logg"
29 "github.com/gohugoio/hugo/bufferpool"
30 "github.com/gohugoio/hugo/deps"
31 "github.com/gohugoio/hugo/hugofs"
32 "github.com/gohugoio/hugo/hugofs/files"
33 "github.com/gohugoio/hugo/hugofs/glob"
34 "github.com/gohugoio/hugo/hugolib/doctree"
35 "github.com/gohugoio/hugo/hugolib/pagesfromdata"
36 "github.com/gohugoio/hugo/hugolib/segments"
37 "github.com/gohugoio/hugo/identity"
38 "github.com/gohugoio/hugo/output"
39 "github.com/gohugoio/hugo/publisher"
40 "github.com/gohugoio/hugo/source"
41 "github.com/gohugoio/hugo/tpl"
42
43 "github.com/gohugoio/hugo/common/herrors"
44 "github.com/gohugoio/hugo/common/loggers"
45 "github.com/gohugoio/hugo/common/para"
46 "github.com/gohugoio/hugo/common/paths"
47 "github.com/gohugoio/hugo/common/rungroup"
48 "github.com/gohugoio/hugo/config"
49 "github.com/gohugoio/hugo/resources/page"
50 "github.com/gohugoio/hugo/resources/page/siteidentities"
51 "github.com/gohugoio/hugo/resources/postpub"
52
53 "github.com/spf13/afero"
54
55 "github.com/fsnotify/fsnotify"
56 )
57
58 // Build builds all sites. If filesystem events are provided,
59 // this is considered to be a potential partial rebuild.
60 func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error {
61 infol := h.Log.InfoCommand("build")
62 defer loggers.TimeTrackf(infol, time.Now(), nil, "")
63 defer func() {
64 h.buildCounter.Add(1)
65 }()
66
67 if h.Deps == nil {
68 panic("must have deps")
69 }
70
71 if !config.NoBuildLock {
72 unlock, err := h.BaseFs.LockBuild()
73 if err != nil {
74 return fmt.Errorf("failed to acquire a build lock: %w", err)
75 }
76 defer unlock()
77 }
78
79 defer func() {
80 for _, s := range h.Sites {
81 s.Deps.BuildEndListeners.Notify()
82 }
83 }()
84
85 errCollector := h.StartErrorCollector()
86 errs := make(chan error)
87
88 go func(from, to chan error) {
89 var errors []error
90 i := 0
91 for e := range from {
92 i++
93 if i > 50 {
94 break
95 }
96 errors = append(errors, e)
97 }
98 to <- h.pickOneAndLogTheRest(errors)
99
100 close(to)
101 }(errCollector, errs)
102
103 for _, s := range h.Sites {
104 s.state = siteStateInit
105 }
106
107 if h.Metrics != nil {
108 h.Metrics.Reset()
109 }
110
111 h.buildCounters = config.testCounters
112 if h.buildCounters == nil {
113 h.buildCounters = &buildCounters{}
114 }
115
116 // Need a pointer as this may be modified.
117 conf := &config
118 if conf.WhatChanged == nil {
119 // Assume everything has changed
120 conf.WhatChanged = &WhatChanged{needsPagesAssembly: true}
121 }
122
123 var prepareErr error
124
125 if !config.PartialReRender {
126 prepare := func() error {
127 init := func(conf *BuildCfg) error {
128 for _, s := range h.Sites {
129 s.Deps.BuildStartListeners.Notify()
130 }
131
132 if len(events) > 0 || len(conf.WhatChanged.Changes()) > 0 {
133 // Rebuild
134 if err := h.initRebuild(conf); err != nil {
135 return fmt.Errorf("initRebuild: %w", err)
136 }
137 } else {
138 if err := h.initSites(conf); err != nil {
139 return fmt.Errorf("initSites: %w", err)
140 }
141 }
142
143 return nil
144 }
145
146 ctx := context.Background()
147
148 if err := h.process(ctx, infol, conf, init, events...); err != nil {
149 return fmt.Errorf("process: %w", err)
150 }
151
152 if err := h.assemble(ctx, infol, conf); err != nil {
153 return fmt.Errorf("assemble: %w", err)
154 }
155
156 return nil
157 }
158
159 if prepareErr = prepare(); prepareErr != nil {
160 h.SendError(prepareErr)
161 }
162 }
163
164 for _, s := range h.Sites {
165 s.state = siteStateReady
166 }
167
168 if prepareErr == nil {
169 if err := h.render(infol, conf); err != nil {
170 h.SendError(fmt.Errorf("render: %w", err))
171 }
172
173 // Make sure to write any build stats to disk first so it's available
174 // to the post processors.
175 if err := h.writeBuildStats(); err != nil {
176 return err
177 }
178
179 // We need to do this before render deferred.
180 if err := h.printPathWarningsOnce(); err != nil {
181 h.SendError(fmt.Errorf("printPathWarnings: %w", err))
182 }
183
184 if err := h.renderDeferred(infol); err != nil {
185 h.SendError(fmt.Errorf("renderDeferred: %w", err))
186 }
187
188 // This needs to be done after the deferred rendering to get complete template usage coverage.
189 if err := h.printUnusedTemplatesOnce(); err != nil {
190 h.SendError(fmt.Errorf("printPathWarnings: %w", err))
191 }
192
193 if err := h.postProcess(infol); err != nil {
194 h.SendError(fmt.Errorf("postProcess: %w", err))
195 }
196 }
197
198 if h.Metrics != nil {
199 var b bytes.Buffer
200 h.Metrics.WriteMetrics(&b)
201
202 h.Log.Printf("\nTemplate Metrics:\n\n")
203 h.Log.Println(b.String())
204 }
205
206 h.StopErrorCollector()
207
208 err := <-errs
209 if err != nil {
210 return err
211 }
212
213 if err := h.fatalErrorHandler.getErr(); err != nil {
214 return err
215 }
216
217 errorCount := h.Log.LoggCount(logg.LevelError) + loggers.Log().LoggCount(logg.LevelError)
218 if errorCount > 0 {
219 return fmt.Errorf("logged %d error(s)", errorCount)
220 }
221
222 return nil
223 }
224
225 // Build lifecycle methods below.
226 // The order listed matches the order of execution.
227
228 func (h *HugoSites) initSites(config *BuildCfg) error {
229 h.reset(config)
230 return nil
231 }
232
233 func (h *HugoSites) initRebuild(config *BuildCfg) error {
234 if !h.Configs.Base.Internal.Watch {
235 return errors.New("rebuild called when not in watch mode")
236 }
237
238 h.pageTrees.treePagesResources.WalkPrefixRaw("", func(key string, n contentNodeI) bool {
239 n.resetBuildState()
240 return false
241 })
242
243 for _, s := range h.Sites {
244 s.resetBuildState(config.WhatChanged.needsPagesAssembly)
245 }
246
247 h.reset(config)
248 h.resetLogs()
249
250 return nil
251 }
252
253 // process prepares the Sites' sources for a full or partial rebuild.
254 // This will also parse the source and create all the Page objects.
255 func (h *HugoSites) process(ctx context.Context, l logg.LevelLogger, config *BuildCfg, init func(config *BuildCfg) error, events ...fsnotify.Event) error {
256 l = l.WithField("step", "process")
257 defer loggers.TimeTrackf(l, time.Now(), nil, "")
258
259 if len(events) > 0 {
260 // This is a rebuild triggered from file events.
261 return h.processPartialFileEvents(ctx, l, config, init, events)
262 } else if len(config.WhatChanged.Changes()) > 0 {
263 // Rebuild triggered from remote events.
264 if err := init(config); err != nil {
265 return err
266 }
267 return h.processPartialRebuildChanges(ctx, l, config)
268 }
269 return h.processFull(ctx, l, config)
270 }
271
272 // assemble creates missing sections, applies aggregate values (e.g. dates, cascading params),
273 // removes disabled pages etc.
274 func (h *HugoSites) assemble(ctx context.Context, l logg.LevelLogger, bcfg *BuildCfg) error {
275 l = l.WithField("step", "assemble")
276 defer loggers.TimeTrackf(l, time.Now(), nil, "")
277
278 if !bcfg.WhatChanged.needsPagesAssembly {
279 changes := bcfg.WhatChanged.Drain()
280 if len(changes) > 0 {
281 if err := h.resolveAndClearStateForIdentities(ctx, l, nil, changes); err != nil {
282 return err
283 }
284 }
285 return nil
286 }
287
288 h.translationKeyPages.Reset()
289 assemblers := make([]*sitePagesAssembler, len(h.Sites))
290 // Changes detected during assembly (e.g. aggregate date changes)
291
292 for i, s := range h.Sites {
293 assemblers[i] = &sitePagesAssembler{
294 Site: s,
295 assembleChanges: bcfg.WhatChanged,
296 ctx: ctx,
297 }
298 }
299
300 g, _ := h.workersSite.Start(ctx)
301 for _, s := range assemblers {
302 s := s
303 g.Run(func() error {
304 return s.assemblePagesStep1(ctx)
305 })
306 }
307 if err := g.Wait(); err != nil {
308 return err
309 }
310
311 changes := bcfg.WhatChanged.Drain()
312
313 // Changes from the assemble step (e.g. lastMod, cascade) needs a re-calculation
314 // of what needs to be re-built.
315 if len(changes) > 0 {
316 if err := h.resolveAndClearStateForIdentities(ctx, l, nil, changes); err != nil {
317 return err
318 }
319 }
320
321 for _, s := range assemblers {
322 if err := s.assemblePagesStep2(); err != nil {
323 return err
324 }
325 }
326
327 // Handle new terms from assemblePagesStep2.
328 changes = bcfg.WhatChanged.Drain()
329 if len(changes) > 0 {
330 if err := h.resolveAndClearStateForIdentities(ctx, l, nil, changes); err != nil {
331 return err
332 }
333 }
334
335 h.renderFormats = output.Formats{}
336 for _, s := range h.Sites {
337 s.s.initRenderFormats()
338 h.renderFormats = append(h.renderFormats, s.renderFormats...)
339 }
340
341 for _, s := range assemblers {
342 if err := s.assemblePagesStepFinal(); err != nil {
343 return err
344 }
345 }
346
347 return nil
348 }
349
350 // render renders the sites.
351 func (h *HugoSites) render(l logg.LevelLogger, config *BuildCfg) error {
352 l = l.WithField("step", "render")
353 start := time.Now()
354 defer func() {
355 loggers.TimeTrackf(l, start, h.buildCounters.loggFields(), "")
356 }()
357
358 siteRenderContext := &siteRenderContext{cfg: config, infol: l, multihost: h.Configs.IsMultihost}
359
360 renderErr := func(err error) error {
361 if err == nil {
362 return nil
363 }
364 // In Hugo 0.141.0 we replaced the special error handling for resources.GetRemote
365 // with the more general try.
366 if strings.Contains(err.Error(), "can't evaluate field Err in type") {
367 if strings.Contains(err.Error(), "resource.Resource") {
368 return fmt.Errorf("%s: Resource.Err was removed in Hugo v0.141.0 and replaced with a new try keyword, see https://gohugo.io/functions/go-template/try/", err)
369 } else if strings.Contains(err.Error(), "template.HTML") {
370 return fmt.Errorf("%s: the return type of transform.ToMath was changed in Hugo v0.141.0 and the error handling replaced with a new try keyword, see https://gohugo.io/functions/go-template/try/", err)
371 }
372 }
373 return err
374 }
375
376 i := 0
377 for _, s := range h.Sites {
378 segmentFilter := s.conf.C.SegmentFilter
379 if segmentFilter.ShouldExcludeCoarse(segments.SegmentMatcherFields{Lang: s.language.Lang}) {
380 l.Logf("skip language %q not matching segments set in --renderSegments", s.language.Lang)
381 continue
382 }
383
384 siteRenderContext.languageIdx = s.languagei
385 h.currentSite = s
386 for siteOutIdx, renderFormat := range s.renderFormats {
387 if segmentFilter.ShouldExcludeCoarse(segments.SegmentMatcherFields{Output: renderFormat.Name, Lang: s.language.Lang}) {
388 l.Logf("skip output format %q for language %q not matching segments set in --renderSegments", renderFormat.Name, s.language.Lang)
389 continue
390 }
391
392 if err := func() error {
393 rc := tpl.RenderingContext{Site: s, SiteOutIdx: siteOutIdx}
394 h.BuildState.StartStageRender(rc)
395 defer h.BuildState.StopStageRender(rc)
396
397 siteRenderContext.outIdx = siteOutIdx
398 siteRenderContext.sitesOutIdx = i
399 i++
400
401 select {
402 case <-h.Done():
403 return nil
404 default:
405 for _, s2 := range h.Sites {
406 if err := s2.preparePagesForRender(s == s2, siteRenderContext.sitesOutIdx); err != nil {
407 return err
408 }
409 }
410 if !config.SkipRender {
411 ll := l.WithField("substep", "pages").
412 WithField("site", s.language.Lang).
413 WithField("outputFormat", renderFormat.Name)
414
415 start := time.Now()
416
417 if config.PartialReRender {
418 if err := s.renderPages(siteRenderContext); err != nil {
419 return err
420 }
421 } else {
422 if err := s.render(siteRenderContext); err != nil {
423 return renderErr(err)
424 }
425 }
426 loggers.TimeTrackf(ll, start, nil, "")
427 }
428 }
429 return nil
430 }(); err != nil {
431 return err
432 }
433
434 }
435 }
436
437 return nil
438 }
439
440 func (h *HugoSites) renderDeferred(l logg.LevelLogger) error {
441 l = l.WithField("step", "render deferred")
442 start := time.Now()
443
444 var deferredCount int
445
446 for rc, de := range h.Deps.BuildState.DeferredExecutionsGroupedByRenderingContext {
447 if de.FilenamesWithPostPrefix.Len() == 0 {
448 continue
449 }
450
451 deferredCount += de.FilenamesWithPostPrefix.Len()
452
453 s := rc.Site.(*Site)
454 for _, s2 := range h.Sites {
455 if err := s2.preparePagesForRender(s == s2, rc.SiteOutIdx); err != nil {
456 return err
457 }
458 }
459 if err := s.executeDeferredTemplates(de); err != nil {
460 return herrors.ImproveRenderErr(err)
461 }
462 }
463
464 loggers.TimeTrackf(l, start, logg.Fields{
465 logg.Field{Name: "count", Value: deferredCount},
466 }, "")
467
468 return nil
469 }
470
471 func (s *Site) executeDeferredTemplates(de *deps.DeferredExecutions) error {
472 handleFile := func(filename string) error {
473 content, err := afero.ReadFile(s.BaseFs.PublishFs, filename)
474 if err != nil {
475 return err
476 }
477
478 k := 0
479 changed := false
480
481 for {
482 if k >= len(content) {
483 break
484 }
485 l := bytes.Index(content[k:], []byte(tpl.HugoDeferredTemplatePrefix))
486 if l == -1 {
487 break
488 }
489 m := bytes.Index(content[k+l:], []byte(tpl.HugoDeferredTemplateSuffix)) + len(tpl.HugoDeferredTemplateSuffix)
490
491 low, high := k+l, k+l+m
492
493 forward := l + m
494 id := string(content[low:high])
495
496 if err := func() error {
497 deferred, found := de.Executions.Get(id)
498 if !found {
499 panic(fmt.Sprintf("deferred execution with id %q not found", id))
500 }
501 deferred.Mu.Lock()
502 defer deferred.Mu.Unlock()
503
504 if !deferred.Executed {
505 tmpl := s.Deps.GetTemplateStore()
506 ti := s.TemplateStore.LookupByPath(deferred.TemplatePath)
507 if ti == nil {
508 panic(fmt.Sprintf("template %q not found", deferred.TemplatePath))
509 }
510
511 if err := func() error {
512 buf := bufferpool.GetBuffer()
513 defer bufferpool.PutBuffer(buf)
514
515 err = tmpl.ExecuteWithContext(deferred.Ctx, ti, buf, deferred.Data)
516 if err != nil {
517 return err
518 }
519 deferred.Result = buf.String()
520 deferred.Executed = true
521
522 return nil
523 }(); err != nil {
524 return err
525 }
526 }
527
528 content = append(content[:low], append([]byte(deferred.Result), content[high:]...)...)
529 forward = len(deferred.Result)
530 changed = true
531
532 return nil
533 }(); err != nil {
534 return err
535 }
536
537 k += forward
538 }
539
540 if changed {
541 return afero.WriteFile(s.BaseFs.PublishFs, filename, content, 0o666)
542 }
543
544 return nil
545 }
546
547 g := rungroup.Run[string](context.Background(), rungroup.Config[string]{
548 NumWorkers: s.h.numWorkers,
549 Handle: func(ctx context.Context, filename string) error {
550 return handleFile(filename)
551 },
552 })
553
554 de.FilenamesWithPostPrefix.ForEeach(func(filename string, _ bool) bool {
555 g.Enqueue(filename)
556 return true
557 })
558
559 return g.Wait()
560 }
561
562 // printPathWarningsOnce prints path warnings if enabled.
563 func (h *HugoSites) printPathWarningsOnce() error {
564 h.printPathWarningsInit.Do(func() {
565 conf := h.Configs.Base
566 if conf.PrintPathWarnings {
567 // We need to do this before any post processing, as that may write to the same files twice
568 // and create false positives.
569 hugofs.WalkFilesystems(h.Fs.PublishDir, func(fs afero.Fs) bool {
570 if dfs, ok := fs.(hugofs.DuplicatesReporter); ok {
571 dupes := dfs.ReportDuplicates()
572 if dupes != "" {
573 h.Log.Warnln("Duplicate target paths:", dupes)
574 }
575 }
576 return false
577 })
578 }
579 })
580 return nil
581 }
582
583 // / printUnusedTemplatesOnce prints unused templates if enabled.
584 func (h *HugoSites) printUnusedTemplatesOnce() error {
585 h.printUnusedTemplatesInit.Do(func() {
586 conf := h.Configs.Base
587 if conf.PrintUnusedTemplates {
588 unusedTemplates := h.GetTemplateStore().UnusedTemplates()
589 for _, unusedTemplate := range unusedTemplates {
590 if unusedTemplate.Fi != nil {
591 h.Log.Warnf("Template %s is unused, source %q", unusedTemplate.PathInfo.Path(), unusedTemplate.Fi.Meta().Filename)
592 } else {
593 h.Log.Warnf("Template %s is unused", unusedTemplate.PathInfo.Path())
594 }
595 }
596 }
597 })
598 return nil
599 }
600
601 // postProcess runs the post processors, e.g. writing the hugo_stats.json file.
602 func (h *HugoSites) postProcess(l logg.LevelLogger) error {
603 l = l.WithField("step", "postProcess")
604 defer loggers.TimeTrackf(l, time.Now(), nil, "")
605
606 // This will only be set when js.Build have been triggered with
607 // imports that resolves to the project or a module.
608 // Write a jsconfig.json file to the project's /asset directory
609 // to help JS IntelliSense in VS Code etc.
610 if !h.ResourceSpec.BuildConfig().NoJSConfigInAssets {
611 handleJSConfig := func(fi os.FileInfo) {
612 m := fi.(hugofs.FileMetaInfo).Meta()
613 if !m.IsProject {
614 return
615 }
616
617 if jsConfig := h.ResourceSpec.JSConfigBuilder.Build(m.SourceRoot); jsConfig != nil {
618 b, err := json.MarshalIndent(jsConfig, "", " ")
619 if err != nil {
620 h.Log.Warnf("Failed to create jsconfig.json: %s", err)
621 } else {
622 filename := filepath.Join(m.SourceRoot, "jsconfig.json")
623 if h.Configs.Base.Internal.Running {
624 h.skipRebuildForFilenamesMu.Lock()
625 h.skipRebuildForFilenames[filename] = true
626 h.skipRebuildForFilenamesMu.Unlock()
627 }
628 // Make sure it's written to the OS fs as this is used by
629 // editors.
630 if err := afero.WriteFile(hugofs.Os, filename, b, 0o666); err != nil {
631 h.Log.Warnf("Failed to write jsconfig.json: %s", err)
632 }
633 }
634 }
635 }
636
637 fi, err := h.BaseFs.Assets.Fs.Stat("")
638 if err != nil {
639 if !herrors.IsNotExist(err) {
640 h.Log.Warnf("Failed to resolve jsconfig.json dir: %s", err)
641 }
642 } else {
643 handleJSConfig(fi)
644 }
645 }
646
647 var toPostProcess []postpub.PostPublishedResource
648 for _, r := range h.ResourceSpec.PostProcessResources {
649 toPostProcess = append(toPostProcess, r)
650 }
651
652 if len(toPostProcess) == 0 {
653 // Nothing more to do.
654 return nil
655 }
656
657 workers := para.New(config.GetNumWorkerMultiplier())
658 g, _ := workers.Start(context.Background())
659
660 handleFile := func(filename string) error {
661 content, err := afero.ReadFile(h.BaseFs.PublishFs, filename)
662 if err != nil {
663 return err
664 }
665
666 k := 0
667 changed := false
668
669 for {
670 l := bytes.Index(content[k:], []byte(postpub.PostProcessPrefix))
671 if l == -1 {
672 break
673 }
674 m := bytes.Index(content[k+l:], []byte(postpub.PostProcessSuffix)) + len(postpub.PostProcessSuffix)
675
676 low, high := k+l, k+l+m
677
678 field := content[low:high]
679
680 forward := l + m
681
682 for i, r := range toPostProcess {
683 if r == nil {
684 panic(fmt.Sprintf("resource %d to post process is nil", i+1))
685 }
686 v, ok := r.GetFieldString(string(field))
687 if ok {
688 content = append(content[:low], append([]byte(v), content[high:]...)...)
689 changed = true
690 forward = len(v)
691 break
692 }
693 }
694
695 k += forward
696 }
697
698 if changed {
699 return afero.WriteFile(h.BaseFs.PublishFs, filename, content, 0o666)
700 }
701
702 return nil
703 }
704
705 filenames := h.Deps.BuildState.GetFilenamesWithPostPrefix()
706 for _, filename := range filenames {
707 filename := filename
708 g.Run(func() error {
709 return handleFile(filename)
710 })
711 }
712
713 // Prepare for a new build.
714 for _, s := range h.Sites {
715 s.ResourceSpec.PostProcessResources = make(map[string]postpub.PostPublishedResource)
716 }
717
718 return g.Wait()
719 }
720
721 func (h *HugoSites) writeBuildStats() error {
722 if h.ResourceSpec == nil {
723 panic("h.ResourceSpec is nil")
724 }
725 if !h.ResourceSpec.BuildConfig().BuildStats.Enabled() {
726 return nil
727 }
728
729 htmlElements := &publisher.HTMLElements{}
730 for _, s := range h.Sites {
731 stats := s.publisher.PublishStats()
732 htmlElements.Merge(stats.HTMLElements)
733 }
734
735 htmlElements.Sort()
736
737 stats := publisher.PublishStats{
738 HTMLElements: *htmlElements,
739 }
740
741 var buf bytes.Buffer
742 enc := json.NewEncoder(&buf)
743 enc.SetEscapeHTML(false)
744 enc.SetIndent("", " ")
745 err := enc.Encode(stats)
746 if err != nil {
747 return err
748 }
749 js := buf.Bytes()
750
751 filename := filepath.Join(h.Configs.LoadingInfo.BaseConfig.WorkingDir, files.FilenameHugoStatsJSON)
752
753 if existingContent, err := afero.ReadFile(hugofs.Os, filename); err == nil {
754 // Check if the content has changed.
755 if bytes.Equal(existingContent, js) {
756 return nil
757 }
758 }
759
760 // Make sure it's always written to the OS fs.
761 if err := afero.WriteFile(hugofs.Os, filename, js, 0o666); err != nil {
762 return err
763 }
764
765 // Write to the destination as well if it's a in-memory fs.
766 if !hugofs.IsOsFs(h.Fs.Source) {
767 if err := afero.WriteFile(h.Fs.WorkingDirWritable, filename, js, 0o666); err != nil {
768 return err
769 }
770 }
771
772 // This step may be followed by a post process step that may
773 // rebuild e.g. CSS, so clear any cache that's defined for the hugo_stats.json.
774 h.dynacacheGCFilenameIfNotWatchedAndDrainMatching(filename)
775
776 return nil
777 }
778
779 type pathChange struct {
780 // The path to the changed file.
781 p *paths.Path
782
783 // If true, this is a structural change (e.g. a delete or a rename).
784 structural bool
785
786 // If true, this is a directory.
787 isDir bool
788 }
789
790 func (p pathChange) isStructuralChange() bool {
791 return p.structural || p.isDir
792 }
793
794 func (h *HugoSites) processPartialRebuildChanges(ctx context.Context, l logg.LevelLogger, config *BuildCfg) error {
795 if err := h.resolveAndClearStateForIdentities(ctx, l, nil, config.WhatChanged.Drain()); err != nil {
796 return err
797 }
798
799 if err := h.processContentAdaptersOnRebuild(ctx, config); err != nil {
800 return err
801 }
802 return nil
803 }
804
805 // processPartialFileEvents prepares the Sites' sources for a partial rebuild.
806 func (h *HugoSites) processPartialFileEvents(ctx context.Context, l logg.LevelLogger, config *BuildCfg, init func(config *BuildCfg) error, events []fsnotify.Event) error {
807 h.Log.Trace(logg.StringFunc(func() string {
808 var sb strings.Builder
809 sb.WriteString("File events:\n")
810 for _, ev := range events {
811 sb.WriteString(ev.String())
812 sb.WriteString("\n")
813 }
814 return sb.String()
815 }))
816
817 // For a list of events for the different OSes, see the test output in https://github.com/bep/fsnotifyeventlister/.
818 events = h.fileEventsFilter(events)
819 events = h.fileEventsTrim(events)
820 eventInfos := h.fileEventsApplyInfo(events)
821
822 logger := h.Log
823
824 var (
825 tmplAdded bool
826 tmplChanged bool
827 i18nChanged bool
828 needsPagesAssemble bool
829 )
830
831 changedPaths := struct {
832 changedFiles []*paths.Path
833 changedDirs []*paths.Path
834 deleted []*paths.Path
835 }{}
836
837 removeDuplicatePaths := func(ps []*paths.Path) []*paths.Path {
838 seen := make(map[string]bool)
839 var filtered []*paths.Path
840 for _, p := range ps {
841 if !seen[p.Path()] {
842 seen[p.Path()] = true
843 filtered = append(filtered, p)
844 }
845 }
846 return filtered
847 }
848
849 var (
850 cacheBusters []func(string) bool
851 deletedDirs []string
852 addedContentPaths []*paths.Path
853 )
854
855 var (
856 addedOrChangedContent []pathChange
857 changes []identity.Identity
858 )
859
860 for _, ev := range eventInfos {
861 cpss := h.BaseFs.ResolvePaths(ev.Name)
862 pss := make([]*paths.Path, len(cpss))
863 for i, cps := range cpss {
864 p := cps.Path
865 if ev.removed && !paths.HasExt(p) {
866 // Assume this is a renamed/removed directory.
867 // For deletes, we walk up the tree to find the container (e.g. branch bundle),
868 // so we will catch this even if it is a file without extension.
869 // This avoids us walking up to the home page bundle for the common case
870 // of renaming root sections.
871 p = p + "/_index.md"
872 deletedDirs = append(deletedDirs, cps.Path)
873 }
874
875 pss[i] = h.Configs.ContentPathParser.Parse(cps.Component, p)
876 if ev.added && !ev.isChangedDir && cps.Component == files.ComponentFolderContent {
877 addedContentPaths = append(addedContentPaths, pss[i])
878 }
879
880 // Compile cache buster.
881 np := glob.NormalizePath(path.Join(cps.Component, cps.Path))
882 g, err := h.ResourceSpec.BuildConfig().MatchCacheBuster(h.Log, np)
883 if err == nil && g != nil {
884 cacheBusters = append(cacheBusters, g)
885 }
886
887 if ev.added {
888 changes = append(changes, identity.StructuralChangeAdd)
889 }
890 if ev.removed {
891 changes = append(changes, identity.StructuralChangeRemove)
892 }
893 }
894
895 if ev.removed {
896 changedPaths.deleted = append(changedPaths.deleted, pss...)
897 } else if ev.isChangedDir {
898 changedPaths.changedDirs = append(changedPaths.changedDirs, pss...)
899 } else {
900 changedPaths.changedFiles = append(changedPaths.changedFiles, pss...)
901 }
902 }
903
904 // Find the most specific identity possible.
905 handleChange := func(pathInfo *paths.Path, delete, isDir bool) {
906 switch pathInfo.Component() {
907 case files.ComponentFolderContent:
908 logger.Println("Source changed", pathInfo.Path())
909 isContentDataFile := pathInfo.IsContentData()
910 if !isContentDataFile {
911 if ids := h.pageTrees.collectAndMarkStaleIdentities(pathInfo); len(ids) > 0 {
912 changes = append(changes, ids...)
913 }
914 } else {
915 h.pageTrees.treePagesFromTemplateAdapters.DeleteAllFunc(pathInfo.Base(),
916 func(s string, n *pagesfromdata.PagesFromTemplate) bool {
917 changes = append(changes, n.DependencyManager)
918
919 // Try to open the file to see if has been deleted.
920 f, err := n.GoTmplFi.Meta().Open()
921 if err == nil {
922 f.Close()
923 }
924 if err != nil {
925 // Remove all pages and resources below.
926 prefix := pathInfo.Base() + "/"
927 h.pageTrees.treePages.DeletePrefixAll(prefix)
928 h.pageTrees.resourceTrees.DeletePrefixAll(prefix)
929 changes = append(changes, identity.NewGlobIdentity(prefix+"*"))
930 }
931 return err != nil
932 })
933 }
934
935 needsPagesAssemble = true
936
937 if config.RecentlyTouched != nil {
938 // Fast render mode. Adding them to the visited queue
939 // avoids rerendering them on navigation.
940 for _, id := range changes {
941 if p, ok := id.(page.Page); ok {
942 config.RecentlyTouched.Add(p.RelPermalink())
943 }
944 }
945 }
946
947 h.pageTrees.treeTaxonomyEntries.DeletePrefix("")
948
949 if delete && !isContentDataFile {
950 _, ok := h.pageTrees.treePages.LongestPrefixAll(pathInfo.Base())
951 if ok {
952 h.pageTrees.treePages.DeleteAll(pathInfo.Base())
953 h.pageTrees.resourceTrees.DeleteAll(pathInfo.Base())
954 if pathInfo.IsBundle() {
955 // Assume directory removed.
956 h.pageTrees.treePages.DeletePrefixAll(pathInfo.Base() + "/")
957 h.pageTrees.resourceTrees.DeletePrefixAll(pathInfo.Base() + "/")
958 }
959 } else {
960 h.pageTrees.resourceTrees.DeleteAll(pathInfo.Base())
961 }
962 }
963
964 addedOrChangedContent = append(addedOrChangedContent, pathChange{p: pathInfo, structural: delete, isDir: isDir})
965
966 case files.ComponentFolderLayouts:
967 tmplChanged = true
968 templatePath := pathInfo.Unnormalized().TrimLeadingSlash().PathNoLang()
969 if !h.GetTemplateStore().HasTemplate(templatePath) {
970 tmplAdded = true
971 }
972
973 if tmplAdded {
974 logger.Println("Template added", pathInfo.Path())
975 // A new template may require a more coarse grained build.
976 base := pathInfo.Base()
977 if strings.Contains(base, "_markup") {
978 // It's hard to determine the exact change set of this,
979 // so be very coarse grained.
980 changes = append(changes, identity.GenghisKhan)
981 }
982 if strings.Contains(base, "shortcodes") {
983 changes = append(changes, identity.NewGlobIdentity(fmt.Sprintf("shortcodes/%s*", pathInfo.BaseNameNoIdentifier())))
984 } else {
985 changes = append(changes, pathInfo)
986 }
987 } else {
988 logger.Println("Template changed", pathInfo.Path())
989 id := h.GetTemplateStore().GetIdentity(pathInfo.Path())
990 if id != nil {
991 changes = append(changes, id)
992 } else {
993 changes = append(changes, pathInfo)
994 }
995 }
996 case files.ComponentFolderAssets:
997 logger.Println("Asset changed", pathInfo.Path())
998 changes = append(changes, pathInfo)
999 case files.ComponentFolderData:
1000 logger.Println("Data changed", pathInfo.Path())
1001
1002 // This should cover all usage of site.Data.
1003 // Currently very coarse grained.
1004 changes = append(changes, siteidentities.Data)
1005 h.init.data.Reset()
1006 case files.ComponentFolderI18n:
1007 logger.Println("i18n changed", pathInfo.Path())
1008 i18nChanged = true
1009 // It's hard to determine the exact change set of this,
1010 // so be very coarse grained for now.
1011 changes = append(changes, identity.GenghisKhan)
1012 case files.ComponentFolderArchetypes:
1013 // Ignore for now.
1014 default:
1015 panic(fmt.Sprintf("unknown component: %q", pathInfo.Component()))
1016 }
1017 }
1018
1019 changedPaths.deleted = removeDuplicatePaths(changedPaths.deleted)
1020 changedPaths.changedFiles = removeDuplicatePaths(changedPaths.changedFiles)
1021
1022 h.Log.Trace(logg.StringFunc(func() string {
1023 var sb strings.Builder
1024 sb.WriteString("Resolved paths:\n")
1025 sb.WriteString("Deleted:\n")
1026 for _, p := range changedPaths.deleted {
1027 sb.WriteString("path: " + p.Path())
1028 sb.WriteString("\n")
1029 }
1030 sb.WriteString("Changed:\n")
1031 for _, p := range changedPaths.changedFiles {
1032 sb.WriteString("path: " + p.Path())
1033 sb.WriteString("\n")
1034 }
1035 return sb.String()
1036 }))
1037
1038 for _, deletedDir := range deletedDirs {
1039 prefix := deletedDir + "/"
1040 predicate := func(id identity.Identity) bool {
1041 // This will effectively reset all pages below this dir.
1042 return strings.HasPrefix(paths.AddLeadingSlash(id.IdentifierBase()), prefix)
1043 }
1044 // Test in both directions.
1045 changes = append(changes, identity.NewPredicateIdentity(
1046 // Is dependent.
1047 predicate,
1048 // Is dependency.
1049 predicate,
1050 ),
1051 )
1052 }
1053
1054 if len(addedContentPaths) > 0 {
1055 // These content files are new and not in use anywhere.
1056 // To make sure that these gets listed in any site.RegularPages ranges or similar
1057 // we could invalidate everything, but first try to collect a sample set
1058 // from the surrounding pages.
1059 var surroundingIDs []identity.Identity
1060 for _, p := range addedContentPaths {
1061 if ids := h.pageTrees.collectIdentitiesSurrounding(p.Base(), 10); len(ids) > 0 {
1062 surroundingIDs = append(surroundingIDs, ids...)
1063 }
1064 }
1065
1066 if len(surroundingIDs) > 0 {
1067 changes = append(changes, surroundingIDs...)
1068 } else {
1069 // No surrounding pages found, so invalidate everything.
1070 changes = append(changes, identity.GenghisKhan)
1071 }
1072 }
1073
1074 for _, deleted := range changedPaths.deleted {
1075 handleChange(deleted, true, false)
1076 }
1077
1078 for _, id := range changedPaths.changedFiles {
1079 handleChange(id, false, false)
1080 }
1081
1082 for _, id := range changedPaths.changedDirs {
1083 handleChange(id, false, true)
1084 }
1085
1086 for _, id := range changes {
1087 if id == identity.GenghisKhan {
1088 for i, cp := range addedOrChangedContent {
1089 cp.structural = true
1090 addedOrChangedContent[i] = cp
1091 }
1092 break
1093 }
1094 }
1095
1096 resourceFiles := h.fileEventsContentPaths(addedOrChangedContent)
1097
1098 changed := &WhatChanged{
1099 needsPagesAssembly: needsPagesAssemble,
1100 }
1101 changed.Add(changes...)
1102
1103 config.WhatChanged = changed
1104
1105 if err := init(config); err != nil {
1106 return err
1107 }
1108
1109 var cacheBusterOr func(string) bool
1110 if len(cacheBusters) > 0 {
1111 cacheBusterOr = func(s string) bool {
1112 for _, cb := range cacheBusters {
1113 if cb(s) {
1114 return true
1115 }
1116 }
1117 return false
1118 }
1119 }
1120
1121 changes2 := changed.Changes()
1122 h.Deps.OnChangeListeners.Notify(changes2...)
1123
1124 if err := h.resolveAndClearStateForIdentities(ctx, l, cacheBusterOr, changed.Drain()); err != nil {
1125 return err
1126 }
1127
1128 if tmplChanged {
1129 if err := loggers.TimeTrackfn(func() (logg.LevelLogger, error) {
1130 depsFinder := identity.NewFinder(identity.FinderConfig{})
1131 ll := l.WithField("substep", "rebuild templates")
1132 s := h.Sites[0]
1133 if err := s.Deps.TemplateStore.RefreshFiles(func(fi hugofs.FileMetaInfo) bool {
1134 pi := fi.Meta().PathInfo
1135 for _, id := range changes2 {
1136 if depsFinder.Contains(pi, id, -1) > 0 {
1137 return true
1138 }
1139 }
1140 return false
1141 }); err != nil {
1142 return ll, err
1143 }
1144
1145 return ll, nil
1146 }); err != nil {
1147 return err
1148 }
1149 }
1150
1151 if i18nChanged {
1152 if err := loggers.TimeTrackfn(func() (logg.LevelLogger, error) {
1153 ll := l.WithField("substep", "rebuild i18n")
1154 var prototype *deps.Deps
1155 for i, s := range h.Sites {
1156 if err := s.Deps.Compile(prototype); err != nil {
1157 return ll, err
1158 }
1159 if i == 0 {
1160 prototype = s.Deps
1161 }
1162 }
1163 return ll, nil
1164 }); err != nil {
1165 return err
1166 }
1167 }
1168
1169 if resourceFiles != nil {
1170 if err := h.processFiles(ctx, l, config, resourceFiles...); err != nil {
1171 return err
1172 }
1173 }
1174
1175 if h.isRebuild() {
1176 if err := h.processContentAdaptersOnRebuild(ctx, config); err != nil {
1177 return err
1178 }
1179 }
1180
1181 return nil
1182 }
1183
1184 func (h *HugoSites) LogServerAddresses() {
1185 if h.hugoInfo.IsMultihost() {
1186 for _, s := range h.Sites {
1187 h.Log.Printf("Web Server is available at %s (bind address %s) %s\n", s.conf.C.BaseURL, s.conf.C.ServerInterface, s.Language().Lang)
1188 }
1189 } else {
1190 s := h.Sites[0]
1191 h.Log.Printf("Web Server is available at %s (bind address %s)\n", s.conf.C.BaseURL, s.conf.C.ServerInterface)
1192 }
1193 }
1194
1195 func (h *HugoSites) processFull(ctx context.Context, l logg.LevelLogger, config *BuildCfg) (err error) {
1196 if err = h.processFiles(ctx, l, config); err != nil {
1197 err = fmt.Errorf("readAndProcessContent: %w", err)
1198 return
1199 }
1200 return err
1201 }
1202
1203 func (s *Site) handleContentAdapterChanges(bi pagesfromdata.BuildInfo, buildConfig *BuildCfg) {
1204 if !s.h.isRebuild() {
1205 return
1206 }
1207
1208 if len(bi.ChangedIdentities) > 0 {
1209 buildConfig.WhatChanged.Add(bi.ChangedIdentities...)
1210 buildConfig.WhatChanged.needsPagesAssembly = true
1211 }
1212
1213 for _, p := range bi.DeletedPaths {
1214 pp := path.Join(bi.Path.Base(), p)
1215 if v, ok := s.pageMap.treePages.Delete(pp); ok {
1216 buildConfig.WhatChanged.Add(v.GetIdentity())
1217 }
1218 }
1219 }
1220
1221 func (h *HugoSites) processContentAdaptersOnRebuild(ctx context.Context, buildConfig *BuildCfg) error {
1222 g := rungroup.Run[*pagesfromdata.PagesFromTemplate](ctx, rungroup.Config[*pagesfromdata.PagesFromTemplate]{
1223 NumWorkers: h.numWorkers,
1224 Handle: func(ctx context.Context, p *pagesfromdata.PagesFromTemplate) error {
1225 bi, err := p.Execute(ctx)
1226 if err != nil {
1227 return err
1228 }
1229 s := p.Site.(*Site)
1230 s.handleContentAdapterChanges(bi, buildConfig)
1231 return nil
1232 },
1233 })
1234
1235 h.pageTrees.treePagesFromTemplateAdapters.WalkPrefixRaw(doctree.LockTypeRead, "", func(key string, p *pagesfromdata.PagesFromTemplate) (bool, error) {
1236 if p.StaleVersion() > 0 {
1237 g.Enqueue(p)
1238 }
1239 return false, nil
1240 })
1241
1242 return g.Wait()
1243 }
1244
1245 func (s *HugoSites) processFiles(ctx context.Context, l logg.LevelLogger, buildConfig *BuildCfg, filenames ...pathChange) error {
1246 if s.Deps == nil {
1247 panic("nil deps on site")
1248 }
1249
1250 sourceSpec := source.NewSourceSpec(s.PathSpec, buildConfig.ContentInclusionFilter, s.BaseFs.Content.Fs)
1251
1252 // For inserts, we can pick an arbitrary pageMap.
1253 pageMap := s.Sites[0].pageMap
1254
1255 c := newPagesCollector(ctx, s.h, sourceSpec, s.Log, l, pageMap, buildConfig, filenames)
1256
1257 if err := c.Collect(); err != nil {
1258 return err
1259 }
1260
1261 return nil
1262 }