Create robots.txt in the domain root directory - hugo - [fork] hugo port for 9front
HTML git clone git@git.drkhsh.at/hugo.git
DIR Log
DIR Files
DIR Refs
DIR Submodules
DIR README
DIR LICENSE
---
DIR commit 3466884e364d0df39394742697d6fa0b37d49e64
DIR parent 6ff435aa3f19a628f26d5ce755238e359eb77df6
HTML Author: Nico <nico.duesing@secure.mailbox.org>
Date: Sat, 20 Jun 2020 15:00:25 +0200
Create robots.txt in the domain root directory
Before a robots.txt is created in every Site. So in public/robots.txt if there are no languages (was correct). But if there are multiple languages in every language directory, too (was wrong). If defaultContentLanguageInSubdir is true, no language is created into the root directory, so no robots.txt is in the root directory (was wrong). If multihosts are configured for each language, that is the only case where one robots.txt must be created in each language directory (was correct).
I've changed the behaviour, that only in the multihost case the robots.txt is created in the language directories. In any other case it is created in public/robots.txt. I've also added tests that files are not created in the wrong directories.
Fixes #5160
See also #4193
Diffstat:
M hugolib/hugo_sites.go | 35 ++++++++++++++++++++++++++++++-
M hugolib/hugo_sites_build.go | 5 ++++-
M hugolib/hugo_sites_build_test.go | 6 ++++--
M hugolib/hugo_sites_multihost_test.… | 7 +++++++
M hugolib/site.go | 6 ++++--
M hugolib/testhelpers_test.go | 6 ++++++
6 files changed, 59 insertions(+), 6 deletions(-)
---
DIR diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go
@@ -49,6 +49,7 @@ import (
"github.com/gohugoio/hugo/langs/i18n"
"github.com/gohugoio/hugo/resources/page"
+ "github.com/gohugoio/hugo/resources/page/pagemeta"
"github.com/gohugoio/hugo/tpl"
"github.com/gohugoio/hugo/tpl/tplimpl"
)
@@ -690,7 +691,7 @@ func (cfg *BuildCfg) shouldRender(p *pageState) bool {
return false
}
-func (h *HugoSites) renderCrossSitesArtifacts() error {
+func (h *HugoSites) renderCrossSitesSitemap() error {
if !h.multilingual.enabled() || h.IsMultihost() {
return nil
@@ -716,6 +717,38 @@ func (h *HugoSites) renderCrossSitesArtifacts() error {
s.siteCfg.sitemap.Filename, h.toSiteInfos(), templ)
}
+func (h *HugoSites) renderCrossSitesRobotsTXT() error {
+ if h.multihost {
+ return nil
+ }
+ if !h.Cfg.GetBool("enableRobotsTXT") {
+ return nil
+ }
+
+ s := h.Sites[0]
+
+ p, err := newPageStandalone(&pageMeta{
+ s: s,
+ kind: kindRobotsTXT,
+ urlPaths: pagemeta.URLPath{
+ URL: "robots.txt",
+ },
+ },
+ output.RobotsTxtFormat)
+
+ if err != nil {
+ return err
+ }
+
+ if !p.render {
+ return nil
+ }
+
+ templ := s.lookupLayouts("robots.txt", "_default/robots.txt", "_internal/_default/robots.txt")
+
+ return s.renderAndWritePage(&s.PathSpec.ProcessingStats.Pages, "Robots Txt", "robots.txt", p, templ)
+}
+
func (h *HugoSites) removePageByFilename(filename string) {
h.getContentMaps().withMaps(func(m *pageMap) error {
m.deleteBundleMatching(func(b *contentNode) bool {
DIR diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go
@@ -334,7 +334,10 @@ func (h *HugoSites) render(config *BuildCfg) error {
}
if !config.SkipRender {
- if err := h.renderCrossSitesArtifacts(); err != nil {
+ if err := h.renderCrossSitesSitemap(); err != nil {
+ return err
+ }
+ if err := h.renderCrossSitesRobotsTXT(); err != nil {
return err
}
}
DIR diff --git a/hugolib/hugo_sites_build_test.go b/hugolib/hugo_sites_build_test.go
@@ -255,8 +255,10 @@ func doTestMultiSitesBuild(t *testing.T, configTemplate, configSuffix string) {
b.AssertFileContent("public/fr/404.html", "404|fr|404 Page not found")
// Check robots.txt
- b.AssertFileContent("public/en/robots.txt", "robots|en|")
- b.AssertFileContent("public/nn/robots.txt", "robots|nn|")
+ // the domain root is the public directory, so the robots.txt has to be created there and not in the language directories
+ b.AssertFileContent("public/robots.txt", "robots")
+ b.AssertFileDoesNotExist("public/en/robots.txt")
+ b.AssertFileDoesNotExist("public/nn/robots.txt")
b.AssertFileContent("public/en/sect/doc1-slug/index.html", "Permalink: http://example.com/blog/en/sect/doc1-slug/")
b.AssertFileContent("public/en/sect/doc2/index.html", "Permalink: http://example.com/blog/en/sect/doc2/")
DIR diff --git a/hugolib/hugo_sites_multihost_test.go b/hugolib/hugo_sites_multihost_test.go
@@ -19,6 +19,7 @@ disablePathToLower = true
defaultContentLanguage = "fr"
defaultContentLanguageInSubdir = false
staticDir = ["s1", "s2"]
+enableRobotsTXT = true
[permalinks]
other = "/somewhere/else/:filename"
@@ -73,6 +74,12 @@ languageName = "Nynorsk"
c.Assert(pageWithURLInFrontMatter.RelPermalink(), qt.Equals, "/docs/superbob/")
b.AssertFileContent("public/en/superbob/index.html", "doc3|Hello|en")
+ // the domain root is the language directory for each language, so the robots.txt is created in the language directories
+ b.AssertFileContent("public/en/robots.txt", "robots|en")
+ b.AssertFileContent("public/fr/robots.txt", "robots|fr")
+ b.AssertFileContent("public/nn/robots.txt", "robots|nn")
+ b.AssertFileDoesNotExist("public/robots.txt")
+
// check alias:
b.AssertFileContent("public/en/al/alias1/index.html", `content="0; url=https://example.com/docs/superbob/"`)
b.AssertFileContent("public/en/al/alias2/index.html", `content="0; url=https://example.com/docs/superbob/"`)
DIR diff --git a/hugolib/site.go b/hugolib/site.go
@@ -1209,8 +1209,10 @@ func (s *Site) render(ctx *siteRenderContext) (err error) {
return
}
- if err = s.renderRobotsTXT(); err != nil {
- return
+ if ctx.multihost {
+ if err = s.renderRobotsTXT(); err != nil {
+ return
+ }
}
if err = s.render404(); err != nil {
DIR diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go
@@ -726,6 +726,12 @@ func (s *sitesBuilder) AssertFileContent(filename string, matches ...string) {
}
}
+func (s *sitesBuilder) AssertFileDoesNotExist(filename string) {
+ if s.CheckExists(filename) {
+ s.Fatalf("File %q exists but must not exist.", filename)
+ }
+}
+
func (s *sitesBuilder) AssertImage(width, height int, filename string) {
filename = filepath.Join(s.workingDir, filename)
f, err := s.Fs.Destination.Open(filename)