Read/reread individual source content files next is incremental conversion - hugo - [fork] hugo port for 9front
HTML git clone git@git.drkhsh.at/hugo.git
DIR Log
DIR Files
DIR Refs
DIR Submodules
DIR README
DIR LICENSE
---
DIR commit 9f3796a31dc217b2b8094f948266b23ac3808aa6
DIR parent ca6ca4f4fc29906e491b5ac6b63fb65125c9c9e4
HTML Author: Steve Francia <steve.francia@gmail.com>
Date: Thu, 7 Jan 2016 21:48:13 -0500
Read/reread individual source content files
next is incremental conversion
Diffstat:
M commands/hugo.go | 4 ++--
M docs/content/meta/roadmap.md | 1 -
M hugolib/handler_page.go | 1 +
M hugolib/page.go | 69 +++++++++++++++++++++++--------
M hugolib/site.go | 166 +++++++++++++++++++++++++++----
M source/file.go | 12 +++++++++---
M source/filesystem.go | 71 +++++++++++++++++--------------
7 files changed, 248 insertions(+), 76 deletions(-)
---
DIR diff --git a/commands/hugo.go b/commands/hugo.go
@@ -594,7 +594,7 @@ func NewWatcher(port int) error {
for _, ev := range evs {
ext := filepath.Ext(ev.Name)
- istemp := strings.HasSuffix(ext, "~") || (ext == ".swp") || (ext == ".swx") || (ext == ".tmp") || strings.HasPrefix(ext, ".goutputstream") || strings.HasSuffix(ext, "jb_old___")|| strings.HasSuffix(ext, "jb_bak___")
+ istemp := strings.HasSuffix(ext, "~") || (ext == ".swp") || (ext == ".swx") || (ext == ".tmp") || strings.HasPrefix(ext, ".goutputstream") || strings.HasSuffix(ext, "jb_old___") || strings.HasSuffix(ext, "jb_bak___")
if istemp {
continue
}
@@ -703,7 +703,7 @@ func NewWatcher(port int) error {
fmt.Println(time.Now().Format(layout))
//TODO here
- // utils.CheckErr(buildSite(true))
+ // utils.CheckErr(buildSite(true))
rebuildSite(dynamicFilesChanged)
if !BuildWatch && !viper.GetBool("DisableLiveReload") {
DIR diff --git a/docs/content/meta/roadmap.md b/docs/content/meta/roadmap.md
@@ -19,7 +19,6 @@ In no particular order, here is what we are working on:
* Import from other website systems
* from Drupal (See https://bitbucket.org/rickb777/drupal2hugo by Rick Beton (@rickb777))
* from WordPress (See [#100][], especially https://github.com/SchumacherFM/wordpress-to-hugo-exporter by Cyrill Schumacher (@SchumacherFM), but volunteers are needed to make it work with latest versions of WordPress.)
- * from Jekyll (See [#101][])
* An interactive web based editor (See http://discuss.gohugo.io/t/web-based-editor/155)
* Additional [themes](https://github.com/spf13/hugoThemes) (always on-going, contributions welcome!)
* Dynamic image resizing via shortcodes
DIR diff --git a/hugolib/handler_page.go b/hugolib/handler_page.go
@@ -32,6 +32,7 @@ type basicPageHandler Handle
func (b basicPageHandler) Read(f *source.File, s *Site) HandledResult {
page, err := NewPage(f.Path())
+
if err != nil {
return HandledResult{file: f, err: err}
}
DIR diff --git a/hugolib/page.go b/hugolib/page.go
@@ -48,20 +48,19 @@ var (
)
type Page struct {
- Params map[string]interface{}
- Content template.HTML
- Summary template.HTML
- Aliases []string
- Status string
- Images []Image
- Videos []Video
- TableOfContents template.HTML
- Truncated bool
- Draft bool
- PublishDate time.Time
- Tmpl tpl.Template
- Markup string
-
+ Params map[string]interface{}
+ Content template.HTML
+ Summary template.HTML
+ Aliases []string
+ Status string
+ Images []Image
+ Videos []Video
+ TableOfContents template.HTML
+ Truncated bool
+ Draft bool
+ PublishDate time.Time
+ Tmpl tpl.Template
+ Markup string
extension string
contentType string
renderable bool
@@ -77,13 +76,13 @@ type Page struct {
plainSecondaryInit sync.Once
renderingConfig *helpers.Blackfriday
renderingConfigInit sync.Once
+ pageMenus PageMenus
+ pageMenusInit sync.Once
+ isCJKLanguage bool
PageMeta
Source
Position `json:"-"`
Node
- pageMenus PageMenus
- pageMenusInit sync.Once
- isCJKLanguage bool
}
type Source struct {
@@ -106,6 +105,42 @@ type Position struct {
}
type Pages []*Page
+//
+//func (ps Pages) Replace(page *Page) {
+// if i := ps.FindPagePos(page); i >= 0 {
+// ps[i] = page
+// }
+//}
+
+//func (ps Pages) FindPageByFilePath(inPath string) *Page {
+// for _, x := range ps {
+// if x.Source.LogicalName() == inPath {
+// return x
+// }
+// }
+// return nil
+//}
+
+// FindPagePos Given a page, it will find the position in Pages
+// will return -1 if not found
+func (ps Pages) FindPagePos(page *Page) int {
+ for i, x := range ps {
+ if x.Source.LogicalName() == page.Source.LogicalName() {
+ return i
+ }
+ }
+ return -1
+}
+
+// FindPage Given a page, it will return the page in Pages
+// will return nil if not found
+//func (ps Pages) FindPage(page *Page) *Page {
+// if i := ps.FindPagePos(page); i >= 0 {
+// return ps[i]
+// }
+//
+// return nil
+//}
func (p *Page) Plain() string {
p.initPlain()
DIR diff --git a/hugolib/site.go b/hugolib/site.go
@@ -451,7 +451,6 @@ func (s *Site) ReBuild(changed map[string]bool) error {
}
}
-
if len(tmplChanged) > 0 {
s.prepTemplates()
s.Tmpl.PrintErrors()
@@ -462,10 +461,41 @@ func (s *Site) ReBuild(changed map[string]bool) error {
s.ReadDataFromSourceFS()
}
- if len (sourceChanged) > 0 {
- if err = s.CreatePages(); err != nil {
- return err
+ if len(sourceChanged) > 0 {
+
+ results := make(chan HandledResult)
+ filechan := make(chan *source.File)
+ errs := make(chan error)
+ wg := &sync.WaitGroup{}
+
+ wg.Add(2)
+ for i := 0; i < 2; i++ {
+ go sourceReader(s, filechan, results, wg)
+ }
+
+ go incrementalReadCollator(s, results, errs)
+
+ for _, x := range sourceChanged {
+ file, err := s.ReReadFile(x)
+ if err != nil {
+ errs <- err
+ }
+
+ filechan <- file
}
+
+ close(filechan)
+ wg.Wait()
+ close(results)
+
+ s.timerStep("read pages from source")
+
+ //renderErrs := <-s.ConvertSource()
+ s.timerStep("convert source")
+ // TODO(spf13) port this
+
+ fmt.Errorf("%s", errs)
+
s.setupPrevNext()
if err = s.BuildSiteMeta(); err != nil {
return err
@@ -497,7 +527,6 @@ func (s *Site) ReBuild(changed map[string]bool) error {
return nil
}
-
func (s *Site) Analyze() error {
if err := s.Process(); err != nil {
return err
@@ -764,6 +793,47 @@ type pageResult struct {
err error
}
+// ReReadFile resets file to be read from disk again
+func (s *Site) ReReadFile(absFilePath string) (*source.File, error) {
+ fmt.Println("rereading", absFilePath)
+ var file *source.File
+
+ reader, err := source.NewLazyFileReader(absFilePath)
+ if err != nil {
+ return nil, err
+ }
+ fmt.Println(s.absDataDir())
+
+ file, err = source.NewFileFromAbs(s.absContentDir(), absFilePath, reader)
+
+ fmt.Println("file created", file.Path())
+
+ if err != nil {
+ return nil, err
+ }
+
+ // maybe none of this rest needs to be here.
+ // replaced := false
+
+ // fmt.Println(len(s.Files))
+
+ // for i, x := range s.Files {
+ // fmt.Println(x)
+ // fmt.Println("*** COMPARING:")
+ // fmt.Println(" ", x.LogicalName())
+ // fmt.Println(" ", absFilePath)
+ // if x.LogicalName() == absFilePath {
+ // s.Files[i] = file
+ // replaced = true
+ // }
+ // }
+
+ // if !replaced {
+ // s.Files = append(s.Files, file)
+ // }
+ return file, nil
+}
+
func (s *Site) ReadPagesFromSource() chan error {
if s.Source == nil {
panic(fmt.Sprintf("s.Source not set %s", s.absContentDir()))
@@ -856,12 +926,17 @@ func (s *Site) CreatePages() error {
func sourceReader(s *Site, files <-chan *source.File, results chan<- HandledResult, wg *sync.WaitGroup) {
defer wg.Done()
for file := range files {
- h := NewMetaHandler(file.Extension())
- if h != nil {
- h.Read(file, s, results)
- } else {
- jww.ERROR.Println("Unsupported File Type", file.Path())
- }
+ fmt.Println("reading", file.Path())
+ readSourceFile(s, file, results)
+ }
+}
+
+func readSourceFile(s *Site, file *source.File, results chan<- HandledResult) {
+ h := NewMetaHandler(file.Extension())
+ if h != nil {
+ h.Read(file, s, results)
+ } else {
+ jww.ERROR.Println("Unsupported File Type", file.Path())
}
}
@@ -905,7 +980,41 @@ func converterCollator(s *Site, results <-chan HandledResult, errs chan<- error)
errs <- fmt.Errorf("Errors rendering pages: %s", strings.Join(errMsgs, "\n"))
}
-func readCollator(s *Site, results <-chan HandledResult, errs chan<- error) {
+func (s *Site) AddPage(page *Page) {
+ if page.ShouldBuild() {
+ s.Pages = append(s.Pages, page)
+ }
+
+ if page.IsDraft() {
+ s.draftCount++
+ }
+
+ if page.IsFuture() {
+ s.futureCount++
+ }
+}
+
+func (s *Site) RemovePage(page *Page) {
+ if i := s.Pages.FindPagePos(page); i >= 0 {
+ if page.IsDraft() {
+ s.draftCount--
+ }
+
+ if page.IsFuture() {
+ s.futureCount--
+ }
+
+ s.Pages = append(s.Pages[:i], s.Pages[i+1:]...)
+ }
+}
+
+func (s *Site) ReplacePage(page *Page) {
+ // will find existing page that matches filepath and remove it
+ s.RemovePage(page)
+ s.AddPage(page)
+}
+
+func incrementalReadCollator(s *Site, results <-chan HandledResult, errs chan<- error) {
errMsgs := []string{}
for r := range results {
if r.err != nil {
@@ -915,19 +1024,34 @@ func readCollator(s *Site, results <-chan HandledResult, errs chan<- error) {
// !page == file
if r.page == nil {
+ // TODO(spf13): Make this incremental as well
s.Files = append(s.Files, r.file)
} else {
- if r.page.ShouldBuild() {
- s.Pages = append(s.Pages, r.page)
- }
+ s.ReplacePage(r.page)
+ }
+ }
- if r.page.IsDraft() {
- s.draftCount++
- }
+ s.Pages.Sort()
+ if len(errMsgs) == 0 {
+ errs <- nil
+ return
+ }
+ errs <- fmt.Errorf("Errors reading pages: %s", strings.Join(errMsgs, "\n"))
+}
- if r.page.IsFuture() {
- s.futureCount++
- }
+func readCollator(s *Site, results <-chan HandledResult, errs chan<- error) {
+ errMsgs := []string{}
+ for r := range results {
+ if r.err != nil {
+ errMsgs = append(errMsgs, r.Error())
+ continue
+ }
+
+ // !page == file
+ if r.page == nil {
+ s.Files = append(s.Files, r.file)
+ } else {
+ s.AddPage(r.page)
}
}
DIR diff --git a/source/file.go b/source/file.go
@@ -14,14 +14,16 @@
package source
import (
- "github.com/spf13/hugo/helpers"
"io"
"path/filepath"
"strings"
+
+ "github.com/spf13/hugo/helpers"
)
+// All paths are relative from the source directory base
type File struct {
- relpath string // Original Full Path eg. /Users/Home/Hugo/foo.txt
+ relpath string // Original Full Path eg. content/foo.txt
logicalName string // foo.txt
Contents io.Reader
section string // The first directory
@@ -30,6 +32,7 @@ type File struct {
uniqueID string // MD5 of the filename
}
+// UniqueID: MD5 of the filename
func (f *File) UniqueID() string {
return f.uniqueID
}
@@ -42,15 +45,17 @@ func (f *File) Bytes() []byte {
return helpers.ReaderToBytes(f.Contents)
}
-// Filename without extension
+// BaseFileName Filename without extension
func (f *File) BaseFileName() string {
return helpers.Filename(f.LogicalName())
}
+// Section The first directory
func (f *File) Section() string {
return f.section
}
+// LogicalName The filename and extension of the file
func (f *File) LogicalName() string {
return f.logicalName
}
@@ -71,6 +76,7 @@ func (f *File) Ext() string {
return f.Extension()
}
+// Path the relative path including file name and extension from the base of the source directory
func (f *File) Path() string {
return f.relpath
}
DIR diff --git a/source/filesystem.go b/source/filesystem.go
@@ -14,13 +14,14 @@
package source
import (
- "github.com/spf13/viper"
"io"
"os"
"path/filepath"
"regexp"
"strings"
+ "github.com/spf13/viper"
+
"github.com/spf13/hugo/helpers"
jww "github.com/spf13/jwalterweatherman"
)
@@ -59,14 +60,11 @@ func (f *Filesystem) Files() []*File {
return f.files
}
+// add populates a file in the Filesystem.files
func (f *Filesystem) add(name string, reader io.Reader) (err error) {
var file *File
- //if f.Base == "" {
- //file = NewFileWithContents(name, reader)
- //} else {
file, err = NewFileFromAbs(f.Base, name, reader)
- //}
if err == nil {
f.files = append(f.files, file)
@@ -79,48 +77,57 @@ func (f *Filesystem) getRelativePath(name string) (final string, err error) {
}
func (f *Filesystem) captureFiles() {
-
walker := func(filePath string, fi os.FileInfo, err error) error {
if err != nil {
return nil
}
- if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
- link, err := filepath.EvalSymlinks(filePath)
- if err != nil {
- jww.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", filePath, err)
- return nil
- }
- linkfi, err := os.Stat(link)
+ b, err := f.shouldRead(filePath, fi)
+ if err != nil {
+ return err
+ }
+ if b {
+ rd, err := NewLazyFileReader(filePath)
if err != nil {
- jww.ERROR.Printf("Cannot stat '%s', error was: %s", link, err)
- return nil
- }
- if !linkfi.Mode().IsRegular() {
- jww.ERROR.Printf("Symbolic links for directories not supported, skipping '%s'", filePath)
+ return err
}
- return nil
+ f.add(filePath, rd)
}
+ return err
+ }
- if fi.IsDir() {
- if f.avoid(filePath) || isNonProcessablePath(filePath) {
- return filepath.SkipDir
- }
- return nil
- }
+ filepath.Walk(f.Base, walker)
+}
- if isNonProcessablePath(filePath) {
- return nil
+func (f *Filesystem) shouldRead(filePath string, fi os.FileInfo) (bool, error) {
+ if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
+ link, err := filepath.EvalSymlinks(filePath)
+ if err != nil {
+ jww.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", filePath, err)
+ return false, nil
}
- rd, err := NewLazyFileReader(filePath)
+ linkfi, err := os.Stat(link)
if err != nil {
- return err
+ jww.ERROR.Printf("Cannot stat '%s', error was: %s", link, err)
+ return false, nil
}
- f.add(filePath, rd)
- return nil
+ if !linkfi.Mode().IsRegular() {
+ jww.ERROR.Printf("Symbolic links for directories not supported, skipping '%s'", filePath)
+ }
+ return false, nil
}
- filepath.Walk(f.Base, walker)
+ if fi.IsDir() {
+ if f.avoid(filePath) || isNonProcessablePath(filePath) {
+ return false, filepath.SkipDir
+ }
+ return false, nil
+ }
+
+ if isNonProcessablePath(filePath) {
+ return false, nil
+ }
+ return true, nil
}
func (f *Filesystem) avoid(filePath string) bool {