Handle remove & rename source operations incrementally - hugo - [fork] hugo port for 9front
HTML git clone git@git.drkhsh.at/hugo.git
DIR Log
DIR Files
DIR Refs
DIR Submodules
DIR README
DIR LICENSE
---
DIR commit 7e196a82944148ed3f78f334303b452ab2bd4078
DIR parent e01c340915f971e86fef9a1c013a3183c50e51fc
HTML Author: Steve Francia <steve.francia@gmail.com>
Date: Mon, 11 Jan 2016 12:06:52 -0500
Handle remove & rename source operations incrementally
Diffstat:
M commands/hugo.go | 95 ++++++++++++++++---------------
M hugolib/page.go | 18 +++++++++---------
M hugolib/site.go | 59 ++++++++++++++++++++++---------
3 files changed, 101 insertions(+), 71 deletions(-)
---
DIR diff --git a/commands/hugo.go b/commands/hugo.go
@@ -546,9 +546,9 @@ func buildSite(watching ...bool) (err error) {
return nil
}
-func rebuildSite(changes map[string]bool) error {
+func rebuildSite(events []fsnotify.Event) error {
startTime := time.Now()
- err := mainSite.ReBuild(changes)
+ err := mainSite.ReBuild(events)
if err != nil {
return err
}
@@ -585,12 +585,10 @@ func NewWatcher(port int) error {
for {
select {
case evs := <-watcher.Events:
- jww.INFO.Println("File System Event:", evs)
+ jww.INFO.Println("Recieved System Events:", evs)
- staticChanged := false
- dynamicChanged := false
- staticFilesChanged := make(map[string]bool)
- dynamicFilesChanged := make(map[string]bool)
+ staticEvents := []fsnotify.Event{} //ev make(map[string]bool)
+ dynamicEvents := []fsnotify.Event{} //make(map[string]bool)
for _, ev := range evs {
ext := filepath.Ext(ev.Name)
@@ -598,10 +596,6 @@ func NewWatcher(port int) error {
if istemp {
continue
}
- // renames are always followed with Create/Modify
- if ev.Op&fsnotify.Rename == fsnotify.Rename {
- continue
- }
// Write and rename operations are often followed by CHMOD.
// There may be valid use cases for rebuilding the site on CHMOD,
@@ -615,27 +609,24 @@ func NewWatcher(port int) error {
continue
}
- isstatic := strings.HasPrefix(ev.Name, helpers.GetStaticDirPath()) || (len(helpers.GetThemesDirPath()) > 0 && strings.HasPrefix(ev.Name, helpers.GetThemesDirPath()))
- staticChanged = staticChanged || isstatic
- dynamicChanged = dynamicChanged || !isstatic
-
- if isstatic {
- if staticPath, err := helpers.MakeStaticPathRelative(ev.Name); err == nil {
- staticFilesChanged[staticPath] = true
- }
- } else {
- dynamicFilesChanged[ev.Name] = true
- }
-
// add new directory to watch list
if s, err := os.Stat(ev.Name); err == nil && s.Mode().IsDir() {
if ev.Op&fsnotify.Create == fsnotify.Create {
watcher.Add(ev.Name)
}
}
+
+ isstatic := strings.HasPrefix(ev.Name, helpers.GetStaticDirPath()) || (len(helpers.GetThemesDirPath()) > 0 && strings.HasPrefix(ev.Name, helpers.GetThemesDirPath()))
+
+ if isstatic {
+ staticEvents = append(staticEvents, ev)
+// }
+ } else {
+ dynamicEvents = append(dynamicEvents, ev)
+ }
}
- if staticChanged {
+ if len(staticEvents) > 0 {
jww.FEEDBACK.Printf("Static file changed, syncing\n")
if viper.GetBool("ForceSyncStatic") {
jww.FEEDBACK.Printf("Syncing all static files\n")
@@ -645,7 +636,6 @@ func NewWatcher(port int) error {
utils.StopOnErr(err, fmt.Sprintf("Error copying static files to %s", helpers.AbsPathify(viper.GetString("PublishDir"))))
}
} else {
-
syncer := fsync.NewSyncer()
syncer.NoTimes = viper.GetBool("notimes")
syncer.SrcFs = hugofs.SourceFs
@@ -660,24 +650,40 @@ func NewWatcher(port int) error {
staticDir := helpers.GetStaticDirPath()
themeStaticDir := helpers.GetThemesDirPath()
- jww.FEEDBACK.Printf("StaticDir '%s'\nThemeStaticDir '%s'\n", staticDir, themeStaticDir)
+ jww.FEEDBACK.Printf("Syncing from: \n \tStaticDir: '%s'\n\tThemeStaticDir: '%s'\n", staticDir, themeStaticDir)
- for path := range staticFilesChanged {
+ for _, ev := range staticEvents {
+ fmt.Println(ev)
+ fromPath := ev.Name
var publishPath string
- if strings.HasPrefix(path, staticDir) {
- publishPath = filepath.Join(publishDir, strings.TrimPrefix(path, staticDir))
- } else if strings.HasPrefix(path, themeStaticDir) {
- publishPath = filepath.Join(publishDir, strings.TrimPrefix(path, themeStaticDir))
+ // If we are here we already know the event took place in a static dir
+ relPath, err := helpers.MakeStaticPathRelative(fromPath)
+ if err != nil {
+ fmt.Println(err)
+ continue
}
- jww.FEEDBACK.Printf("Syncing file '%s'", path)
-
- if _, err := os.Stat(path); err == nil {
- jww.INFO.Println("syncing from ", path, " to ", publishPath)
- err := syncer.Sync(publishPath, path)
- if err != nil {
- jww.FEEDBACK.Printf("Error on syncing file '%s'\n", path)
- }
+
+ if strings.HasPrefix(fromPath, staticDir) {
+ publishPath = filepath.Join(publishDir, strings.TrimPrefix(fromPath, staticDir))
+ } else if strings.HasPrefix(relPath, themeStaticDir) {
+ publishPath = filepath.Join(publishDir, strings.TrimPrefix(fromPath, themeStaticDir))
+ }
+ jww.FEEDBACK.Println("Syncing file", relPath)
+
+ // Due to our approach of layering many directories onto one we can't accurately
+ // remove file not in one of the source directories.
+ // If a file is in the local static dir and also in the theme static dir and we remove
+ // it from one of those locations we expect it to still exist in the destination
+
+ // if remove or rename ignore
+ if ev.Op&fsnotify.Rename == fsnotify.Rename || ev.Op&fsnotify.Remove == fsnotify.Remove {
+ continue
+ }
+
+ jww.INFO.Println("syncing from ", fromPath, " to ", publishPath)
+ if er := syncer.Sync(publishPath, fromPath); er != nil {
+ jww.ERROR.Printf("Error on syncing file '%s'\n %s\n", relPath, er)
}
}
}
@@ -686,8 +692,9 @@ func NewWatcher(port int) error {
// Will block forever trying to write to a channel that nobody is reading if livereload isn't initalized
// force refresh when more than one file
- if len(staticFilesChanged) == 1 {
- for path := range staticFilesChanged {
+ if len(staticEvents) == 1 {
+ for _, ev := range staticEvents {
+ path, _ := helpers.MakeStaticPathRelative(ev.Name)
livereload.RefreshPath(path)
}
@@ -697,14 +704,12 @@ func NewWatcher(port int) error {
}
}
- if dynamicChanged {
+ if len(dynamicEvents) >0 {
fmt.Print("\nChange detected, rebuilding site\n")
const layout = "2006-01-02 15:04 -0700"
fmt.Println(time.Now().Format(layout))
- //TODO here
- // utils.CheckErr(buildSite(true))
- rebuildSite(dynamicFilesChanged)
+ rebuildSite(dynamicEvents)
if !BuildWatch && !viper.GetBool("DisableLiveReload") {
// Will block forever trying to write to a channel that nobody is reading if livereload isn't initalized
DIR diff --git a/hugolib/page.go b/hugolib/page.go
@@ -112,20 +112,20 @@ type Pages []*Page
// }
//}
-//func (ps Pages) FindPageByFilePath(inPath string) *Page {
-// for _, x := range ps {
-// if x.Source.LogicalName() == inPath {
-// return x
-// }
-// }
-// return nil
-//}
+func (ps Pages) FindPagePosByFilePath(inPath string) int {
+ for i, x := range ps {
+ if x.Source.Path() == inPath {
+ return i
+ }
+ }
+ return -1
+}
// FindPagePos Given a page, it will find the position in Pages
// will return -1 if not found
func (ps Pages) FindPagePos(page *Page) int {
for i, x := range ps {
- if x.Source.LogicalName() == page.Source.LogicalName() {
+ if x.Source.Path() == page.Source.Path() {
return i
}
}
DIR diff --git a/hugolib/site.go b/hugolib/site.go
@@ -43,6 +43,7 @@ import (
jww "github.com/spf13/jwalterweatherman"
"github.com/spf13/nitro"
"github.com/spf13/viper"
+ "gopkg.in/fsnotify.v1"
)
var _ = transform.AbsURL
@@ -426,28 +427,29 @@ func (s *Site) Build() (err error) {
return nil
}
-func (s *Site) ReBuild(changed map[string]bool) error {
+func (s *Site) ReBuild(events []fsnotify.Event) error {
s.timerStep("initialize rebuild")
// First we need to determine what changed
- sourceChanged := []string{}
- tmplChanged := []string{}
- dataChanged := []string{}
+ sourceChanged := []fsnotify.Event{}
+ tmplChanged := []fsnotify.Event{}
+ dataChanged := []fsnotify.Event{}
+
var err error
- for f := range changed {
+ for _, ev := range events {
// Need to re-read source
- if strings.HasPrefix(f, s.absContentDir()) {
- fmt.Println("Source changed", f)
- sourceChanged = append(sourceChanged, f)
+ if strings.HasPrefix(ev.Name, s.absContentDir()) {
+ fmt.Println("Source changed", ev)
+ sourceChanged = append(sourceChanged, ev)
}
- if strings.HasPrefix(f, s.absLayoutDir()) || strings.HasPrefix(f, s.absThemeDir()) {
- fmt.Println("Template changed", f)
- tmplChanged = append(tmplChanged, f)
+ if strings.HasPrefix(ev.Name, s.absLayoutDir()) || strings.HasPrefix(ev.Name, s.absThemeDir()) {
+ fmt.Println("Template changed", ev)
+ tmplChanged = append(tmplChanged, ev)
}
- if strings.HasPrefix(f, s.absDataDir()) {
- fmt.Println("Data changed", f)
- dataChanged = append(dataChanged, f)
+ if strings.HasPrefix(ev.Name, s.absDataDir()) {
+ fmt.Println("Data changed", ev)
+ dataChanged = append(dataChanged,ev)
}
}
@@ -497,8 +499,15 @@ func (s *Site) ReBuild(changed map[string]bool) error {
go incrementalReadCollator(s, readResults, pageChan, fileConvChan, coordinator, errs)
go converterCollator(s, convertResults, errs)
- for _, x := range sourceChanged {
- file, err := s.ReReadFile(x)
+ for _, ev := range sourceChanged {
+ if ev.Op&fsnotify.Rename == fsnotify.Rename || ev.Op&fsnotify.Remove == fsnotify.Remove {
+ //remove the file & a create will follow
+ path, _ := helpers.GetRelativePath(ev.Name, s.absContentDir())
+ s.RemovePageByPath(path)
+ continue
+ }
+
+ file, err := s.ReReadFile(ev.Name)
if err != nil {
errs <- err
}
@@ -540,7 +549,6 @@ func (s *Site) ReBuild(changed map[string]bool) error {
if err = s.Render(); err != nil {
// Better reporting when the template is missing (commit 2bbecc7b)
jww.ERROR.Printf("Error rendering site: %s", err)
-
jww.ERROR.Printf("Available templates:")
var keys []string
for _, template := range s.Tmpl.Templates() {
@@ -1005,6 +1013,23 @@ func (s *Site) AddPage(page *Page) {
}
}
+
+func (s *Site) RemovePageByPath(path string) {
+ if i := s.Pages.FindPagePosByFilePath(path); i >= 0 {
+ page := s.Pages[i]
+
+ if page.IsDraft() {
+ s.draftCount--
+ }
+
+ if page.IsFuture() {
+ s.futureCount--
+ }
+
+ s.Pages = append(s.Pages[:i], s.Pages[i+1:]...)
+ }
+}
+
func (s *Site) RemovePage(page *Page) {
if i := s.Pages.FindPagePos(page); i >= 0 {
if page.IsDraft() {