tpl/data: Clean up data namespace - hugo - [fork] hugo port for 9front
HTML git clone git@git.drkhsh.at/hugo.git
DIR Log
DIR Files
DIR Refs
DIR Submodules
DIR README
DIR LICENSE
---
DIR commit 08c0de5cc37cd4e512268b8f72ec5a6c68cd5754
DIR parent 1cf2f3dc4fa81503485a73db21bfda6e965dee15
HTML Author: Cameron Moore <moorereason@gmail.com>
Date: Mon, 1 May 2017 22:41:08 -0500
tpl/data: Clean up data namespace
- Move the main GetCSV and GetJSON into data.go.
- Add error returns to GetCSV and GetJSON.
- Add http client to Namespace for test mocking.
- Send accept headers on remote requests. Fixes #3395
- Return an error on non-2XX HTTP response codes and don't retry.
- Move cache tests to cache_test.go.
Diffstat:
A tpl/data/cache_test.go | 63 +++++++++++++++++++++++++++++++
M tpl/data/data.go | 114 ++++++++++++++++++++++++++++++-
A tpl/data/data_test.go | 251 +++++++++++++++++++++++++++++++
M tpl/data/resources.go | 122 ++++++-------------------------
M tpl/data/resources_test.go | 234 ++++---------------------------
5 files changed, 475 insertions(+), 309 deletions(-)
---
DIR diff --git a/tpl/data/cache_test.go b/tpl/data/cache_test.go
@@ -0,0 +1,63 @@
+// Copyright 2017 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package data
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/spf13/afero"
+ "github.com/spf13/viper"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestCache(t *testing.T) {
+ t.Parallel()
+
+ fs := new(afero.MemMapFs)
+
+ for i, test := range []struct {
+ path string
+ content []byte
+ ignore bool
+ }{
+ {"http://Foo.Bar/foo_Bar-Foo", []byte(`T€st Content 123`), false},
+ {"fOO,bar:foo%bAR", []byte(`T€st Content 123 fOO,bar:foo%bAR`), false},
+ {"FOo/BaR.html", []byte(`FOo/BaR.html T€st Content 123`), false},
+ {"трям/трям", []byte(`T€st трям/трям Content 123`), false},
+ {"은행", []byte(`T€st C은행ontent 123`), false},
+ {"Банковский кассир", []byte(`Банковский кассир T€st Content 123`), false},
+ {"Банковский кассир", []byte(`Банковский кассир T€st Content 456`), true},
+ } {
+ msg := fmt.Sprintf("Test #%d: %v", i, test)
+
+ cfg := viper.New()
+
+ c, err := getCache(test.path, fs, cfg, test.ignore)
+ assert.NoError(t, err, msg)
+ assert.Nil(t, c, msg)
+
+ err = writeCache(test.path, test.content, fs, cfg, test.ignore)
+ assert.NoError(t, err, msg)
+
+ c, err = getCache(test.path, fs, cfg, test.ignore)
+ assert.NoError(t, err, msg)
+
+ if test.ignore {
+ assert.Nil(t, c, msg)
+ } else {
+ assert.Equal(t, string(test.content), string(c))
+ }
+ }
+}
DIR diff --git a/tpl/data/data.go b/tpl/data/data.go
@@ -13,16 +13,126 @@
package data
-import "github.com/spf13/hugo/deps"
+import (
+ "bytes"
+ "encoding/csv"
+ "encoding/json"
+ "errors"
+ "net/http"
+ "strings"
+ "time"
+
+ "github.com/spf13/hugo/deps"
+ jww "github.com/spf13/jwalterweatherman"
+)
// New returns a new instance of the data-namespaced template functions.
func New(deps *deps.Deps) *Namespace {
return &Namespace{
- deps: deps,
+ deps: deps,
+ client: http.DefaultClient,
}
}
// Namespace provides template functions for the "data" namespace.
type Namespace struct {
deps *deps.Deps
+
+ client *http.Client
+}
+
+// GetCSV expects a data separator and one or n-parts of a URL to a resource which
+// can either be a local or a remote one.
+// The data separator can be a comma, semi-colon, pipe, etc, but only one character.
+// If you provide multiple parts for the URL they will be joined together to the final URL.
+// GetCSV returns nil or a slice slice to use in a short code.
+func (ns *Namespace) GetCSV(sep string, urlParts ...string) (d [][]string, err error) {
+ url := strings.Join(urlParts, "")
+
+ var clearCacheSleep = func(i int, u string) {
+ jww.ERROR.Printf("Retry #%d for %s and sleeping for %s", i, url, resSleep)
+ time.Sleep(resSleep)
+ deleteCache(url, ns.deps.Fs.Source, ns.deps.Cfg)
+ }
+
+ for i := 0; i <= resRetries; i++ {
+ var req *http.Request
+ req, err = http.NewRequest("GET", url, nil)
+ if err != nil {
+ jww.ERROR.Printf("Failed to create request for getJSON: %s", err)
+ return nil, err
+ }
+
+ req.Header.Add("Accept", "text/csv")
+ req.Header.Add("Accept", "text/plain")
+
+ var c []byte
+ c, err = ns.getResource(req)
+ if err != nil {
+ jww.ERROR.Printf("Failed to read csv resource %q with error message %s", url, err)
+ return nil, err
+ }
+
+ if !bytes.Contains(c, []byte(sep)) {
+ err = errors.New("Cannot find separator " + sep + " in CSV.")
+ return
+ }
+
+ if d, err = parseCSV(c, sep); err != nil {
+ jww.ERROR.Printf("Failed to parse csv file %s with error message %s", url, err)
+ clearCacheSleep(i, url)
+ continue
+ }
+ break
+ }
+ return
+}
+
+// GetJSON expects one or n-parts of a URL to a resource which can either be a local or a remote one.
+// If you provide multiple parts they will be joined together to the final URL.
+// GetJSON returns nil or parsed JSON to use in a short code.
+func (ns *Namespace) GetJSON(urlParts ...string) (v interface{}, err error) {
+ url := strings.Join(urlParts, "")
+
+ for i := 0; i <= resRetries; i++ {
+ var req *http.Request
+ req, err = http.NewRequest("GET", url, nil)
+ if err != nil {
+ jww.ERROR.Printf("Failed to create request for getJSON: %s", err)
+ return nil, err
+ }
+
+ req.Header.Add("Accept", "application/json")
+
+ var c []byte
+ c, err = ns.getResource(req)
+ if err != nil {
+ jww.ERROR.Printf("Failed to get json resource %s with error message %s", url, err)
+ return nil, err
+ }
+
+ err = json.Unmarshal(c, &v)
+ if err != nil {
+ jww.ERROR.Printf("Cannot read json from resource %s with error message %s", url, err)
+ jww.ERROR.Printf("Retry #%d for %s and sleeping for %s", i, url, resSleep)
+ time.Sleep(resSleep)
+ deleteCache(url, ns.deps.Fs.Source, ns.deps.Cfg)
+ continue
+ }
+ break
+ }
+ return
+}
+
+// parseCSV parses bytes of CSV data into a slice slice string or an error
+func parseCSV(c []byte, sep string) ([][]string, error) {
+ if len(sep) != 1 {
+ return nil, errors.New("Incorrect length of csv separator: " + sep)
+ }
+ b := bytes.NewReader(c)
+ r := csv.NewReader(b)
+ rSep := []rune(sep)
+ r.Comma = rSep[0]
+ r.FieldsPerRecord = 0
+ return r.ReadAll()
}
DIR diff --git a/tpl/data/data_test.go b/tpl/data/data_test.go
@@ -0,0 +1,251 @@
+// Copyright 2017 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package data
+
+import (
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/spf13/viper"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestGetCSV(t *testing.T) {
+ t.Parallel()
+
+ ns := New(newDeps(viper.New()))
+
+ for i, test := range []struct {
+ sep string
+ url string
+ content string
+ expect interface{}
+ }{
+ // Remotes
+ {
+ ",",
+ `http://success/`,
+ "gomeetup,city\nyes,Sydney\nyes,San Francisco\nyes,Stockholm\n",
+ [][]string{{"gomeetup", "city"}, {"yes", "Sydney"}, {"yes", "San Francisco"}, {"yes", "Stockholm"}},
+ },
+ {
+ ",",
+ `http://error.extra.field/`,
+ "gomeetup,city\nyes,Sydney\nyes,San Francisco\nyes,Stockholm,EXTRA\n",
+ false,
+ },
+ {
+ ",",
+ `http://error.no.sep/`,
+ "gomeetup;city\nyes;Sydney\nyes;San Francisco\nyes;Stockholm\n",
+ false,
+ },
+ {
+ ",",
+ `http://nofound/404`,
+ ``,
+ false,
+ },
+
+ // Locals
+ {
+ ";",
+ "pass/semi",
+ "gomeetup;city\nyes;Sydney\nyes;San Francisco\nyes;Stockholm\n",
+ [][]string{{"gomeetup", "city"}, {"yes", "Sydney"}, {"yes", "San Francisco"}, {"yes", "Stockholm"}},
+ },
+ {
+ ";",
+ "fail/no-file",
+ "",
+ false,
+ },
+ } {
+ msg := fmt.Sprintf("Test %d", i)
+
+ // Setup HTTP test server
+ var srv *httptest.Server
+ srv, ns.client = getTestServer(func(w http.ResponseWriter, r *http.Request) {
+ if !haveHeader(r.Header, "Accept", "text/csv") && !haveHeader(r.Header, "Accept", "text/plain") {
+ http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
+ return
+ }
+
+ if r.URL.Path == "/404" {
+ http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
+ return
+ }
+
+ w.Header().Add("Content-type", "text/csv")
+
+ w.Write([]byte(test.content))
+ })
+ defer func() { srv.Close() }()
+
+ // Setup local test file for schema-less URLs
+ if !strings.Contains(test.url, ":") && !strings.HasPrefix(test.url, "fail/") {
+ f, err := ns.deps.Fs.Source.Create(filepath.Join(ns.deps.Cfg.GetString("workingDir"), test.url))
+ require.NoError(t, err, msg)
+ f.WriteString(test.content)
+ f.Close()
+ }
+
+ // Get on with it
+ got, err := ns.GetCSV(test.sep, test.url)
+
+ if _, ok := test.expect.(bool); ok {
+ assert.Error(t, err, msg)
+ continue
+ }
+ require.NoError(t, err, msg)
+ require.NotNil(t, got, msg)
+
+ assert.EqualValues(t, test.expect, got, msg)
+ }
+}
+
+func TestGetJSON(t *testing.T) {
+ t.Parallel()
+
+ ns := New(newDeps(viper.New()))
+
+ for i, test := range []struct {
+ url string
+ content string
+ expect interface{}
+ }{
+ {
+ `http://success/`,
+ `{"gomeetup":["Sydney","San Francisco","Stockholm"]}`,
+ map[string]interface{}{"gomeetup": []interface{}{"Sydney", "San Francisco", "Stockholm"}},
+ },
+ {
+ `http://malformed/`,
+ `{gomeetup:["Sydney","San Francisco","Stockholm"]}`,
+ false,
+ },
+ {
+ `http://nofound/404`,
+ ``,
+ false,
+ },
+ // Locals
+ {
+ "pass/semi",
+ `{"gomeetup":["Sydney","San Francisco","Stockholm"]}`,
+ map[string]interface{}{"gomeetup": []interface{}{"Sydney", "San Francisco", "Stockholm"}},
+ },
+ {
+ "fail/no-file",
+ "",
+ false,
+ },
+ } {
+ msg := fmt.Sprintf("Test %d", i)
+
+ // Setup HTTP test server
+ var srv *httptest.Server
+ srv, ns.client = getTestServer(func(w http.ResponseWriter, r *http.Request) {
+ if !haveHeader(r.Header, "Accept", "application/json") {
+ http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
+ return
+ }
+
+ if r.URL.Path == "/404" {
+ http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
+ return
+ }
+
+ w.Header().Add("Content-type", "application/json")
+
+ w.Write([]byte(test.content))
+ })
+ defer func() { srv.Close() }()
+
+ // Setup local test file for schema-less URLs
+ if !strings.Contains(test.url, ":") && !strings.HasPrefix(test.url, "fail/") {
+ f, err := ns.deps.Fs.Source.Create(filepath.Join(ns.deps.Cfg.GetString("workingDir"), test.url))
+ require.NoError(t, err, msg)
+ f.WriteString(test.content)
+ f.Close()
+ }
+
+ // Get on with it
+ got, err := ns.GetJSON(test.url)
+
+ if _, ok := test.expect.(bool); ok {
+ assert.Error(t, err, msg)
+ continue
+ }
+ require.NoError(t, err, msg)
+ require.NotNil(t, got, msg)
+
+ assert.EqualValues(t, test.expect, got, msg)
+ }
+}
+
+func TestParseCSV(t *testing.T) {
+ t.Parallel()
+
+ for i, test := range []struct {
+ csv []byte
+ sep string
+ exp string
+ err bool
+ }{
+ {[]byte("a,b,c\nd,e,f\n"), "", "", true},
+ {[]byte("a,b,c\nd,e,f\n"), "~/", "", true},
+ {[]byte("a,b,c\nd,e,f"), "|", "a,b,cd,e,f", false},
+ {[]byte("q,w,e\nd,e,f"), ",", "qwedef", false},
+ {[]byte("a|b|c\nd|e|f|g"), "|", "abcdefg", true},
+ {[]byte("z|y|c\nd|e|f"), "|", "zycdef", false},
+ } {
+ msg := fmt.Sprintf("Test %d: %v", i, test)
+
+ csv, err := parseCSV(test.csv, test.sep)
+ if test.err {
+ assert.Error(t, err, msg)
+ continue
+ }
+ require.NoError(t, err, msg)
+
+ act := ""
+ for _, v := range csv {
+ act = act + strings.Join(v, "")
+ }
+
+ assert.Equal(t, test.exp, act, msg)
+ }
+}
+
+func haveHeader(m http.Header, key, needle string) bool {
+ var s []string
+ var ok bool
+
+ if s, ok = m[key]; !ok {
+ return false
+ }
+
+ for _, v := range s {
+ if v == needle {
+ return true
+ }
+ }
+ return false
+}
DIR diff --git a/tpl/data/resources.go b/tpl/data/resources.go
@@ -14,14 +14,10 @@
package data
import (
- "bytes"
- "encoding/csv"
- "encoding/json"
- "errors"
+ "fmt"
"io/ioutil"
"net/http"
"path/filepath"
- "strings"
"sync"
"time"
@@ -67,14 +63,16 @@ func (l *remoteLock) URLUnlock(url string) {
}
// getRemote loads the content of a remote file. This method is thread safe.
-func getRemote(url string, fs afero.Fs, cfg config.Provider, hc *http.Client) ([]byte, error) {
+func getRemote(req *http.Request, fs afero.Fs, cfg config.Provider, hc *http.Client) ([]byte, error) {
+ url := req.URL.String()
+
c, err := getCache(url, fs, cfg, cfg.GetBool("ignoreCache"))
- if c != nil && err == nil {
- return c, nil
- }
if err != nil {
return nil, err
}
+ if c != nil {
+ return c, nil
+ }
// avoid race condition with locks, block other goroutines if the current url is processing
remoteURLLock.URLLock(url)
@@ -82,27 +80,34 @@ func getRemote(url string, fs afero.Fs, cfg config.Provider, hc *http.Client) ([
// avoid multiple locks due to calling getCache twice
c, err = getCache(url, fs, cfg, cfg.GetBool("ignoreCache"))
- if c != nil && err == nil {
- return c, nil
- }
if err != nil {
return nil, err
}
+ if c != nil {
+ return c, nil
+ }
jww.INFO.Printf("Downloading: %s ...", url)
- res, err := hc.Get(url)
+ res, err := hc.Do(req)
if err != nil {
return nil, err
}
+
+ if res.StatusCode < 200 || res.StatusCode > 299 {
+ return nil, fmt.Errorf("Failed to retrieve remote file: %s", http.StatusText(res.StatusCode))
+ }
+
c, err = ioutil.ReadAll(res.Body)
res.Body.Close()
if err != nil {
return nil, err
}
+
err = writeCache(url, c, fs, cfg, cfg.GetBool("ignoreCache"))
if err != nil {
return nil, err
}
+
jww.INFO.Printf("... and cached to: %s", getCacheFileID(cfg, url))
return c, nil
}
@@ -119,90 +124,11 @@ func getLocal(url string, fs afero.Fs, cfg config.Provider) ([]byte, error) {
}
// getResource loads the content of a local or remote file
-func (ns *Namespace) getResource(url string) ([]byte, error) {
- if url == "" {
- return nil, nil
- }
- if strings.Contains(url, "://") {
- return getRemote(url, ns.deps.Fs.Source, ns.deps.Cfg, http.DefaultClient)
- }
- return getLocal(url, ns.deps.Fs.Source, ns.deps.Cfg)
-}
-
-// GetJSON expects one or n-parts of a URL to a resource which can either be a local or a remote one.
-// If you provide multiple parts they will be joined together to the final URL.
-// GetJSON returns nil or parsed JSON to use in a short code.
-func (ns *Namespace) GetJSON(urlParts ...string) interface{} {
- var v interface{}
- url := strings.Join(urlParts, "")
-
- for i := 0; i <= resRetries; i++ {
- c, err := ns.getResource(url)
- if err != nil {
- jww.ERROR.Printf("Failed to get json resource %s with error message %s", url, err)
- return nil
- }
-
- err = json.Unmarshal(c, &v)
- if err != nil {
- jww.ERROR.Printf("Cannot read json from resource %s with error message %s", url, err)
- jww.ERROR.Printf("Retry #%d for %s and sleeping for %s", i, url, resSleep)
- time.Sleep(resSleep)
- deleteCache(url, ns.deps.Fs.Source, ns.deps.Cfg)
- continue
- }
- break
- }
- return v
-}
-
-// parseCSV parses bytes of CSV data into a slice slice string or an error
-func parseCSV(c []byte, sep string) ([][]string, error) {
- if len(sep) != 1 {
- return nil, errors.New("Incorrect length of csv separator: " + sep)
- }
- b := bytes.NewReader(c)
- r := csv.NewReader(b)
- rSep := []rune(sep)
- r.Comma = rSep[0]
- r.FieldsPerRecord = 0
- return r.ReadAll()
-}
-
-// GetCSV expects a data separator and one or n-parts of a URL to a resource which
-// can either be a local or a remote one.
-// The data separator can be a comma, semi-colon, pipe, etc, but only one character.
-// If you provide multiple parts for the URL they will be joined together to the final URL.
-// GetCSV returns nil or a slice slice to use in a short code.
-func (ns *Namespace) GetCSV(sep string, urlParts ...string) [][]string {
- var d [][]string
- url := strings.Join(urlParts, "")
-
- var clearCacheSleep = func(i int, u string) {
- jww.ERROR.Printf("Retry #%d for %s and sleeping for %s", i, url, resSleep)
- time.Sleep(resSleep)
- deleteCache(url, ns.deps.Fs.Source, ns.deps.Cfg)
- }
-
- for i := 0; i <= resRetries; i++ {
- c, err := ns.getResource(url)
-
- if err == nil && !bytes.Contains(c, []byte(sep)) {
- err = errors.New("Cannot find separator " + sep + " in CSV.")
- }
-
- if err != nil {
- jww.ERROR.Printf("Failed to read csv resource %s with error message %s", url, err)
- clearCacheSleep(i, url)
- continue
- }
-
- if d, err = parseCSV(c, sep); err != nil {
- jww.ERROR.Printf("Failed to parse csv file %s with error message %s", url, err)
- clearCacheSleep(i, url)
- continue
- }
- break
+func (ns *Namespace) getResource(req *http.Request) ([]byte, error) {
+ switch req.URL.Scheme {
+ case "":
+ return getLocal(req.URL.String(), ns.deps.Fs.Source, ns.deps.Cfg)
+ default:
+ return getRemote(req, ns.deps.Fs.Source, ns.deps.Cfg, ns.client)
}
- return d
}
DIR diff --git a/tpl/data/resources_test.go b/tpl/data/resources_test.go
@@ -19,7 +19,6 @@ import (
"net/http"
"net/http/httptest"
"net/url"
- "strings"
"sync"
"testing"
"time"
@@ -31,58 +30,9 @@ import (
"github.com/spf13/hugo/hugofs"
"github.com/spf13/viper"
"github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
)
-func TestScpCache(t *testing.T) {
- t.Parallel()
-
- tests := []struct {
- path string
- content []byte
- ignore bool
- }{
- {"http://Foo.Bar/foo_Bar-Foo", []byte(`T€st Content 123`), false},
- {"fOO,bar:foo%bAR", []byte(`T€st Content 123 fOO,bar:foo%bAR`), false},
- {"FOo/BaR.html", []byte(`FOo/BaR.html T€st Content 123`), false},
- {"трям/трям", []byte(`T€st трям/трям Content 123`), false},
- {"은행", []byte(`T€st C은행ontent 123`), false},
- {"Банковский кассир", []byte(`Банковский кассир T€st Content 123`), false},
- {"Банковский кассир", []byte(`Банковский кассир T€st Content 456`), true},
- }
-
- fs := new(afero.MemMapFs)
-
- for _, test := range tests {
- cfg := viper.New()
- c, err := getCache(test.path, fs, cfg, test.ignore)
- if err != nil {
- t.Errorf("Error getting cache: %s", err)
- }
- if c != nil {
- t.Errorf("There is content where there should not be anything: %s", string(c))
- }
-
- err = writeCache(test.path, test.content, fs, cfg, test.ignore)
- if err != nil {
- t.Errorf("Error writing cache: %s", err)
- }
-
- c, err = getCache(test.path, fs, cfg, test.ignore)
- if err != nil {
- t.Errorf("Error getting cache after writing: %s", err)
- }
- if test.ignore {
- if c != nil {
- t.Errorf("Cache ignored but content is not nil: %s", string(c))
- }
- } else {
- if !bytes.Equal(c, test.content) {
- t.Errorf("\nExpected: %s\nActual: %s\n", string(test.content), string(c))
- }
- }
- }
-}
-
func TestScpGetLocal(t *testing.T) {
t.Parallel()
v := viper.New()
@@ -146,6 +96,10 @@ func TestScpGetRemote(t *testing.T) {
}
for _, test := range tests {
+ msg := fmt.Sprintf("%v", test)
+
+ req, err := http.NewRequest("GET", test.path, nil)
+ require.NoError(t, err, msg)
srv, cl := getTestServer(func(w http.ResponseWriter, r *http.Request) {
w.Write(test.content)
@@ -154,41 +108,38 @@ func TestScpGetRemote(t *testing.T) {
cfg := viper.New()
- c, err := getRemote(test.path, fs, cfg, cl)
- if err != nil {
- t.Errorf("Error getting resource content: %s", err)
- }
- if !bytes.Equal(c, test.content) {
- t.Errorf("\nNet Expected: %s\nNet Actual: %s\n", string(test.content), string(c))
- }
- cc, cErr := getCache(test.path, fs, cfg, test.ignore)
- if cErr != nil {
- t.Error(cErr)
- }
+ c, err := getRemote(req, fs, cfg, cl)
+ require.NoError(t, err, msg)
+ assert.Equal(t, string(test.content), string(c))
+
+ c, err = getCache(req.URL.String(), fs, cfg, test.ignore)
+ require.NoError(t, err, msg)
+
if test.ignore {
- if cc != nil {
- t.Errorf("Cache ignored but content is not nil: %s", string(cc))
- }
+ assert.Empty(t, c, msg)
} else {
- if !bytes.Equal(cc, test.content) {
- t.Errorf("\nCache Expected: %s\nCache Actual: %s\n", string(test.content), string(cc))
- }
+ assert.Equal(t, string(test.content), string(c))
+
}
}
}
func TestScpGetRemoteParallel(t *testing.T) {
t.Parallel()
- fs := new(afero.MemMapFs)
+
+ ns := New(newDeps(viper.New()))
+
content := []byte(`T€st Content 123`)
- url := "http://Foo.Bar/foo_Bar-Foo"
srv, cl := getTestServer(func(w http.ResponseWriter, r *http.Request) {
w.Write(content)
})
defer func() { srv.Close() }()
- for _, ignoreCache := range []bool{false, true} {
+ url := "http://Foo.Bar/foo_Bar-Foo"
+ req, err := http.NewRequest("GET", url, nil)
+ require.NoError(t, err)
+ for _, ignoreCache := range []bool{false, true} {
cfg := viper.New()
cfg.Set("ignoreCache", ignoreCache)
@@ -199,13 +150,9 @@ func TestScpGetRemoteParallel(t *testing.T) {
go func(gor int) {
defer wg.Done()
for j := 0; j < 10; j++ {
- c, err := getRemote(url, fs, cfg, cl)
- if err != nil {
- t.Errorf("Error getting resource content: %s", err)
- }
- if !bytes.Equal(c, content) {
- t.Errorf("\nNet Expected: %s\nNet Actual: %s\n", string(content), string(c))
- }
+ c, err := getRemote(req, ns.deps.Fs.Source, ns.deps.Cfg, cl)
+ assert.NoError(t, err)
+ assert.Equal(t, string(content), string(c))
time.Sleep(23 * time.Millisecond)
}
@@ -214,137 +161,6 @@ func TestScpGetRemoteParallel(t *testing.T) {
wg.Wait()
}
-
- t.Log("Done!")
-}
-
-func TestParseCSV(t *testing.T) {
- t.Parallel()
-
- tests := []struct {
- csv []byte
- sep string
- exp string
- err bool
- }{
- {[]byte("a,b,c\nd,e,f\n"), "", "", true},
- {[]byte("a,b,c\nd,e,f\n"), "~/", "", true},
- {[]byte("a,b,c\nd,e,f"), "|", "a,b,cd,e,f", false},
- {[]byte("q,w,e\nd,e,f"), ",", "qwedef", false},
- {[]byte("a|b|c\nd|e|f|g"), "|", "abcdefg", true},
- {[]byte("z|y|c\nd|e|f"), "|", "zycdef", false},
- }
- for _, test := range tests {
- csv, err := parseCSV(test.csv, test.sep)
- if test.err && err == nil {
- t.Error("Expecting an error")
- }
- if test.err {
- continue
- }
- if !test.err && err != nil {
- t.Error(err)
- }
-
- act := ""
- for _, v := range csv {
- act = act + strings.Join(v, "")
- }
-
- if act != test.exp {
- t.Errorf("\nExpected: %s\nActual: %s\n%#v\n", test.exp, act, csv)
- }
-
- }
-}
-
-func TestGetJSONFailParse(t *testing.T) {
- t.Parallel()
-
- ns := New(newDeps(viper.New()))
-
- reqCount := 0
- ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- if reqCount > 0 {
- w.Header().Add("Content-type", "application/json")
- fmt.Fprintln(w, `{"gomeetup":["Sydney", "San Francisco", "Stockholm"]}`)
- } else {
- w.WriteHeader(http.StatusInternalServerError)
- fmt.Fprintln(w, `ERROR 500`)
- }
- reqCount++
- }))
- defer ts.Close()
- url := ts.URL + "/test.json"
-
- want := map[string]interface{}{"gomeetup": []interface{}{"Sydney", "San Francisco", "Stockholm"}}
- have := ns.GetJSON(url)
- assert.NotNil(t, have)
- if have != nil {
- assert.EqualValues(t, want, have)
- }
-}
-
-func TestGetCSVFailParseSep(t *testing.T) {
- t.Parallel()
-
- ns := New(newDeps(viper.New()))
-
- reqCount := 0
- ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- if reqCount > 0 {
- w.Header().Add("Content-type", "application/json")
- fmt.Fprintln(w, `gomeetup,city`)
- fmt.Fprintln(w, `yes,Sydney`)
- fmt.Fprintln(w, `yes,San Francisco`)
- fmt.Fprintln(w, `yes,Stockholm`)
- } else {
- w.WriteHeader(http.StatusInternalServerError)
- fmt.Fprintln(w, `ERROR 500`)
- }
- reqCount++
- }))
- defer ts.Close()
- url := ts.URL + "/test.csv"
-
- want := [][]string{{"gomeetup", "city"}, {"yes", "Sydney"}, {"yes", "San Francisco"}, {"yes", "Stockholm"}}
- have := ns.GetCSV(",", url)
- assert.NotNil(t, have)
- if have != nil {
- assert.EqualValues(t, want, have)
- }
-}
-
-func TestGetCSVFailParse(t *testing.T) {
- t.Parallel()
-
- ns := New(newDeps(viper.New()))
-
- reqCount := 0
- ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- w.Header().Add("Content-type", "application/json")
- if reqCount > 0 {
- fmt.Fprintln(w, `gomeetup,city`)
- fmt.Fprintln(w, `yes,Sydney`)
- fmt.Fprintln(w, `yes,San Francisco`)
- fmt.Fprintln(w, `yes,Stockholm`)
- } else {
- fmt.Fprintln(w, `gomeetup,city`)
- fmt.Fprintln(w, `yes,Sydney,Bondi,`) // wrong number of fields in line
- fmt.Fprintln(w, `yes,San Francisco`)
- fmt.Fprintln(w, `yes,Stockholm`)
- }
- reqCount++
- }))
- defer ts.Close()
- url := ts.URL + "/test.csv"
-
- want := [][]string{{"gomeetup", "city"}, {"yes", "Sydney"}, {"yes", "San Francisco"}, {"yes", "Stockholm"}}
- have := ns.GetCSV(",", url)
- assert.NotNil(t, have)
- if have != nil {
- assert.EqualValues(t, want, have)
- }
}
func newDeps(cfg config.Provider) *deps.Deps {