feat(pkger): allow for base github content urls for json/yaml/jsonnet templates

closes: #18479

CHANGELOG.md

@@ -5,6 +5,7 @@
 1. [18387](https://github.com/influxdata/influxdb/pull/18387): Integrate query cancellation after queries have been submitted
 1. [18515](https://github.com/influxdata/influxdb/pull/18515): Extend templates with the source file|url|reader.
 1. [18539](https://github.com/influxdata/influxdb/pull/18539): Collect stats on installed influxdata community template usage.
+1. [18541](https://github.com/influxdata/influxdb/pull/18541): Pkger allow raw github.com host URLs for yaml|json|jsonnet URLs
 
 ## v2.0.0-beta.12 [2020-06-12]
 

pkger/parser.go

@@ -9,6 +9,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/url"
+	"path"
 	"regexp"
 	"sort"
 	"strconv"
@@ -140,12 +141,15 @@ func FromString(s string) ReaderFn {
 	}
 }
 
+var defaultHTTPClient = &http.Client{
+	Timeout: time.Minute,
+}
+
 // FromHTTPRequest parses a pkg from the request body of a HTTP request. This is
 // very useful when using packages that are hosted..
 func FromHTTPRequest(addr string) ReaderFn {
 	return func() (io.Reader, string, error) {
-		client := http.Client{Timeout: 5 * time.Minute}
-		resp, err := client.Get(addr)
+		resp, err := defaultHTTPClient.Get(normalizeGithubURLToContent(addr))
 		if err != nil {
 			return nil, addr, err
 		}
@@ -167,6 +171,35 @@ func FromHTTPRequest(addr string) ReaderFn {
 	}
 }
 
+const (
+	githubRawContentHost = "raw.githubusercontent.com"
+	githubHost           = "github.com"
+)
+
+func normalizeGithubURLToContent(addr string) string {
+	u, err := url.Parse(addr)
+	if err != nil {
+		return addr
+	}
+
+	if u.Host == githubHost {
+		switch path.Ext(u.Path) {
+		case ".yaml", ".yml", ".json", ".jsonnet":
+		default:
+			return u.String()
+		}
+
+		parts := strings.Split(u.Path, "/")
+		if len(parts) < 4 {
+			return u.String()
+		}
+		u.Host = githubRawContentHost
+		u.Path = path.Join(append(parts[:3], parts[4:]...)...)
+	}
+
+	return u.String()
+}
+
 func parseJSON(r io.Reader, opts ...ValidateOptFn) (*Pkg, error) {
 	return parse(json.NewDecoder(r), opts...)
 }

pkger/parser_test.go

@@ -3979,6 +3979,70 @@ spec:
 	})
 }
 
+func Test_normalizeGithubURLToContent(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		expected string
+	}{
+		{
+			name:     "raw url passes untouched",
+			input:    "https://raw.githubusercontent.com/influxdata/community-templates/master/github/github.yml",
+			expected: "https://raw.githubusercontent.com/influxdata/community-templates/master/github/github.yml",
+		},
+		{
+			name:     "URL that is to short is unchanged",
+			input:    "https://github.com/influxdata/community-templates",
+			expected: "https://github.com/influxdata/community-templates",
+		},
+		{
+			name:     "URL that does not end in required extention is unchanged",
+			input:    "https://github.com/influxdata/community-templates/master/github",
+			expected: "https://github.com/influxdata/community-templates/master/github",
+		},
+		{
+			name:     "converts base url with ext yaml to raw content url",
+			input:    "https://github.com/influxdata/community-templates/blob/master/github/github.yaml",
+			expected: "https://raw.githubusercontent.com/influxdata/community-templates/master/github/github.yaml",
+		},
+		{
+			name:     "converts base url with ext yml to raw content url",
+			input:    "https://github.com/influxdata/community-templates/blob/master/github/github.yml",
+			expected: "https://raw.githubusercontent.com/influxdata/community-templates/master/github/github.yml",
+		},
+		{
+			name:     "converts base url with ext json to raw content url",
+			input:    "https://github.com/influxdata/community-templates/blob/master/github/github.json",
+			expected: "https://raw.githubusercontent.com/influxdata/community-templates/master/github/github.json",
+		},
+		{
+			name:     "converts base url with ext jsonnet to raw content url",
+			input:    "https://github.com/influxdata/community-templates/blob/master/github/github.jsonnet",
+			expected: "https://raw.githubusercontent.com/influxdata/community-templates/master/github/github.jsonnet",
+		},
+		{
+			name:     "url with unexpected content type is unchanged 1",
+			input:    "https://github.com/influxdata/community-templates/blob/master/github/github.jason",
+			expected: "https://github.com/influxdata/community-templates/blob/master/github/github.jason",
+		},
+		{
+			name:     "url with unexpected content type is unchanged 2",
+			input:    "https://github.com/influxdata/community-templates/blob/master/github/github.rando",
+			expected: "https://github.com/influxdata/community-templates/blob/master/github/github.rando",
+		},
+	}
+
+	for _, tt := range tests {
+		fn := func(t *testing.T) {
+			actual := normalizeGithubURLToContent(tt.input)
+
+			assert.Equal(t, tt.expected, actual)
+		}
+
+		t.Run(tt.name, fn)
+	}
+}
+
 func Test_IsParseError(t *testing.T) {
 	tests := []struct {
 		name     string

pkger/service_metrics.go

@@ -3,6 +3,7 @@ package pkger
 import (
 	"context"
 	"net/url"
+	"path"
 	"strings"
 
 	"github.com/influxdata/influxdb/v2"
@@ -110,8 +111,8 @@ func normalizeRemoteSources(sources []string) []url.URL {
 		if !strings.HasPrefix(u.Scheme, "http") {
 			continue
 		}
-		if u.Host == "raw.githubusercontent.com" {
-			u.Host = "github.com"
+		if u.Host == githubRawContentHost {
+			u.Host = githubHost
 			u.Path = normalizeRawGithubPath(u.Path)
 		}
 		out = append(out, *u)
@@ -127,5 +128,5 @@ func normalizeRawGithubPath(rawPath string) string {
 	// keep /account/repo as base, then append the blob to it
 	tail := append([]string{"blob"}, parts[3:]...)
 	parts = append(parts[:3], tail...)
-	return strings.Join(parts, "/")
+	return path.Join(parts...)
 }

pkger/service_test.go

@@ -3261,8 +3261,10 @@ func Test_normalizeRemoteSources(t *testing.T) {
 	for _, tt := range tests {
 		fn := func(t *testing.T) {
 			actual := normalizeRemoteSources(tt.input)
-
-			assert.Equal(t, tt.expected, actual)
+			require.Len(t, actual, len(tt.expected))
+			for i, expected := range tt.expected {
+				assert.Equal(t, expected.String(), actual[i].String())
+			}
 		}
 		t.Run(tt.name, fn)
 	}