Add support for server templates

instead of sending title/message templates as part of the publishing request, systems can simply reference existing templates from the server. this has two advantages a) publish URLs become shorter b) publish URLs become easier to reuse the changes backwards-compatible, as the `tpl` parameter continues to support truthy boolean values. the feature has to be enabled on the server. available templates and their content are exposed via new API endpoints.
binwiederhier · Nov 11, 2024 · 823070b · 823070b
1 parent 630f295
commit 823070b
Show file tree

Hide file tree

Showing 12 changed files with 599 additions and 48 deletions.
diff --git a/cmd/serve.go b/cmd/serve.go
@@ -100,6 +100,7 @@ var flagsServe = append(
 	altsrc.NewStringFlag(&cli.StringFlag{Name: "web-push-file", Aliases: []string{"web_push_file"}, EnvVars: []string{"NTFY_WEB_PUSH_FILE"}, Usage: "file used to store web push subscriptions"}),
 	altsrc.NewStringFlag(&cli.StringFlag{Name: "web-push-email-address", Aliases: []string{"web_push_email_address"}, EnvVars: []string{"NTFY_WEB_PUSH_EMAIL_ADDRESS"}, Usage: "e-mail address of sender, required to use browser push services"}),
 	altsrc.NewStringFlag(&cli.StringFlag{Name: "web-push-startup-queries", Aliases: []string{"web_push_startup_queries"}, EnvVars: []string{"NTFY_WEB_PUSH_STARTUP_QUERIES"}, Usage: "queries run when the web push database is initialized"}),
+	altsrc.NewStringFlag(&cli.StringFlag{Name: "template-directory", Aliases: []string{"template_directory"}, EnvVars: []string{"NTFY_TEMPLATE_DIRECTORY"}, Usage: "Directory to serve message templates from"}),
 )
 
 var cmdServe = &cli.Command{
@@ -140,6 +141,7 @@ func execServe(c *cli.Context) error {
 	webPushFile := c.String("web-push-file")
 	webPushEmailAddress := c.String("web-push-email-address")
 	webPushStartupQueries := c.String("web-push-startup-queries")
+	templateDirectory := c.String("template-directory")
 	cacheFile := c.String("cache-file")
 	cacheDurationStr := c.String("cache-duration")
 	cacheStartupQueries := c.String("cache-startup-queries")
@@ -256,6 +258,8 @@ func execServe(c *cli.Context) error {
 		return errors.New("if set, FCM key file must exist")
 	} else if webPushPublicKey != "" && (webPushPrivateKey == "" || webPushFile == "" || webPushEmailAddress == "" || baseURL == "") {
 		return errors.New("if web push is enabled, web-push-private-key, web-push-public-key, web-push-file, web-push-email-address, and base-url should be set. run 'ntfy webpush keys' to generate keys")
+	} else if templateDirectory != "" && !util.DirectoryExists(templateDirectory) {
+		return fmt.Errorf("templates directory %q does not exist", templateDirectory)
 	} else if keepaliveInterval < 5*time.Second {
 		return errors.New("keepalive interval cannot be lower than five seconds")
 	} else if managerInterval < 5*time.Second {
@@ -417,6 +421,7 @@ func execServe(c *cli.Context) error {
 	conf.WebPushFile = webPushFile
 	conf.WebPushEmailAddress = webPushEmailAddress
 	conf.WebPushStartupQueries = webPushStartupQueries
+	conf.TemplateDirectory = templateDirectory
 
 	// Set up hot-reloading of config
 	go sigHandlerConfigReload(config)

diff --git a/docs/publish.md b/docs/publish.md
@@ -1075,6 +1075,33 @@ This example uses the `message`/`m` and `title`/`t` query parameters, but obviou
 `Message`/`Title` headers. It will send a notification with a title `phil-pc: A severe error has occurred` and a message
 `Error message: Disk has run out of space`.
 
+### Server templates
+
+In order to avoid running into limitations related to the URL length and to simplify reusing templates across publishing systems,
+title/message templates can also be provided by the server and then simply referenced during publishing.
+
+This feature has to be enabled on the server first, by specifying a base directory for templates using the `--template-directory`.
+All files within this directory (subdirectories are not supported) with the file extension *.tpl* are considered
+templates to be used during publishing, referenced by their basename (without the extension).
+
+To use a template, use `server` instead of `yes`/`1` for the templating parameter. The `message` and `title` parameters
+are used as template name.
+
+!!! info
+    The templating feature applies to both the title and message both in equal parts. You can *not* mix and match,
+    e.g. by specifying a template name for the title and an inline template for the body.
+
+Assuming the server is running with the following template directory:
+
+```
+/etc/ntfy/templates
+|_ grafana_title.tpl
+|_ grafana_body.tpl
+\_ hello_world.tpl
+```
+
+You can now publish your payload to the following endpoint: `https://ntfy.sh/mytopic?tpl=server&title=hello_world&message=grafana_body`
+
 ## Publish as JSON
 _Supported on:_ :material-android: :material-apple: :material-firefox:
 

diff --git a/server/config.go b/server/config.go
@@ -161,6 +161,7 @@ type Config struct {
 	WebPushStartupQueries                string
 	WebPushExpiryDuration                time.Duration
 	WebPushExpiryWarningDuration         time.Duration
+	TemplateDirectory                    string
 }
 
 // NewConfig instantiates a default new server config
@@ -248,5 +249,6 @@ func NewConfig() *Config {
 		WebPushEmailAddress:                  "",
 		WebPushExpiryDuration:                DefaultWebPushExpiryDuration,
 		WebPushExpiryWarningDuration:         DefaultWebPushExpiryWarningDuration,
+		TemplateDirectory:                    "",
 	}
 }
diff --git a/server/errors.go b/server/errors.go
@@ -123,6 +123,7 @@ var (
 	errHTTPBadRequestTemplateDisallowedFunctionCalls = &errHTTP{40044, http.StatusBadRequest, "invalid request: template contains disallowed function calls, e.g. template, call, or define", "https://ntfy.sh/docs/publish/#message-templating", nil}
 	errHTTPBadRequestTemplateExecuteFailed           = &errHTTP{40045, http.StatusBadRequest, "invalid request: template execution failed", "https://ntfy.sh/docs/publish/#message-templating", nil}
 	errHTTPBadRequestInvalidUsername                 = &errHTTP{40046, http.StatusBadRequest, "invalid request: invalid username", "", nil}
+	errHTTPBadRequestTemplatesNotEnabled             = &errHTTP{40047, http.StatusBadRequest, "invalid request: templates not enabled", "https://ntfy.sh/docs/config", nil}
 	errHTTPNotFound                                  = &errHTTP{40401, http.StatusNotFound, "page not found", "", nil}
 	errHTTPUnauthorized                              = &errHTTP{40101, http.StatusUnauthorized, "unauthorized", "https://ntfy.sh/docs/publish/#authentication", nil}
 	errHTTPForbidden                                 = &errHTTP{40301, http.StatusForbidden, "forbidden", "https://ntfy.sh/docs/publish/#authentication", nil}

diff --git a/server/server.go b/server/server.go
@@ -23,7 +23,6 @@ import (
 	"strconv"
 	"strings"
 	"sync"
-	"text/template"
 	"time"
 	"unicode/utf8"
 
@@ -88,6 +87,7 @@ var (
 	apiHealthPath                                        = "/v1/health"
 	apiStatsPath                                         = "/v1/stats"
 	apiWebPushPath                                       = "/v1/webpush"
+	apiTemplatesPath                                     = "/v1/templates"
 	apiTiersPath                                         = "/v1/tiers"
 	apiUsersPath                                         = "/v1/users"
 	apiUsersAccessPath                                   = "/v1/users/access"
@@ -505,6 +505,8 @@ func (s *Server) handleInternal(w http.ResponseWriter, r *http.Request, v *visit
 		return s.handleStats(w, r, v)
 	} else if r.Method == http.MethodGet && r.URL.Path == apiTiersPath {
 		return s.ensurePaymentsEnabled(s.handleBillingTiersGet)(w, r, v)
+	} else if r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, apiTemplatesPath) {
+		return s.ensureTemplatesEnabled(s.limitRequests(s.handleTemplates))(w, r, v)
 	} else if r.Method == http.MethodGet && r.URL.Path == matrixPushPath {
 		return s.handleMatrixDiscovery(w)
 	} else if r.Method == http.MethodGet && r.URL.Path == metricsPath && s.metricsHandler != nil {
@@ -617,6 +619,27 @@ func (s *Server) handleWebManifest(w http.ResponseWriter, _ *http.Request, _ *vi
 	return s.writeJSONWithContentType(w, response, "application/manifest+json")
 }
 
+// handleTemplates either writes a list of available templates (if only the base API path
+// is requested) or serves the requested template file.
+func (s *Server) handleTemplates(w http.ResponseWriter, r *http.Request, _ *visitor) error {
+	path := r.URL.Path[len(apiTemplatesPath):]
+
+	if path == "" || path == "/" {
+		ls, err := s.listTemplates()
+		if err != nil {
+			return err
+		}
+
+		response := &templateNamesResponse{
+			Templates: ls,
+		}
+
+		return s.writeJSON(w, response)
+	}
+
+	return s.serveTemplate(w, r, path)
+}
+
 // handleMetrics returns Prometheus metrics. This endpoint is only called if enable-metrics is set,
 // and listen-metrics-http is not set.
 func (s *Server) handleMetrics(w http.ResponseWriter, r *http.Request, _ *visitor) error {
@@ -933,7 +956,7 @@ func (s *Server) forwardPollRequest(v *visitor, m *message) {
 	}
 }
 
-func (s *Server) parsePublishParams(r *http.Request, m *message) (cache bool, firebase bool, email, call string, template bool, unifiedpush bool, err *errHTTP) {
+func (s *Server) parsePublishParams(r *http.Request, m *message) (cache bool, firebase bool, email, call string, template templateFeature, unifiedpush bool, err *errHTTP) {
 	cache = readBoolParam(r, true, "x-cache", "cache")
 	firebase = readBoolParam(r, true, "x-firebase", "firebase")
 	m.Title = readParam(r, "x-title", "title", "t")
@@ -949,7 +972,7 @@ func (s *Server) parsePublishParams(r *http.Request, m *message) (cache bool, fi
 	}
 	if attach != "" {
 		if !urlRegex.MatchString(attach) {
-			return false, false, "", "", false, false, errHTTPBadRequestAttachmentURLInvalid
+			return false, false, "", "", templateFeatureDisabled, false, errHTTPBadRequestAttachmentURLInvalid
 		}
 		m.Attachment.URL = attach
 		if m.Attachment.Name == "" {
@@ -967,19 +990,19 @@ func (s *Server) parsePublishParams(r *http.Request, m *message) (cache bool, fi
 	}
 	if icon != "" {
 		if !urlRegex.MatchString(icon) {
-			return false, false, "", "", false, false, errHTTPBadRequestIconURLInvalid
+			return false, false, "", "", templateFeatureDisabled, false, errHTTPBadRequestIconURLInvalid
 		}
 		m.Icon = icon
 	}
 	email = readParam(r, "x-email", "x-e-mail", "email", "e-mail", "mail", "e")
 	if s.smtpSender == nil && email != "" {
-		return false, false, "", "", false, false, errHTTPBadRequestEmailDisabled
+		return false, false, "", "", templateFeatureDisabled, false, errHTTPBadRequestEmailDisabled
 	}
 	call = readParam(r, "x-call", "call")
 	if call != "" && (s.config.TwilioAccount == "" || s.userManager == nil) {
-		return false, false, "", "", false, false, errHTTPBadRequestPhoneCallsDisabled
+		return false, false, "", "", templateFeatureDisabled, false, errHTTPBadRequestPhoneCallsDisabled
 	} else if call != "" && !isBoolValue(call) && !phoneNumberRegex.MatchString(call) {
-		return false, false, "", "", false, false, errHTTPBadRequestPhoneNumberInvalid
+		return false, false, "", "", templateFeatureDisabled, false, errHTTPBadRequestPhoneNumberInvalid
 	}
 	messageStr := strings.ReplaceAll(readParam(r, "x-message", "message", "m"), "\\n", "\n")
 	if messageStr != "" {
@@ -988,42 +1011,42 @@ func (s *Server) parsePublishParams(r *http.Request, m *message) (cache bool, fi
 	var e error
 	m.Priority, e = util.ParsePriority(readParam(r, "x-priority", "priority", "prio", "p"))
 	if e != nil {
-		return false, false, "", "", false, false, errHTTPBadRequestPriorityInvalid
+		return false, false, "", "", templateFeatureDisabled, false, errHTTPBadRequestPriorityInvalid
 	}
 	m.Tags = readCommaSeparatedParam(r, "x-tags", "tags", "tag", "ta")
 	delayStr := readParam(r, "x-delay", "delay", "x-at", "at", "x-in", "in")
 	if delayStr != "" {
 		if !cache {
-			return false, false, "", "", false, false, errHTTPBadRequestDelayNoCache
+			return false, false, "", "", templateFeatureDisabled, false, errHTTPBadRequestDelayNoCache
 		}
 		if email != "" {
-			return false, false, "", "", false, false, errHTTPBadRequestDelayNoEmail // we cannot store the email address (yet)
+			return false, false, "", "", templateFeatureDisabled, false, errHTTPBadRequestDelayNoEmail // we cannot store the email address (yet)
 		}
 		if call != "" {
-			return false, false, "", "", false, false, errHTTPBadRequestDelayNoCall // we cannot store the phone number (yet)
+			return false, false, "", "", templateFeatureDisabled, false, errHTTPBadRequestDelayNoCall // we cannot store the phone number (yet)
 		}
 		delay, err := util.ParseFutureTime(delayStr, time.Now())
 		if err != nil {
-			return false, false, "", "", false, false, errHTTPBadRequestDelayCannotParse
+			return false, false, "", "", templateFeatureDisabled, false, errHTTPBadRequestDelayCannotParse
 		} else if delay.Unix() < time.Now().Add(s.config.MessageDelayMin).Unix() {
-			return false, false, "", "", false, false, errHTTPBadRequestDelayTooSmall
+			return false, false, "", "", templateFeatureDisabled, false, errHTTPBadRequestDelayTooSmall
 		} else if delay.Unix() > time.Now().Add(s.config.MessageDelayMax).Unix() {
-			return false, false, "", "", false, false, errHTTPBadRequestDelayTooLarge
+			return false, false, "", "", templateFeatureDisabled, false, errHTTPBadRequestDelayTooLarge
 		}
 		m.Time = delay.Unix()
 	}
 	actionsStr := readParam(r, "x-actions", "actions", "action")
 	if actionsStr != "" {
 		m.Actions, e = parseActions(actionsStr)
 		if e != nil {
-			return false, false, "", "", false, false, errHTTPBadRequestActionsInvalid.Wrap(e.Error())
+			return false, false, "", "", templateFeatureDisabled, false, errHTTPBadRequestActionsInvalid.Wrap(e.Error())
 		}
 	}
 	contentType, markdown := readParam(r, "content-type", "content_type"), readBoolParam(r, false, "x-markdown", "markdown", "md")
 	if markdown || strings.ToLower(contentType) == "text/markdown" {
 		m.ContentType = "text/markdown"
 	}
-	template = readBoolParam(r, false, "x-template", "template", "tpl")
+	template = parseTemplateFeature(readParam(r, "x-template", "template", "tpl"))
 	unifiedpush = readBoolParam(r, false, "x-unifiedpush", "unifiedpush", "up") // see GET too!
 	if unifiedpush {
 		firebase = false
@@ -1050,11 +1073,13 @@ func (s *Server) parsePublishParams(r *http.Request, m *message) (cache bool, fi
 //     Body must be attachment, because we passed a filename
 //  5. curl -H "Template: yes" -T file.txt ntfy.sh/mytopic
 //     If templating is enabled, read up to 32k and treat message body as JSON
-//  6. curl -T file.txt ntfy.sh/mytopic
-//     If file.txt is <= 4096 (message limit) and valid UTF-8, treat it as a message
+//  6. curl -H "Template: server" -T file.txt ntfy.sh/mytopic?m=foobar
+//     Read foobar template from filesystem and treat message body as JSON
 //  7. curl -T file.txt ntfy.sh/mytopic
+//     If file.txt is <= 4096 (message limit) and valid UTF-8, treat it as a message
+//  8. curl -T file.txt ntfy.sh/mytopic
 //     In all other cases, mostly if file.txt is > message limit, treat it as an attachment
-func (s *Server) handlePublishBody(r *http.Request, v *visitor, m *message, body *util.PeekedReadCloser, template, unifiedpush bool) error {
+func (s *Server) handlePublishBody(r *http.Request, v *visitor, m *message, body *util.PeekedReadCloser, template templateFeature, unifiedpush bool) error {
 	if m.Event == pollRequestEvent { // Case 1
 		return s.handleBodyDiscard(body)
 	} else if unifiedpush {
@@ -1063,12 +1088,12 @@ func (s *Server) handlePublishBody(r *http.Request, v *visitor, m *message, body
 		return s.handleBodyAsTextMessage(m, body) // Case 3
 	} else if m.Attachment != nil && m.Attachment.Name != "" {
 		return s.handleBodyAsAttachment(r, v, m, body) // Case 4
-	} else if template {
-		return s.handleBodyAsTemplatedTextMessage(m, body) // Case 5
+	} else if template != templateFeatureDisabled {
+		return s.handleBodyAsTemplatedTextMessage(template, v, m, body) // Case 5&6
 	} else if !body.LimitReached && utf8.Valid(body.PeekedBytes) {
-		return s.handleBodyAsTextMessage(m, body) // Case 6
+		return s.handleBodyAsTextMessage(m, body) // Case 7
 	}
-	return s.handleBodyAsAttachment(r, v, m, body) // Case 7
+	return s.handleBodyAsAttachment(r, v, m, body) // Case 8
 }
 
 func (s *Server) handleBodyDiscard(body *util.PeekedReadCloser) error {
@@ -1100,45 +1125,51 @@ func (s *Server) handleBodyAsTextMessage(m *message, body *util.PeekedReadCloser
 	return nil
 }
 
-func (s *Server) handleBodyAsTemplatedTextMessage(m *message, body *util.PeekedReadCloser) error {
+func (s *Server) handleBodyAsTemplatedTextMessage(t templateFeature, v *visitor, m *message, body *util.PeekedReadCloser) error {
+	var internalError error
+	defer func() {
+		if internalError != nil {
+			logvm(v, m).Err(internalError).Info("Failed to render %s templated message", t)
+		}
+	}()
+
 	body, err := util.Peek(body, max(s.config.MessageSizeLimit, jsonBodyBytesLimit))
 	if err != nil {
 		return err
 	} else if body.LimitReached {
 		return errHTTPEntityTooLargeJSONBody
 	}
-	peekedBody := strings.TrimSpace(string(body.PeekedBytes))
-	if m.Message, err = replaceTemplate(m.Message, peekedBody); err != nil {
-		return err
+	peekedBody := bytes.TrimSpace(body.PeekedBytes)
+	var data any
+	if err := json.Unmarshal(peekedBody, &data); err != nil {
+		return errHTTPBadRequestTemplateMessageNotJSON
 	}
-	if m.Title, err = replaceTemplate(m.Title, peekedBody); err != nil {
-		return err
+
+	if t == templateFeatureInline {
+		if m.Message, internalError, err = s.replaceTemplate(m.Message, data); err != nil {
+			return err
+		}
+		if m.Title, internalError, err = s.replaceTemplate(m.Title, data); err != nil {
+			return err
+		}
+	} else if t == templateFeatureServer {
+		if s.config.TemplateDirectory == "" {
+			return errHTTPBadRequestTemplatesNotEnabled
+		}
+		if m.Message, internalError, err = s.replaceTemplateFile(m.Message, data); err != nil {
+			return err
+		}
+		if m.Title, internalError, err = s.replaceTemplateFile(m.Title, data); err != nil {
+			return err
+		}
 	}
+
 	if len(m.Message) > s.config.MessageSizeLimit {
 		return errHTTPBadRequestTemplateMessageTooLarge
 	}
 	return nil
 }
 
-func replaceTemplate(tpl string, source string) (string, error) {
-	if templateDisallowedRegex.MatchString(tpl) {
-		return "", errHTTPBadRequestTemplateDisallowedFunctionCalls
-	}
-	var data any
-	if err := json.Unmarshal([]byte(source), &data); err != nil {
-		return "", errHTTPBadRequestTemplateMessageNotJSON
-	}
-	t, err := template.New("").Parse(tpl)
-	if err != nil {
-		return "", errHTTPBadRequestTemplateInvalid
-	}
-	var buf bytes.Buffer
-	if err := t.Execute(util.NewTimeoutWriter(&buf, templateMaxExecutionTime), data); err != nil {
-		return "", errHTTPBadRequestTemplateExecuteFailed
-	}
-	return buf.String(), nil
-}
-
 func (s *Server) handleBodyAsAttachment(r *http.Request, v *visitor, m *message, body *util.PeekedReadCloser) error {
 	if s.fileCache == nil || s.config.BaseURL == "" || s.config.AttachmentCacheDir == "" {
 		return errHTTPBadRequestAttachmentsDisallowed.With(m)

diff --git a/server/server_middleware.go b/server/server_middleware.go
@@ -121,6 +121,15 @@ func (s *Server) ensureStripeCustomer(next handleFunc) handleFunc {
 	})
 }
 
+func (s *Server) ensureTemplatesEnabled(next handleFunc) handleFunc {
+	return func(w http.ResponseWriter, r *http.Request, v *visitor) error {
+		if s.config.TemplateDirectory == "" {
+			return errHTTPNotFound
+		}
+		return next(w, r, v)
+	}
+}
+
 func (s *Server) withAccountSync(next handleFunc) handleFunc {
 	return func(w http.ResponseWriter, r *http.Request, v *visitor) error {
 		err := next(w, r, v)