[feature] Push notifications (#3587)

* Update push subscription API model to be Mastodon 4.0 compatible

* Add webpush-go dependency

# Conflicts:
#	go.sum

* Single-row table for storing instance's VAPID key pair

* Generate VAPID key pair during startup

* Add VAPID public key to instance info API

* Return VAPID public key when registering an app

* Store Web Push subscriptions in DB

* Add Web Push sender (similar to email sender)

* Add no-op push senders to most processor tests

* Test Web Push notifications from workers

* Delete Web Push subscriptions when account is deleted

* Implement push subscription API

* Linter fixes

* Update Swagger

* Fix enum to int migration

* Fix GetVAPIDKeyPair

* Create web push subscriptions table with indexes

* Log Web Push server error messages

* Send instance URL as Web Push JWT subject

* Accept any 2xx code as a success

* Fix malformed VAPID sub claim

* Use packed notification flags

* Remove unused date columns

* Add notification type for update notifications

Not used yet

* Make GetVAPIDKeyPair idempotent

and remove PutVAPIDKeyPair

* Post-rebase fixes

* go mod tidy

* Special-case 400 errors other than 408/429

Most client errors should remove the subscription.

* Improve titles, trim body to reasonable length

* Disallow cleartext HTTP for Web Push servers

* Fix lint

* Remove redundant index on unique column

Also removes redundant unique and notnull tags on ID column since these are implied by pk

* Make realsender.go more readable

* Use Tobi's style for wrapping errors

* Restore treating all 5xx codes as temporary problems

* Always load target account settings

* Stub `policy` and `standard`

* webpush.Sender: take type converter as ctor param

* Move webpush.MockSender and noopSender into testrig

vendor/github.com/SherClockHolmes/webpush-go/.gitignore

@@ -0,0 +1,4 @@
+vendor/**
+
+.DS_Store
+*.out

vendor/github.com/SherClockHolmes/webpush-go/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2016 Ethan Holmes
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vendor/github.com/SherClockHolmes/webpush-go/README.md

@@ -0,0 +1,63 @@
+# webpush-go
+
+[![Go Report Card](https://goreportcard.com/badge/github.com/SherClockHolmes/webpush-go)](https://goreportcard.com/report/github.com/SherClockHolmes/webpush-go)
+[![GoDoc](https://godoc.org/github.com/SherClockHolmes/webpush-go?status.svg)](https://godoc.org/github.com/SherClockHolmes/webpush-go)
+
+Web Push API Encryption with VAPID support.
+
+```bash
+go get -u github.com/SherClockHolmes/webpush-go
+```
+
+## Example
+
+For a full example, refer to the code in the [example](example/) directory.
+
+```go
+package main
+
+import (
+	"encoding/json"
+
+	webpush "github.com/SherClockHolmes/webpush-go"
+)
+
+func main() {
+	// Decode subscription
+	s := &webpush.Subscription{}
+	json.Unmarshal([]byte("<YOUR_SUBSCRIPTION>"), s)
+
+	// Send Notification
+	resp, err := webpush.SendNotification([]byte("Test"), s, &webpush.Options{
+		Subscriber:      "example@example.com",
+		VAPIDPublicKey:  "<YOUR_VAPID_PUBLIC_KEY>",
+		VAPIDPrivateKey: "<YOUR_VAPID_PRIVATE_KEY>",
+		TTL:             30,
+	})
+	if err != nil {
+		// TODO: Handle error
+	}
+	defer resp.Body.Close()
+}
+```
+
+### Generating VAPID Keys
+
+Use the helper method `GenerateVAPIDKeys` to generate the VAPID key pair.
+
+```golang
+privateKey, publicKey, err := webpush.GenerateVAPIDKeys()
+if err != nil {
+	// TODO: Handle error
+}
+```
+
+## Development
+
+1. Install [Go 1.11+](https://golang.org/)
+2. `go mod vendor`
+3. `go test`
+
+#### For other language implementations visit:
+
+[WebPush Libs](https://github.com/web-push-libs)

vendor/github.com/SherClockHolmes/webpush-go/urgency.go

@@ -0,0 +1,26 @@
+package webpush
+
+// Urgency indicates to the push service how important a message is to the user.
+// This can be used by the push service to help conserve the battery life of a user's device
+// by only waking up for important messages when battery is low.
+type Urgency string
+
+const (
+	// UrgencyVeryLow requires device state: on power and Wi-Fi
+	UrgencyVeryLow Urgency = "very-low"
+	// UrgencyLow requires device state: on either power or Wi-Fi
+	UrgencyLow Urgency = "low"
+	// UrgencyNormal excludes device state: low battery
+	UrgencyNormal Urgency = "normal"
+	// UrgencyHigh admits device state: low battery
+	UrgencyHigh Urgency = "high"
+)
+
+// Checking allowable values for the urgency header
+func isValidUrgency(urgency Urgency) bool {
+	switch urgency {
+	case UrgencyVeryLow, UrgencyLow, UrgencyNormal, UrgencyHigh:
+		return true
+	}
+	return false
+}

vendor/github.com/SherClockHolmes/webpush-go/vapid.go

@@ -0,0 +1,117 @@
+package webpush
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"encoding/base64"
+	"fmt"
+	"math/big"
+	"net/url"
+	"time"
+
+	"github.com/golang-jwt/jwt"
+)
+
+// GenerateVAPIDKeys will create a private and public VAPID key pair
+func GenerateVAPIDKeys() (privateKey, publicKey string, err error) {
+	// Get the private key from the P256 curve
+	curve := elliptic.P256()
+
+	private, x, y, err := elliptic.GenerateKey(curve, rand.Reader)
+	if err != nil {
+		return
+	}
+
+	public := elliptic.Marshal(curve, x, y)
+
+	// Convert to base64
+	publicKey = base64.RawURLEncoding.EncodeToString(public)
+	privateKey = base64.RawURLEncoding.EncodeToString(private)
+
+	return
+}
+
+// Generates the ECDSA public and private keys for the JWT encryption
+func generateVAPIDHeaderKeys(privateKey []byte) *ecdsa.PrivateKey {
+	// Public key
+	curve := elliptic.P256()
+	px, py := curve.ScalarMult(
+		curve.Params().Gx,
+		curve.Params().Gy,
+		privateKey,
+	)
+
+	pubKey := ecdsa.PublicKey{
+		Curve: curve,
+		X:     px,
+		Y:     py,
+	}
+
+	// Private key
+	d := &big.Int{}
+	d.SetBytes(privateKey)
+
+	return &ecdsa.PrivateKey{
+		PublicKey: pubKey,
+		D:         d,
+	}
+}
+
+// getVAPIDAuthorizationHeader
+func getVAPIDAuthorizationHeader(
+	endpoint,
+	subscriber,
+	vapidPublicKey,
+	vapidPrivateKey string,
+	expiration time.Time,
+) (string, error) {
+	// Create the JWT token
+	subURL, err := url.Parse(endpoint)
+	if err != nil {
+		return "", err
+	}
+
+	token := jwt.NewWithClaims(jwt.SigningMethodES256, jwt.MapClaims{
+		"aud": fmt.Sprintf("%s://%s", subURL.Scheme, subURL.Host),
+		"exp": expiration.Unix(),
+		"sub": fmt.Sprintf("mailto:%s", subscriber),
+	})
+
+	// Decode the VAPID private key
+	decodedVapidPrivateKey, err := decodeVapidKey(vapidPrivateKey)
+	if err != nil {
+		return "", err
+	}
+
+	privKey := generateVAPIDHeaderKeys(decodedVapidPrivateKey)
+
+	// Sign token with private key
+	jwtString, err := token.SignedString(privKey)
+	if err != nil {
+		return "", err
+	}
+
+	// Decode the VAPID public key
+	pubKey, err := decodeVapidKey(vapidPublicKey)
+	if err != nil {
+		return "", err
+	}
+
+	return fmt.Sprintf(
+		"vapid t=%s, k=%s",
+		jwtString,
+		base64.RawURLEncoding.EncodeToString(pubKey),
+	), nil
+}
+
+// Need to decode the vapid private key in multiple base64 formats
+// Solution from: https://github.com/SherClockHolmes/webpush-go/issues/29
+func decodeVapidKey(key string) ([]byte, error) {
+	bytes, err := base64.URLEncoding.DecodeString(key)
+	if err == nil {
+		return bytes, nil
+	}
+
+	return base64.RawURLEncoding.DecodeString(key)
+}

vendor/github.com/SherClockHolmes/webpush-go/webpush.go

@@ -0,0 +1,287 @@
+package webpush
+
+import (
+	"bytes"
+	"context"
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/binary"
+	"errors"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"golang.org/x/crypto/hkdf"
+)
+
+const MaxRecordSize uint32 = 4096
+
+var ErrMaxPadExceeded = errors.New("payload has exceeded the maximum length")
+
+// saltFunc generates a salt of 16 bytes
+var saltFunc = func() ([]byte, error) {
+	salt := make([]byte, 16)
+	_, err := io.ReadFull(rand.Reader, salt)
+	if err != nil {
+		return salt, err
+	}
+
+	return salt, nil
+}
+
+// HTTPClient is an interface for sending the notification HTTP request / testing
+type HTTPClient interface {
+	Do(*http.Request) (*http.Response, error)
+}
+
+// Options are config and extra params needed to send a notification
+type Options struct {
+	HTTPClient      HTTPClient // Will replace with *http.Client by default if not included
+	RecordSize      uint32     // Limit the record size
+	Subscriber      string     // Sub in VAPID JWT token
+	Topic           string     // Set the Topic header to collapse a pending messages (Optional)
+	TTL             int        // Set the TTL on the endpoint POST request
+	Urgency         Urgency    // Set the Urgency header to change a message priority (Optional)
+	VAPIDPublicKey  string     // VAPID public key, passed in VAPID Authorization header
+	VAPIDPrivateKey string     // VAPID private key, used to sign VAPID JWT token
+	VapidExpiration time.Time  // optional expiration for VAPID JWT token (defaults to now + 12 hours)
+}
+
+// Keys are the base64 encoded values from PushSubscription.getKey()
+type Keys struct {
+	Auth   string `json:"auth"`
+	P256dh string `json:"p256dh"`
+}
+
+// Subscription represents a PushSubscription object from the Push API
+type Subscription struct {
+	Endpoint string `json:"endpoint"`
+	Keys     Keys   `json:"keys"`
+}
+
+// SendNotification calls SendNotificationWithContext with default context for backwards-compatibility
+func SendNotification(message []byte, s *Subscription, options *Options) (*http.Response, error) {
+	return SendNotificationWithContext(context.Background(), message, s, options)
+}
+
+// SendNotificationWithContext sends a push notification to a subscription's endpoint
+// Message Encryption for Web Push, and VAPID protocols.
+// FOR MORE INFORMATION SEE RFC8291: https://datatracker.ietf.org/doc/rfc8291
+func SendNotificationWithContext(ctx context.Context, message []byte, s *Subscription, options *Options) (*http.Response, error) {
+	// Authentication secret (auth_secret)
+	authSecret, err := decodeSubscriptionKey(s.Keys.Auth)
+	if err != nil {
+		return nil, err
+	}
+
+	// dh (Diffie Hellman)
+	dh, err := decodeSubscriptionKey(s.Keys.P256dh)
+	if err != nil {
+		return nil, err
+	}
+
+	// Generate 16 byte salt
+	salt, err := saltFunc()
+	if err != nil {
+		return nil, err
+	}
+
+	// Create the ecdh_secret shared key pair
+	curve := elliptic.P256()
+
+	// Application server key pairs (single use)
+	localPrivateKey, x, y, err := elliptic.GenerateKey(curve, rand.Reader)
+	if err != nil {
+		return nil, err
+	}
+
+	localPublicKey := elliptic.Marshal(curve, x, y)
+
+	// Combine application keys with receiver's EC public key
+	sharedX, sharedY := elliptic.Unmarshal(curve, dh)
+	if sharedX == nil {
+		return nil, errors.New("Unmarshal Error: Public key is not a valid point on the curve")
+	}
+
+	// Derive ECDH shared secret
+	sx, sy := curve.ScalarMult(sharedX, sharedY, localPrivateKey)
+	if !curve.IsOnCurve(sx, sy) {
+		return nil, errors.New("Encryption error: ECDH shared secret isn't on curve")
+	}
+	mlen := curve.Params().BitSize / 8
+	sharedECDHSecret := make([]byte, mlen)
+	sx.FillBytes(sharedECDHSecret)
+
+	hash := sha256.New
+
+	// ikm
+	prkInfoBuf := bytes.NewBuffer([]byte("WebPush: info\x00"))
+	prkInfoBuf.Write(dh)
+	prkInfoBuf.Write(localPublicKey)
+
+	prkHKDF := hkdf.New(hash, sharedECDHSecret, authSecret, prkInfoBuf.Bytes())
+	ikm, err := getHKDFKey(prkHKDF, 32)
+	if err != nil {
+		return nil, err
+	}
+
+	// Derive Content Encryption Key
+	contentEncryptionKeyInfo := []byte("Content-Encoding: aes128gcm\x00")
+	contentHKDF := hkdf.New(hash, ikm, salt, contentEncryptionKeyInfo)
+	contentEncryptionKey, err := getHKDFKey(contentHKDF, 16)
+	if err != nil {
+		return nil, err
+	}
+
+	// Derive the Nonce
+	nonceInfo := []byte("Content-Encoding: nonce\x00")
+	nonceHKDF := hkdf.New(hash, ikm, salt, nonceInfo)
+	nonce, err := getHKDFKey(nonceHKDF, 12)
+	if err != nil {
+		return nil, err
+	}
+
+	// Cipher
+	c, err := aes.NewCipher(contentEncryptionKey)
+	if err != nil {
+		return nil, err
+	}
+
+	gcm, err := cipher.NewGCM(c)
+	if err != nil {
+		return nil, err
+	}
+
+	// Get the record size
+	recordSize := options.RecordSize
+	if recordSize == 0 {
+		recordSize = MaxRecordSize
+	}
+
+	recordLength := int(recordSize) - 16
+
+	// Encryption Content-Coding Header
+	recordBuf := bytes.NewBuffer(salt)
+
+	rs := make([]byte, 4)
+	binary.BigEndian.PutUint32(rs, recordSize)
+
+	recordBuf.Write(rs)
+	recordBuf.Write([]byte{byte(len(localPublicKey))})
+	recordBuf.Write(localPublicKey)
+
+	// Data
+	dataBuf := bytes.NewBuffer(message)
+
+	// Pad content to max record size - 16 - header
+	// Padding ending delimeter
+	dataBuf.Write([]byte("\x02"))
+	if err := pad(dataBuf, recordLength-recordBuf.Len()); err != nil {
+		return nil, err
+	}
+
+	// Compose the ciphertext
+	ciphertext := gcm.Seal([]byte{}, nonce, dataBuf.Bytes(), nil)
+	recordBuf.Write(ciphertext)
+
+	// POST request
+	req, err := http.NewRequest("POST", s.Endpoint, recordBuf)
+	if err != nil {
+		return nil, err
+	}
+
+	if ctx != nil {
+		req = req.WithContext(ctx)
+	}
+
+	req.Header.Set("Content-Encoding", "aes128gcm")
+	req.Header.Set("Content-Length", strconv.Itoa(len(ciphertext)))
+	req.Header.Set("Content-Type", "application/octet-stream")
+	req.Header.Set("TTL", strconv.Itoa(options.TTL))
+
+	// Сheck the optional headers
+	if len(options.Topic) > 0 {
+		req.Header.Set("Topic", options.Topic)
+	}
+
+	if isValidUrgency(options.Urgency) {
+		req.Header.Set("Urgency", string(options.Urgency))
+	}
+
+	expiration := options.VapidExpiration
+	if expiration.IsZero() {
+		expiration = time.Now().Add(time.Hour * 12)
+	}
+
+	// Get VAPID Authorization header
+	vapidAuthHeader, err := getVAPIDAuthorizationHeader(
+		s.Endpoint,
+		options.Subscriber,
+		options.VAPIDPublicKey,
+		options.VAPIDPrivateKey,
+		expiration,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set("Authorization", vapidAuthHeader)
+
+	// Send the request
+	var client HTTPClient
+	if options.HTTPClient != nil {
+		client = options.HTTPClient
+	} else {
+		client = &http.Client{}
+	}
+
+	return client.Do(req)
+}
+
+// decodeSubscriptionKey decodes a base64 subscription key.
+// if necessary, add "=" padding to the key for URL decode
+func decodeSubscriptionKey(key string) ([]byte, error) {
+	// "=" padding
+	buf := bytes.NewBufferString(key)
+	if rem := len(key) % 4; rem != 0 {
+		buf.WriteString(strings.Repeat("=", 4-rem))
+	}
+
+	bytes, err := base64.StdEncoding.DecodeString(buf.String())
+	if err == nil {
+		return bytes, nil
+	}
+
+	return base64.URLEncoding.DecodeString(buf.String())
+}
+
+// Returns a key of length "length" given an hkdf function
+func getHKDFKey(hkdf io.Reader, length int) ([]byte, error) {
+	key := make([]byte, length)
+	n, err := io.ReadFull(hkdf, key)
+	if n != len(key) || err != nil {
+		return key, err
+	}
+
+	return key, nil
+}
+
+func pad(payload *bytes.Buffer, maxPadLen int) error {
+	payloadLen := payload.Len()
+	if payloadLen > maxPadLen {
+		return ErrMaxPadExceeded
+	}
+
+	padLen := maxPadLen - payloadLen
+
+	padding := make([]byte, padLen)
+	payload.Write(padding)
+
+	return nil
+}