mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-10-29 07:22:24 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

[feature] update proof-of-work to allow setting required rounds (#4186)

Browse source 
# Description

This updates our proof-of-work middleware, NoLLaMas, to work on a more easily configurable algorithm (thank you f0x for bringing this to my attention!). Instead of requiring that a solution with pre-determined number of '0' chars be found, it now pre-computes a result with a pre-determined nonce value that it expects the client to iterate up-to. (though with some level of jitter applied, to prevent it being too-easily gamed). This allows the user to configure roughly how many hash-encode rounds they want their clients to have to complete.

## Checklist

- [x] I/we have read the [GoToSocial contribution guidelines](https://codeberg.org/superseriousbusiness/gotosocial/src/branch/main/CONTRIBUTING.md).
- [x] I/we have discussed the proposed changes already, either in an issue on the repository, or in the Matrix chat.
- [x] I/we have not leveraged AI to create the proposed changes.
- [x] I/we have performed a self-review of added code.
- [x] I/we have written code that is legible and maintainable by others.
- [x] I/we have commented the added code, particularly in hard-to-understand areas.
- [x] I/we have made any necessary changes to documentation.
- [ ] I/we have added tests that cover new code.
- [x] I/we have run tests and they pass locally with the changes.
- [x] I/we have run `go fmt ./...` and `golangci-lint run`.

Reviewed-on: https://codeberg.org/superseriousbusiness/gotosocial/pulls/4186
Co-authored-by: kim <grufwub@gmail.com>
Co-committed-by: kim <grufwub@gmail.com>
This commit is contained in:
kim 2025-05-26 11:57:50 +02:00 • committed by tobi
commit 326e04283a
23 changed files with 4350 additions and 160 deletions
Download patch file Download diff file Expand all files Collapse all files

4
internal/config/config.go
View file

@ -280,6 +280,6 @@ type ThrottlingConfig struct {
}
type ScraperDeterrenceConfig struct {
Enabled bool `name:"enabled" usage:"Enable proof-of-work based scraper deterrence on profile / status pages"`
Difficulty uint8 `name:"difficulty" usage:"The proof-of-work difficulty, which determines how many leading zeros to try solve in hash solutions."`
Enabled bool `name:"enabled" usage:"Enable proof-of-work based scraper deterrence on profile / status pages"`
Difficulty uint32 `name:"difficulty" usage:"The proof-of-work difficulty, which determines roughly how many hash-encode rounds required of each client."`
}

2
internal/config/defaults.go
View file

@ -149,7 +149,7 @@ var Defaults = Configuration{
ScraperDeterrence: ScraperDeterrenceConfig{
Enabled: false,
Difficulty: 4,
Difficulty: 100000,
},
},

14
internal/config/helpers.gen.go
View file

@ -144,7 +144,7 @@ func (cfg *Configuration) RegisterFlags(flags *pflag.FlagSet) {
flags.Int("advanced-throttling-multiplier", cfg.Advanced.Throttling.Multiplier, "Multiplier to use per cpu for http request throttling. 0 or less turns throttling off.")
flags.Duration("advanced-throttling-retry-after", cfg.Advanced.Throttling.RetryAfter, "Retry-After duration response to send for throttled requests.")
flags.Bool("advanced-scraper-deterrence-enabled", cfg.Advanced.ScraperDeterrence.Enabled, "Enable proof-of-work based scraper deterrence on profile / status pages")
flags.Uint8("advanced-scraper-deterrence-difficulty", cfg.Advanced.ScraperDeterrence.Difficulty, "The proof-of-work difficulty, which determines how many leading zeros to try solve in hash solutions.")
flags.Uint32("advanced-scraper-deterrence-difficulty", cfg.Advanced.ScraperDeterrence.Difficulty, "The proof-of-work difficulty, which determines how many leading zeros to try solve in hash solutions.")
flags.StringSlice("http-client-allow-ips", cfg.HTTPClient.AllowIPs, "")
flags.StringSlice("http-client-block-ips", cfg.HTTPClient.BlockIPs, "")
flags.Duration("http-client-timeout", cfg.HTTPClient.Timeout, "")
@ -1356,9 +1356,9 @@ func (cfg *Configuration) UnmarshalMap(cfgmap map[string]any) error {
if ival, ok := cfgmap["advanced-scraper-deterrence-difficulty"]; ok {
var err error
cfg.Advanced.ScraperDeterrence.Difficulty, err = cast.ToUint8E(ival)
cfg.Advanced.ScraperDeterrence.Difficulty, err = cast.ToUint32E(ival)
if err != nil {
return fmt.Errorf("error casting %#v -> uint8 for 'advanced-scraper-deterrence-difficulty': %w", ival, err)
return fmt.Errorf("error casting %#v -> uint32 for 'advanced-scraper-deterrence-difficulty': %w", ival, err)
}
}
@ -4799,7 +4799,7 @@ func AdvancedScraperDeterrenceDifficultyFlag() string {
}
// GetAdvancedScraperDeterrenceDifficulty safely fetches the Configuration value for state's 'Advanced.ScraperDeterrence.Difficulty' field
func (st *ConfigState) GetAdvancedScraperDeterrenceDifficulty() (v uint8) {
func (st *ConfigState) GetAdvancedScraperDeterrenceDifficulty() (v uint32) {
st.mutex.RLock()
v = st.config.Advanced.ScraperDeterrence.Difficulty
st.mutex.RUnlock()
@ -4807,7 +4807,7 @@ func (st *ConfigState) GetAdvancedScraperDeterrenceDifficulty() (v uint8) {
}
// SetAdvancedScraperDeterrenceDifficulty safely sets the Configuration value for state's 'Advanced.ScraperDeterrence.Difficulty' field
func (st *ConfigState) SetAdvancedScraperDeterrenceDifficulty(v uint8) {
func (st *ConfigState) SetAdvancedScraperDeterrenceDifficulty(v uint32) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.Advanced.ScraperDeterrence.Difficulty = v
@ -4815,12 +4815,12 @@ func (st *ConfigState) SetAdvancedScraperDeterrenceDifficulty(v uint8) {
}
// GetAdvancedScraperDeterrenceDifficulty safely fetches the value for global configuration 'Advanced.ScraperDeterrence.Difficulty' field
func GetAdvancedScraperDeterrenceDifficulty() uint8 {
func GetAdvancedScraperDeterrenceDifficulty() uint32 {
return global.GetAdvancedScraperDeterrenceDifficulty()
}
// SetAdvancedScraperDeterrenceDifficulty safely sets the value for global configuration 'Advanced.ScraperDeterrence.Difficulty' field
func SetAdvancedScraperDeterrenceDifficulty(v uint8) {
func SetAdvancedScraperDeterrenceDifficulty(v uint32) {
global.SetAdvancedScraperDeterrenceDifficulty(v)
}

238
internal/middleware/nollamas.go
View file

@ -26,6 +26,7 @@ import (
"hash"
"io"
"net/http"
"strconv"
"time"
apimodel "code.superseriousbusiness.org/gotosocial/internal/api/model"
@ -35,6 +36,7 @@ import (
"code.superseriousbusiness.org/gotosocial/internal/gtserror"
"code.superseriousbusiness.org/gotosocial/internal/log"
"code.superseriousbusiness.org/gotosocial/internal/oauth"
"codeberg.org/gruf/go-bitutil"
"codeberg.org/gruf/go-byteutil"
"github.com/gin-gonic/gin"
)
@ -60,49 +62,79 @@ func NoLLaMas(
return func(*gin.Context) {}
}
seed := make([]byte, 32)
var seed [32]byte
// Read random data for the token seed.
_, err := io.ReadFull(rand.Reader, seed)
_, err := io.ReadFull(rand.Reader, seed[:])
if err != nil {
panic(err)
}
// Configure nollamas.
var nollamas nollamas
nollamas.seed = seed
nollamas.entropy = seed
nollamas.ttl = time.Hour
nollamas.diff = config.GetAdvancedScraperDeterrenceDifficulty()
nollamas.rounds = config.GetAdvancedScraperDeterrenceDifficulty()
nollamas.getInstanceV1 = getInstanceV1
nollamas.policy = cookiePolicy
return nollamas.Serve
}
// i.e. hash slice length.
const hashLen = sha256.Size
// i.e. hex.EncodedLen(hashLen).
const encodedHashLen = 2 * hashLen
// hashWithBufs encompasses a hash along
// with the necessary buffers to generate
// a hashsum and then encode that sum.
type hashWithBufs struct {
hash hash.Hash
hbuf []byte
ebuf []byte
hbuf [hashLen]byte
ebuf [encodedHashLen]byte
}
// write is a passthrough to hash.Hash{}.Write().
func (h *hashWithBufs) write(b []byte) {
_, _ = h.hash.Write(b)
}
// writeString is a passthrough to hash.Hash{}.Write([]byte(s)).
func (h *hashWithBufs) writeString(s string) {
_, _ = h.hash.Write(byteutil.S2B(s))
}
// EncodedSum returns the hex encoded sum of hash.Sum().
func (h *hashWithBufs) EncodedSum() string {
_ = h.hash.Sum(h.hbuf[:0])
hex.Encode(h.ebuf[:], h.hbuf[:])
return string(h.ebuf[:])
}
// Reset will reset hash and buffers.
func (h *hashWithBufs) Reset() {
h.ebuf = [encodedHashLen]byte{}
h.hbuf = [hashLen]byte{}
h.hash.Reset()
}
type nollamas struct {
// our instance cookie policy.
policy apiutil.CookiePolicy
// unique token seed
// unique entropy
// to prevent hashes
// being guessable
seed []byte
entropy [32]byte
// success cookie TTL
ttl time.Duration
// algorithm difficulty knobs.
// diff determines the number
// of leading zeroes required.
diff uint8
// rounds determines roughly how
// many hash-encode rounds each
// client is required to complete.
rounds uint32
// extra fields required for
// our template rendering.
@ -134,18 +166,8 @@ func (m *nollamas) Serve(c *gin.Context) {
return
}
// i.e. outputted hash slice length.
const hashLen = sha256.Size
// i.e. hex.EncodedLen(hashLen).
const encodedHashLen = 2 * hashLen
// Prepare hash + buffers.
hash := hashWithBufs{
hash: sha256.New(),
hbuf: make([]byte, 0, hashLen),
ebuf: make([]byte, encodedHashLen),
}
// Prepare new hash with buffers.
hash := hashWithBufs{hash: sha256.New()}
// Extract client fingerprint data.
userAgent := c.GetHeader("User-Agent")
@ -153,15 +175,7 @@ func (m *nollamas) Serve(c *gin.Context) {
// Generate a unique token for this request,
// only valid for a period of now +- m.ttl.
token := m.token(&hash, userAgent, clientIP)
// For unique challenge string just use a
// single portion of their 'success' token.
// SHA256 is not yet cracked, this is not an
// application of a hash requiring serious
// cryptographic security and it rotates on
// a TTL basis, so it should be fine.
challenge := token[:len(token)/4]
token := m.getToken(&hash, userAgent, clientIP)
// Check for a provided success token.
cookie, _ := c.Cookie("gts-nollamas")
@ -169,8 +183,8 @@ func (m *nollamas) Serve(c *gin.Context) {
// Check whether passed cookie
// is the expected success token.
if subtle.ConstantTimeCompare(
byteutil.S2B(token),
byteutil.S2B(cookie),
byteutil.S2B(token),
) == 1 {
// They passed us a valid, expected
@ -185,10 +199,15 @@ func (m *nollamas) Serve(c *gin.Context) {
// handlers from being called.
c.Abort()
// Generate challenge for this unique (yet deterministic) token,
// returning seed, wanted 'challenge' result and expected solution.
seed, challenge, solution := m.getChallenge(&hash, token)
// Prepare new log entry.
l := log.WithContext(ctx).
WithField("userAgent", userAgent).
WithField("challenge", challenge)
WithField("seed", seed).
WithField("rounds", solution)
// Extract and parse query.
query := c.Request.URL.Query()
@ -196,32 +215,28 @@ func (m *nollamas) Serve(c *gin.Context) {
// Check query to see if an in-progress
// challenge solution has been provided.
nonce := query.Get("nollamas_solution")
if nonce == "" || len(nonce) > 20 {
if nonce == "" {
// noting that here, 20 is
// max integer string len.
//
// An invalid solution string, just
// present them with new challenge.
// No solution given, likely new client!
// Simply present them with challenge.
m.renderChallenge(c, seed, challenge)
l.Info("posing new challenge")
m.renderChallenge(c, challenge)
return
}
// Reset the hash.
hash.hash.Reset()
// Check nonce matches expected.
if subtle.ConstantTimeCompare(
byteutil.S2B(solution),
byteutil.S2B(nonce),
) != 1 {
// Check challenge+nonce as possible solution.
if !m.checkChallenge(&hash, challenge, nonce) {
// They failed challenge,
// re-present challenge page.
l.Info("invalid solution provided")
m.renderChallenge(c, challenge)
// Their nonce failed, re-challenge them.
m.renderChallenge(c, challenge, solution)
l.Infof("invalid solution provided: %s", nonce)
return
}
l.Infof("challenge passed: %s", nonce)
l.Info("challenge passed")
// Drop solution query and encode.
query.Del("nollamas_solution")
@ -233,7 +248,7 @@ func (m *nollamas) Serve(c *gin.Context) {
c.Redirect(http.StatusTemporaryRedirect, c.Request.URL.RequestURI())
}
func (m *nollamas) renderChallenge(c *gin.Context, challenge string) {
func (m *nollamas) renderChallenge(c *gin.Context, seed, challenge string) {
// Fetch current instance information for templating vars.
instance, errWithCode := m.getInstanceV1(c.Request.Context())
if errWithCode != nil {
@ -252,8 +267,8 @@ func (m *nollamas) renderChallenge(c *gin.Context, challenge string) {
"/assets/Fork-Awesome/css/fork-awesome.min.css",
},
Extra: map[string]any{
"challenge": challenge,
"difficulty": m.diff,
"seed": seed,
"challenge": challenge,
},
Javascript: []apiutil.JavascriptEntry{
{
@ -264,23 +279,25 @@ func (m *nollamas) renderChallenge(c *gin.Context, challenge string) {
})
}
func (m *nollamas) token(hash *hashWithBufs, userAgent, clientIP string) string {
// Use our unique seed to seed hash,
// getToken generates a unique yet deterministic token for given HTTP request
// details, seeded by runtime generated entropy data and ttl rounded timestamp.
func (m *nollamas) getToken(hash *hashWithBufs, userAgent, clientIP string) string {
// Reset before
// using hash.
hash.Reset()
// Use our unique entropy to seed hash,
// to ensure we have cryptographically
// unique, yet deterministic, tokens
// generated for a given http client.
hash.hash.Write(m.seed)
// Include difficulty level in
// hash input data so if config
// changes then token invalidates.
hash.hash.Write([]byte{m.diff})
hash.write(m.entropy[:])
// Also seed the generated input with
// current time rounded to TTL, so our
// single comparison handles expiries.
now := time.Now().Round(m.ttl).Unix()
hash.hash.Write([]byte{
hash.write([]byte{
byte(now >> 56),
byte(now >> 48),
byte(now >> 40),
@ -291,37 +308,78 @@ func (m *nollamas) token(hash *hashWithBufs, userAgent, clientIP string) string
byte(now),
})
// Finally, append unique client request data.
hash.hash.Write(byteutil.S2B(userAgent))
hash.hash.Write(byteutil.S2B(clientIP))
// Append client request data.
hash.writeString(userAgent)
hash.writeString(clientIP)
// Return hex encoded hash output.
hash.hbuf = hash.hash.Sum(hash.hbuf[:0])
hex.Encode(hash.ebuf, hash.hbuf)
return string(hash.ebuf)
// Return hex encoded hash.
return hash.EncodedSum()
}
func (m *nollamas) checkChallenge(hash *hashWithBufs, challenge, nonce string) bool {
// Hash and encode input challenge with
// proposed nonce as a possible solution.
hash.hash.Write(byteutil.S2B(challenge))
hash.hash.Write(byteutil.S2B(nonce))
hash.hbuf = hash.hash.Sum(hash.hbuf[:0])
hex.Encode(hash.ebuf, hash.hbuf)
solution := hash.ebuf
// getChallenge prepares a new challenge given the deterministic input token for this request.
// it will return an input seed string, a challenge string which is the end result the client
// should be looking for, and the solution for this such that challenge = hex(sha256(seed + solution)).
// the solution will always be a string-encoded 64bit integer calculated from m.rounds + random jitter.
func (m *nollamas) getChallenge(hash *hashWithBufs, token string) (seed, challenge, solution string) {
// Compiler bound-check hint.
if len(solution) < int(m.diff) {
panic(gtserror.New("BCE"))
// For their unique seed string just use a
// single portion of their 'success' token.
// SHA256 is not yet cracked, this is not an
// application of a hash requiring serious
// cryptographic security and it rotates on
// a TTL basis, so it should be fine.
seed = token[:len(token)/4]
// BEFORE resetting the hash, get the last
// two bytes of NON-hex-encoded data from
// token generation to use for random jitter.
// This is taken from the end of the hash as
// this is the "unseen" end part of token.
//
// (if we used hex-encoded data it would
// only ever be '0-9' or 'a-z' ASCII chars).
//
// Security-wise, same applies as-above.
jitter := int16(hash.hbuf[len(hash.hbuf)-2]) |
int16(hash.hbuf[len(hash.hbuf)-1])<<8
var rounds int64
switch {
// For some small percentage of
// clients we purposely low-ball
// their rounds required, to make
// it so gaming it with a starting
// nonce value may suddenly fail.
case jitter%37 == 0:
rounds = int64(m.rounds/10) + int64(jitter/10)
case jitter%31 == 0:
rounds = int64(m.rounds/5) + int64(jitter/5)
case jitter%29 == 0:
rounds = int64(m.rounds/3) + int64(jitter/3)
case jitter%13 == 0:
rounds = int64(m.rounds/2) + int64(jitter/2)
// Determine an appropriate number of hash rounds
// we want the client to perform on input seed. This
// is determined as configured m.rounds +- jitter.
// This will be the 'solution' to create 'challenge'.
default:
rounds = int64(m.rounds) + int64(jitter) //nolint:gosec
}
// Check that the first 'diff'
// many chars are indeed zeroes.
for i := range m.diff {
if solution[i] != '0' {
return false
}
}
// Encode (positive) determined hash rounds as string.
solution = strconv.FormatInt(bitutil.Abs64(rounds), 10)
return true
// Reset before
// using hash.
hash.Reset()
// Calculate the expected result
// of hex(sha256(seed + solution)),
// i.e. the proposed 'challenge'.
hash.writeString(seed)
hash.writeString(solution)
challenge = hash.EncodedSum()
return
}

35
internal/middleware/nollamas_test.go
View file

@ -95,41 +95,39 @@ func testNoLLaMasMiddleware(t *testing.T, e *gin.Engine, userAgent string) {
panic(err)
}
var seed string
var challenge string
var difficulty uint64
// Parse output body and find the challenge / difficulty.
for _, line := range strings.Split(string(b), "\n") {
line = strings.TrimSpace(line)
switch {
case strings.HasPrefix(line, "data-nollamas-seed=\""):
line = line[20:]
line = line[:len(line)-1]
seed = line
case strings.HasPrefix(line, "data-nollamas-challenge=\""):
line = line[25:]
line = line[:len(line)-1]
challenge = line
case strings.HasPrefix(line, "data-nollamas-difficulty=\""):
line = line[26:]
line = line[:len(line)-1]
var err error
difficulty, err = strconv.ParseUint(line, 10, 8)
assert.NoError(t, err)
}
}
// Ensure valid posed challenge.
assert.NotZero(t, difficulty)
assert.NotEmpty(t, challenge)
assert.NotEmpty(t, seed)
// Prepare a test request for gin engine.
r = httptest.NewRequest("GET", "/", nil)
r.Header.Set("User-Agent", userAgent)
rw = httptest.NewRecorder()
// Now compute and set solution query paramater.
solution := computeSolution(challenge, difficulty)
r.URL.RawQuery = "nollamas_solution=" + solution
t.Logf("seed=%s", seed)
t.Logf("challenge=%s", challenge)
t.Logf("difficulty=%d", difficulty)
// Now compute and set solution query paramater.
solution := computeSolution(seed, challenge)
r.URL.RawQuery = "nollamas_solution=" + solution
t.Logf("solution=%s", solution)
// Pass req through
@ -152,17 +150,14 @@ func testNoLLaMasMiddleware(t *testing.T, e *gin.Engine, userAgent string) {
}
// computeSolution does the functional equivalent of our nollamas workerTask.js.
func computeSolution(challenge string, diff uint64) string {
outer:
func computeSolution(seed, challenge string) string {
for i := 0; ; i++ {
solution := strconv.Itoa(i)
combined := challenge + solution
combined := seed + solution
hash := sha256.Sum256(byteutil.S2B(combined))
encoded := hex.EncodeToString(hash[:])
for i := range diff {
if encoded[i] != '0' {
continue outer
}
if encoded != challenge {
continue
}
return solution
}