mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-11-28 04:23:32 -06:00
Code Issues Projects Releases Packages Wiki Activity Actions

[chore] set max open / idle conns + conn max lifetime for both postgres and sqlite (#1369)

Browse source 
* [chore] set max open / idle conns + conn max lifetime for both postgres and sqlite

* reduce cache size default to 8MiB, reduce connections to 2 * cpu

* introduce max open conns multiplier, tune sqlite and pg separately

* go fmt
This commit is contained in:
tobi 2023-01-26 15:12:48 +01:00 • committed by GitHub
commit 782169da76
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 212 additions and 121 deletions
Download patch file Download diff file Expand all files Collapse all files

25
internal/config/config.go
View file

@ -58,18 +58,19 @@ type Configuration struct {
TrustedProxies []string `name:"trusted-proxies" usage:"Proxies to trust when parsing x-forwarded headers into real IPs."`
SoftwareVersion string `name:"software-version" usage:""`
DbType string `name:"db-type" usage:"Database type: eg., postgres"`
DbAddress string `name:"db-address" usage:"Database ipv4 address, hostname, or filename"`
DbPort int `name:"db-port" usage:"Database port"`
DbUser string `name:"db-user" usage:"Database username"`
DbPassword string `name:"db-password" usage:"Database password"`
DbDatabase string `name:"db-database" usage:"Database name"`
DbTLSMode string `name:"db-tls-mode" usage:"Database tls mode"`
DbTLSCACert string `name:"db-tls-ca-cert" usage:"Path to CA cert for db tls connection"`
DbSqliteJournalMode string `name:"db-sqlite-journal-mode" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_journal_mode"`
DbSqliteSynchronous string `name:"db-sqlite-synchronous" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_synchronous"`
DbSqliteCacheSize bytesize.Size `name:"db-sqlite-cache-size" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_cache_size"`
DbSqliteBusyTimeout time.Duration `name:"db-sqlite-busy-timeout" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_busy_timeout"`
DbType string `name:"db-type" usage:"Database type: eg., postgres"`
DbAddress string `name:"db-address" usage:"Database ipv4 address, hostname, or filename"`
DbPort int `name:"db-port" usage:"Database port"`
DbUser string `name:"db-user" usage:"Database username"`
DbPassword string `name:"db-password" usage:"Database password"`
DbDatabase string `name:"db-database" usage:"Database name"`
DbTLSMode string `name:"db-tls-mode" usage:"Database tls mode"`
DbTLSCACert string `name:"db-tls-ca-cert" usage:"Path to CA cert for db tls connection"`
DbMaxOpenConnsMultiplier int `name:"db-max-open-conns-multiplier" usage:"Multiplier to use per cpu for max open database connections. 0 or less is normalized to 1."`
DbSqliteJournalMode string `name:"db-sqlite-journal-mode" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_journal_mode"`
DbSqliteSynchronous string `name:"db-sqlite-synchronous" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_synchronous"`
DbSqliteCacheSize bytesize.Size `name:"db-sqlite-cache-size" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_cache_size"`
DbSqliteBusyTimeout time.Duration `name:"db-sqlite-busy-timeout" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_busy_timeout"`
WebTemplateBaseDir string `name:"web-template-base-dir" usage:"Basedir for html templating files for rendering pages and composing emails."`
WebAssetBaseDir string `name:"web-asset-base-dir" usage:"Directory to serve static assets from, accessible at example.org/assets/"`

25
internal/config/defaults.go
View file

@ -40,18 +40,19 @@ var Defaults = Configuration{
Port: 8080,
TrustedProxies: []string{"127.0.0.1/32", "::1"}, // localhost
DbType: "postgres",
DbAddress: "",
DbPort: 5432,
DbUser: "",
DbPassword: "",
DbDatabase: "gotosocial",
DbTLSMode: "disable",
DbTLSCACert: "",
DbSqliteJournalMode: "WAL",
DbSqliteSynchronous: "NORMAL",
DbSqliteCacheSize: 64 * bytesize.MiB,
DbSqliteBusyTimeout: time.Minute * 5,
DbType: "postgres",
DbAddress: "",
DbPort: 5432,
DbUser: "",
DbPassword: "",
DbDatabase: "gotosocial",
DbTLSMode: "disable",
DbTLSCACert: "",
DbMaxOpenConnsMultiplier: 8,
DbSqliteJournalMode: "WAL",
DbSqliteSynchronous: "NORMAL",
DbSqliteCacheSize: 8 * bytesize.MiB,
DbSqliteBusyTimeout: time.Minute * 5,
WebTemplateBaseDir: "./web/template/",
WebAssetBaseDir: "./web/assets/",

1
internal/config/flags.go
View file

@ -51,6 +51,7 @@ func (s *ConfigState) AddGlobalFlags(cmd *cobra.Command) {
cmd.PersistentFlags().String(DbDatabaseFlag(), cfg.DbDatabase, fieldtag("DbDatabase", "usage"))
cmd.PersistentFlags().String(DbTLSModeFlag(), cfg.DbTLSMode, fieldtag("DbTLSMode", "usage"))
cmd.PersistentFlags().String(DbTLSCACertFlag(), cfg.DbTLSCACert, fieldtag("DbTLSCACert", "usage"))
cmd.PersistentFlags().Int(DbMaxOpenConnsMultiplierFlag(), cfg.DbMaxOpenConnsMultiplier, fieldtag("DbMaxOpenConnsMultiplier", "usage"))
cmd.PersistentFlags().String(DbSqliteJournalModeFlag(), cfg.DbSqliteJournalMode, fieldtag("DbSqliteJournalMode", "usage"))
cmd.PersistentFlags().String(DbSqliteSynchronousFlag(), cfg.DbSqliteSynchronous, fieldtag("DbSqliteSynchronous", "usage"))
cmd.PersistentFlags().Uint64(DbSqliteCacheSizeFlag(), uint64(cfg.DbSqliteCacheSize), fieldtag("DbSqliteCacheSize", "usage"))

25
internal/config/helpers.gen.go
View file

@ -524,6 +524,31 @@ func GetDbTLSCACert() string { return global.GetDbTLSCACert() }
// SetDbTLSCACert safely sets the value for global configuration 'DbTLSCACert' field
func SetDbTLSCACert(v string) { global.SetDbTLSCACert(v) }
// GetDbMaxOpenConnsMultiplier safely fetches the Configuration value for state's 'DbMaxOpenConnsMultiplier' field
func (st *ConfigState) GetDbMaxOpenConnsMultiplier() (v int) {
st.mutex.Lock()
v = st.config.DbMaxOpenConnsMultiplier
st.mutex.Unlock()
return
}
// SetDbMaxOpenConnsMultiplier safely sets the Configuration value for state's 'DbMaxOpenConnsMultiplier' field
func (st *ConfigState) SetDbMaxOpenConnsMultiplier(v int) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.DbMaxOpenConnsMultiplier = v
st.reloadToViper()
}
// DbMaxOpenConnsMultiplierFlag returns the flag name for the 'DbMaxOpenConnsMultiplier' field
func DbMaxOpenConnsMultiplierFlag() string { return "db-max-open-conns-multiplier" }
// GetDbMaxOpenConnsMultiplier safely fetches the value for global configuration 'DbMaxOpenConnsMultiplier' field
func GetDbMaxOpenConnsMultiplier() int { return global.GetDbMaxOpenConnsMultiplier() }
// SetDbMaxOpenConnsMultiplier safely sets the value for global configuration 'DbMaxOpenConnsMultiplier' field
func SetDbMaxOpenConnsMultiplier(v int) { global.SetDbMaxOpenConnsMultiplier(v) }
// GetDbSqliteJournalMode safely fetches the Configuration value for state's 'DbSqliteJournalMode' field
func (st *ConfigState) GetDbSqliteJournalMode() (v string) {
st.mutex.Lock()

167
internal/db/bundb/bundb.go
View file

@ -222,6 +222,32 @@ func NewBunDBService(ctx context.Context, state *state.State) (db.DB, error) {
return ps, nil
}
func pgConn(ctx context.Context) (*DBConn, error) {
opts, err := deriveBunDBPGOptions() //nolint:contextcheck
if err != nil {
return nil, fmt.Errorf("could not create bundb postgres options: %s", err)
}
sqldb := stdlib.OpenDB(*opts)
// Tune db connections for postgres, see:
// - https://bun.uptrace.dev/guide/running-bun-in-production.html#database-sql
// - https://www.alexedwards.net/blog/configuring-sqldb
sqldb.SetMaxOpenConns(maxOpenConns()) // x number of conns per CPU
sqldb.SetMaxIdleConns(2) // assume default 2; if max idle is less than max open, it will be automatically adjusted
sqldb.SetConnMaxLifetime(5 * time.Minute) // fine to kill old connections
conn := WrapDBConn(bun.NewDB(sqldb, pgdialect.New()))
// ping to check the db is there and listening
if err := conn.PingContext(ctx); err != nil {
return nil, fmt.Errorf("postgres ping: %s", err)
}
log.Info("connected to POSTGRES database")
return conn, nil
}
func sqliteConn(ctx context.Context) (*DBConn, error) {
// validate db address has actually been set
address := config.GetDbAddress()
@ -236,13 +262,10 @@ func sqliteConn(ctx context.Context) (*DBConn, error) {
// Append our own SQLite preferences
address = "file:" + address
var inMem bool
if address == "file::memory:" {
address = fmt.Sprintf("file:%s?mode=memory&cache=shared", uuid.NewString())
log.Infof("using in-memory database address " + address)
log.Warn("sqlite in-memory database should only be used for debugging")
inMem = true
}
// Open new DB instance
@ -254,12 +277,12 @@ func sqliteConn(ctx context.Context) (*DBConn, error) {
return nil, fmt.Errorf("could not open sqlite db: %s", err)
}
if inMem {
// don't close connections on disconnect -- otherwise
// the SQLite database will be deleted when there
// are no active connections
sqldb.SetConnMaxLifetime(0)
}
// Tune db connections for sqlite, see:
// - https://bun.uptrace.dev/guide/running-bun-in-production.html#database-sql
// - https://www.alexedwards.net/blog/configuring-sqldb
sqldb.SetMaxOpenConns(maxOpenConns()) // x number of conns per cpu
sqldb.SetMaxIdleConns(1) // only keep max 1 idle connection around
sqldb.SetConnMaxLifetime(0) // don't kill connections due to age
// Wrap Bun database conn in our own wrapper
conn := WrapDBConn(bun.NewDB(sqldb, sqlitedialect.New()))
@ -276,79 +299,20 @@ func sqliteConn(ctx context.Context) (*DBConn, error) {
return conn, nil
}
func sqlitePragmas(ctx context.Context, conn *DBConn) error {
var pragmas [][]string
if mode := config.GetDbSqliteJournalMode(); mode != "" {
// Set the user provided SQLite journal mode
pragmas = append(pragmas, []string{"journal_mode", mode})
}
if mode := config.GetDbSqliteSynchronous(); mode != "" {
// Set the user provided SQLite synchronous mode
pragmas = append(pragmas, []string{"synchronous", mode})
}
if size := config.GetDbSqliteCacheSize(); size > 0 {
// Set the user provided SQLite cache size (in kibibytes)
// Prepend a '-' character to this to indicate to sqlite
// that we're giving kibibytes rather than num pages.
// https://www.sqlite.org/pragma.html#pragma_cache_size
s := "-" + strconv.FormatUint(uint64(size/bytesize.KiB), 10)
pragmas = append(pragmas, []string{"cache_size", s})
}
if timeout := config.GetDbSqliteBusyTimeout(); timeout > 0 {
t := strconv.FormatInt(timeout.Milliseconds(), 10)
pragmas = append(pragmas, []string{"busy_timeout", t})
}
for _, p := range pragmas {
pk := p[0]
pv := p[1]
if _, err := conn.DB.ExecContext(ctx, "PRAGMA ?=?", bun.Ident(pk), bun.Safe(pv)); err != nil {
return fmt.Errorf("error executing sqlite pragma %s: %w", pk, err)
}
var res string
if err := conn.DB.NewRaw("PRAGMA ?", bun.Ident(pk)).Scan(ctx, &res); err != nil {
return fmt.Errorf("error scanning sqlite pragma %s: %w", pv, err)
}
log.Infof("sqlite pragma %s set to %s", pk, res)
}
return nil
}
func pgConn(ctx context.Context) (*DBConn, error) {
opts, err := deriveBunDBPGOptions() //nolint:contextcheck
if err != nil {
return nil, fmt.Errorf("could not create bundb postgres options: %s", err)
}
sqldb := stdlib.OpenDB(*opts)
// https://bun.uptrace.dev/postgres/running-bun-in-production.html#database-sql
maxOpenConns := 4 * runtime.GOMAXPROCS(0)
sqldb.SetMaxOpenConns(maxOpenConns)
sqldb.SetMaxIdleConns(maxOpenConns)
conn := WrapDBConn(bun.NewDB(sqldb, pgdialect.New()))
// ping to check the db is there and listening
if err := conn.PingContext(ctx); err != nil {
return nil, fmt.Errorf("postgres ping: %s", err)
}
log.Info("connected to POSTGRES database")
return conn, nil
}
/*
HANDY STUFF
*/
// maxOpenConns returns multiplier * GOMAXPROCS,
// clamping multiplier to 1 if it was below 1.
func maxOpenConns() int {
multiplier := config.GetDbMaxOpenConnsMultiplier()
if multiplier < 1 {
multiplier = 1
}
return multiplier * runtime.GOMAXPROCS(0)
}
// deriveBunDBPGOptions takes an application config and returns either a ready-to-use set of options
// with sensible defaults, or an error if it's not satisfied by the provided config.
func deriveBunDBPGOptions() (*pgx.ConnConfig, error) {
@ -434,6 +398,53 @@ func deriveBunDBPGOptions() (*pgx.ConnConfig, error) {
return cfg, nil
}
// sqlitePragmas sets desired sqlite pragmas based on configured values, and
// logs the results of the pragma queries. Errors if something goes wrong.
func sqlitePragmas(ctx context.Context, conn *DBConn) error {
var pragmas [][]string
if mode := config.GetDbSqliteJournalMode(); mode != "" {
// Set the user provided SQLite journal mode
pragmas = append(pragmas, []string{"journal_mode", mode})
}
if mode := config.GetDbSqliteSynchronous(); mode != "" {
// Set the user provided SQLite synchronous mode
pragmas = append(pragmas, []string{"synchronous", mode})
}
if size := config.GetDbSqliteCacheSize(); size > 0 {
// Set the user provided SQLite cache size (in kibibytes)
// Prepend a '-' character to this to indicate to sqlite
// that we're giving kibibytes rather than num pages.
// https://www.sqlite.org/pragma.html#pragma_cache_size
s := "-" + strconv.FormatUint(uint64(size/bytesize.KiB), 10)
pragmas = append(pragmas, []string{"cache_size", s})
}
if timeout := config.GetDbSqliteBusyTimeout(); timeout > 0 {
t := strconv.FormatInt(timeout.Milliseconds(), 10)
pragmas = append(pragmas, []string{"busy_timeout", t})
}
for _, p := range pragmas {
pk := p[0]
pv := p[1]
if _, err := conn.DB.ExecContext(ctx, "PRAGMA ?=?", bun.Ident(pk), bun.Safe(pv)); err != nil {
return fmt.Errorf("error executing sqlite pragma %s: %w", pk, err)
}
var res string
if err := conn.DB.NewRaw("PRAGMA ?", bun.Ident(pk)).Scan(ctx, &res); err != nil {
return fmt.Errorf("error scanning sqlite pragma %s: %w", pv, err)
}
log.Infof("sqlite pragma %s set to %s", pk, res)
}
return nil
}
/*
CONVERSION FUNCTIONS
*/