[chore] update dependencies (#4468)

- github.com/ncruces/go-sqlite3
- codeberg.org/gruf/go-mempool
- codeberg.org/gruf/go-structr (changes related on the above) *
- codeberg.org/gruf/go-mutexes (changes related on the above) *

* this is largely just fiddling around with package internals in structr and mutexes to rely on changes in mempool, which added a new concurrency-safe pool

Reviewed-on: https://codeberg.org/superseriousbusiness/gotosocial/pulls/4468
Co-authored-by: kim <grufwub@gmail.com>
Co-committed-by: kim <grufwub@gmail.com>
This commit is contained in:
kim 2025-10-03 15:29:41 +02:00 committed by kim
commit ff950e94bb
32 changed files with 706 additions and 317 deletions

8
go.mod
View file

@ -25,13 +25,13 @@ require (
codeberg.org/gruf/go-iotools v0.0.0-20240710125620-934ae9c654cf codeberg.org/gruf/go-iotools v0.0.0-20240710125620-934ae9c654cf
codeberg.org/gruf/go-kv/v2 v2.0.7 codeberg.org/gruf/go-kv/v2 v2.0.7
codeberg.org/gruf/go-list v0.0.0-20240425093752-494db03d641f codeberg.org/gruf/go-list v0.0.0-20240425093752-494db03d641f
codeberg.org/gruf/go-mempool v0.0.0-20240507125005-cef10d64a760 codeberg.org/gruf/go-mempool v0.0.0-20251003110531-b54adae66253
codeberg.org/gruf/go-mutexes v1.5.3 codeberg.org/gruf/go-mutexes v1.5.8
codeberg.org/gruf/go-runners v1.6.3 codeberg.org/gruf/go-runners v1.6.3
codeberg.org/gruf/go-sched v1.2.4 codeberg.org/gruf/go-sched v1.2.4
codeberg.org/gruf/go-split v1.2.0 codeberg.org/gruf/go-split v1.2.0
codeberg.org/gruf/go-storage v0.3.1 codeberg.org/gruf/go-storage v0.3.1
codeberg.org/gruf/go-structr v0.9.9 codeberg.org/gruf/go-structr v0.9.12
github.com/DmitriyVTitov/size v1.5.0 github.com/DmitriyVTitov/size v1.5.0
github.com/KimMachineGun/automemlimit v0.7.4 github.com/KimMachineGun/automemlimit v0.7.4
github.com/SherClockHolmes/webpush-go v1.4.0 github.com/SherClockHolmes/webpush-go v1.4.0
@ -53,7 +53,7 @@ require (
github.com/miekg/dns v1.1.68 github.com/miekg/dns v1.1.68
github.com/minio/minio-go/v7 v7.0.95 github.com/minio/minio-go/v7 v7.0.95
github.com/mitchellh/mapstructure v1.5.0 github.com/mitchellh/mapstructure v1.5.0
github.com/ncruces/go-sqlite3 v0.29.0 github.com/ncruces/go-sqlite3 v0.29.1
github.com/oklog/ulid v1.3.1 github.com/oklog/ulid v1.3.1
github.com/pquerna/otp v1.5.0 github.com/pquerna/otp v1.5.0
github.com/rivo/uniseg v0.4.7 github.com/rivo/uniseg v0.4.7

16
go.sum generated
View file

@ -42,10 +42,10 @@ codeberg.org/gruf/go-mangler/v2 v2.0.6 h1:c3cwnI6Mi17EAwGSYGNMN6+9PMzaIj2GLAKx9D
codeberg.org/gruf/go-mangler/v2 v2.0.6/go.mod h1:CXIm7zAWPdNmZVAGM1NRiF/ekJTPE7YTb8kiRxiEFaQ= codeberg.org/gruf/go-mangler/v2 v2.0.6/go.mod h1:CXIm7zAWPdNmZVAGM1NRiF/ekJTPE7YTb8kiRxiEFaQ=
codeberg.org/gruf/go-maps v1.0.4 h1:K+Ww4vvR3TZqm5jqrKVirmguZwa3v1VUvmig2SE8uxY= codeberg.org/gruf/go-maps v1.0.4 h1:K+Ww4vvR3TZqm5jqrKVirmguZwa3v1VUvmig2SE8uxY=
codeberg.org/gruf/go-maps v1.0.4/go.mod h1:ASX7osM7kFwt5O8GfGflcFjrwYGD8eIuRLl/oMjhEi8= codeberg.org/gruf/go-maps v1.0.4/go.mod h1:ASX7osM7kFwt5O8GfGflcFjrwYGD8eIuRLl/oMjhEi8=
codeberg.org/gruf/go-mempool v0.0.0-20240507125005-cef10d64a760 h1:m2/UCRXhjDwAg4vyji6iKCpomKw6P4PmBOUi5DvAMH4= codeberg.org/gruf/go-mempool v0.0.0-20251003110531-b54adae66253 h1:qPAY72xCWlySVROSNZecfLGAyeV/SiXmPmfhUU+o3Xw=
codeberg.org/gruf/go-mempool v0.0.0-20240507125005-cef10d64a760/go.mod h1:E3RcaCFNq4zXpvaJb8lfpPqdUAmSkP5F1VmMiEUYTEk= codeberg.org/gruf/go-mempool v0.0.0-20251003110531-b54adae66253/go.mod h1:761koiXmqfgzvu5mez2Rk7YlwWilpqJ/zv5hIA6NoNI=
codeberg.org/gruf/go-mutexes v1.5.3 h1:RIEy1UuDxKgAiINRMrPxTUWSGW6pFx9DzeJN4WPqra8= codeberg.org/gruf/go-mutexes v1.5.8 h1:HRGnvT4COb3jX9xdeoSUUbjPgmk5kXPuDfld9ksUJKA=
codeberg.org/gruf/go-mutexes v1.5.3/go.mod h1:AnhagsMzUISL/nBVwhnHwDwTZOAxMILwCOG8/wKOblg= codeberg.org/gruf/go-mutexes v1.5.8/go.mod h1:21sy/hWH8dDQBk7ocsxqo2GNpWiIir+e82RG3hjnN20=
codeberg.org/gruf/go-runners v1.6.3 h1:To/AX7eTrWuXrTkA3RA01YTP5zha1VZ68LQ+0D4RY7E= codeberg.org/gruf/go-runners v1.6.3 h1:To/AX7eTrWuXrTkA3RA01YTP5zha1VZ68LQ+0D4RY7E=
codeberg.org/gruf/go-runners v1.6.3/go.mod h1:oXAaUmG2VxoKttpCqZGv5nQBeSvZSR2BzIk7h1yTRlU= codeberg.org/gruf/go-runners v1.6.3/go.mod h1:oXAaUmG2VxoKttpCqZGv5nQBeSvZSR2BzIk7h1yTRlU=
codeberg.org/gruf/go-sched v1.2.4 h1:ddBB9o0D/2oU8NbQ0ldN5aWxogpXPRBATWi58+p++Hw= codeberg.org/gruf/go-sched v1.2.4 h1:ddBB9o0D/2oU8NbQ0ldN5aWxogpXPRBATWi58+p++Hw=
@ -54,8 +54,8 @@ codeberg.org/gruf/go-split v1.2.0 h1:PmzL23nVEVHm8VxjsJmv4m4wGQz2bGgQw52dgSSj65c
codeberg.org/gruf/go-split v1.2.0/go.mod h1:0rejWJpqvOoFAd7nwm5tIXYKaAqjtFGOXmTqQV+VO38= codeberg.org/gruf/go-split v1.2.0/go.mod h1:0rejWJpqvOoFAd7nwm5tIXYKaAqjtFGOXmTqQV+VO38=
codeberg.org/gruf/go-storage v0.3.1 h1:g66UIM/xXnEk9ejT+W0T9s/PODBZhXa/8ajzeY/MELI= codeberg.org/gruf/go-storage v0.3.1 h1:g66UIM/xXnEk9ejT+W0T9s/PODBZhXa/8ajzeY/MELI=
codeberg.org/gruf/go-storage v0.3.1/go.mod h1:r43n/zi7YGOCl2iSl7AMI27D1zcWS65Bi2+5xDzypeo= codeberg.org/gruf/go-storage v0.3.1/go.mod h1:r43n/zi7YGOCl2iSl7AMI27D1zcWS65Bi2+5xDzypeo=
codeberg.org/gruf/go-structr v0.9.9 h1:fwIzi/94yBNSWleXZIfVW/QyNK5+/xxI2reVYzu5V/c= codeberg.org/gruf/go-structr v0.9.12 h1:yMopvexnuKgZme9WgvIhrJaAuAjfper/x38xsVuJOOo=
codeberg.org/gruf/go-structr v0.9.9/go.mod h1:5dsazOsIeJyV8Dl2DdSXqCDEZUx3e3dc41N6f2mPtgw= codeberg.org/gruf/go-structr v0.9.12/go.mod h1:sP2ZSjM5X5XKlxuhAbTKuVQm9DWbHsrQRuTl3MUwbHw=
codeberg.org/gruf/go-xunsafe v0.0.0-20250809104800-512a9df57d73 h1:pRaOwIOS1WSZoPCAvE0H1zpv+D4gF37OVppybffqdI8= codeberg.org/gruf/go-xunsafe v0.0.0-20250809104800-512a9df57d73 h1:pRaOwIOS1WSZoPCAvE0H1zpv+D4gF37OVppybffqdI8=
codeberg.org/gruf/go-xunsafe v0.0.0-20250809104800-512a9df57d73/go.mod h1:9wkq+dmHjUhB/0ZxDUWAwsWuXwwGyx5N1dDCB9hpWs8= codeberg.org/gruf/go-xunsafe v0.0.0-20250809104800-512a9df57d73/go.mod h1:9wkq+dmHjUhB/0ZxDUWAwsWuXwwGyx5N1dDCB9hpWs8=
codeberg.org/superseriousbusiness/go-swagger v0.32.3-gts-go1.23-fix h1:k76/Th+bruqU/d+dB0Ru466ctTF2aVjKpisy/471ILE= codeberg.org/superseriousbusiness/go-swagger v0.32.3-gts-go1.23-fix h1:k76/Th+bruqU/d+dB0Ru466ctTF2aVjKpisy/471ILE=
@ -338,8 +338,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/ncruces/go-sqlite3 v0.29.0 h1:1tsLiagCoqZEfcHDeKsNSv5jvrY/Iu393pAnw2wLNJU= github.com/ncruces/go-sqlite3 v0.29.1 h1:NIi8AISWBToRHyoz01FXiTNvU147Tqdibgj2tFzJCqM=
github.com/ncruces/go-sqlite3 v0.29.0/go.mod h1:r1hSvYKPNJ+OlUA1O3r8o9LAawzPAlqeZiIdxTBBBJ0= github.com/ncruces/go-sqlite3 v0.29.1/go.mod h1:PpccBNNhvjwUOwDQEn2gXQPFPTWdlromj0+fSkd5KSg=
github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M= github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=

View file

@ -31,7 +31,7 @@ import (
// elements to reduce overall memory usage. // elements to reduce overall memory usage.
type SimpleQueue[T any] struct { type SimpleQueue[T any] struct {
l list.List[T] l list.List[T]
p mempool.UnsafePool p mempool.UnsafeSimplePool
w chan struct{} w chan struct{}
m sync.Mutex m sync.Mutex
} }

View file

@ -1,17 +1,17 @@
package mempool package mempool
import ( import (
"sync"
"sync/atomic"
"unsafe" "unsafe"
"golang.org/x/sys/cpu"
) )
const DefaultDirtyFactor = 128 // Pool provides a form of SimplePool
// with the addition of concurrency safety.
// Pool provides a type-safe form
// of UnsafePool using generics.
//
// Note it is NOT safe for concurrent
// use, you must protect it yourself!
type Pool[T any] struct { type Pool[T any] struct {
UnsafePool
// New is an optionally provided // New is an optionally provided
// allocator used when no value // allocator used when no value
@ -21,79 +21,119 @@ type Pool[T any] struct {
// Reset is an optionally provided // Reset is an optionally provided
// value resetting function called // value resetting function called
// on passed value to Put(). // on passed value to Put().
Reset func(T) Reset func(T) bool
}
UnsafePool func NewPool[T any](new func() T, reset func(T) bool, check func(current, victim int) bool) Pool[T] {
return Pool[T]{
New: new,
Reset: reset,
UnsafePool: NewUnsafePool(check),
}
} }
func (p *Pool[T]) Get() T { func (p *Pool[T]) Get() T {
if ptr := p.UnsafePool.Get(); ptr != nil { if ptr := p.UnsafePool.Get(); ptr != nil {
return *(*T)(ptr) return *(*T)(ptr)
} else if p.New != nil {
return p.New()
} }
var z T var t T
return z if p.New != nil {
t = p.New()
}
return t
} }
func (p *Pool[T]) Put(t T) { func (p *Pool[T]) Put(t T) {
if p.Reset != nil { if p.Reset != nil && !p.Reset(t) {
p.Reset(t) return
} }
ptr := unsafe.Pointer(&t) ptr := unsafe.Pointer(&t)
p.UnsafePool.Put(ptr) p.UnsafePool.Put(ptr)
} }
// UnsafePool provides an incredibly // UnsafePool provides a form of UnsafeSimplePool
// simple memory pool implementation // with the addition of concurrency safety.
// that stores ptrs to memory values,
// and regularly flushes internal pool
// structures according to DirtyFactor.
//
// Note it is NOT safe for concurrent
// use, you must protect it yourself!
type UnsafePool struct { type UnsafePool struct {
internal
// DirtyFactor determines the max _ [cache_line_size - unsafe.Sizeof(internal{})%cache_line_size]byte
// number of $dirty count before
// pool is garbage collected. Where:
// $dirty = len(current) - len(victim)
DirtyFactor int
current []unsafe.Pointer
victim []unsafe.Pointer
} }
func (p *UnsafePool) Get() unsafe.Pointer { func NewUnsafePool(check func(current, victim int) bool) UnsafePool {
// First try current list. return UnsafePool{internal: internal{
if len(p.current) > 0 { pool: UnsafeSimplePool{Check: check},
ptr := p.current[len(p.current)-1] }}
p.current = p.current[:len(p.current)-1] }
const (
// current platform integer size.
int_size = 32 << (^uint(0) >> 63)
// platform CPU cache line size to avoid false sharing.
cache_line_size = unsafe.Sizeof(cpu.CacheLinePad{})
)
type internal struct {
// fast-access ring-buffer of
// pointers accessible by index.
//
// if Go ever exposes goroutine IDs
// to us we can make this a lot faster.
ring [int_size / 4]unsafe.Pointer
index atomic.Uint64
// underlying pool and
// slow mutex protection.
pool UnsafeSimplePool
mutex sync.Mutex
}
func (p *internal) Check(fn func(current, victim int) bool) func(current, victim int) bool {
p.mutex.Lock()
if fn == nil {
if p.pool.Check == nil {
fn = defaultCheck
} else {
fn = p.pool.Check
}
} else {
p.pool.Check = fn
}
p.mutex.Unlock()
return fn
}
func (p *internal) Get() unsafe.Pointer {
if ptr := atomic.SwapPointer(&p.ring[p.index.Load()%uint64(cap(p.ring))], nil); ptr != nil {
p.index.Add(^uint64(0)) // i.e. -1
return ptr
}
p.mutex.Lock()
ptr := p.pool.Get()
p.mutex.Unlock()
return ptr return ptr
} }
// Fallback to victim. func (p *internal) Put(ptr unsafe.Pointer) {
if len(p.victim) > 0 { if atomic.CompareAndSwapPointer(&p.ring[p.index.Add(1)%uint64(cap(p.ring))], nil, ptr) {
ptr := p.victim[len(p.victim)-1] return
p.victim = p.victim[:len(p.victim)-1] }
return ptr p.mutex.Lock()
p.pool.Put(ptr)
p.mutex.Unlock()
} }
return nil func (p *internal) GC() {
for i := range p.ring {
atomic.StorePointer(&p.ring[i], nil)
}
p.mutex.Lock()
p.pool.GC()
p.mutex.Unlock()
} }
func (p *UnsafePool) Put(ptr unsafe.Pointer) { func (p *internal) Size() int {
p.current = append(p.current, ptr) p.mutex.Lock()
sz := p.pool.Size()
// Get dirty factor. p.mutex.Unlock()
df := p.DirtyFactor return sz
if df == 0 {
df = DefaultDirtyFactor
}
if len(p.current)-len(p.victim) > df {
// Garbage collection!
p.victim = p.current
p.current = nil
}
} }

111
vendor/codeberg.org/gruf/go-mempool/simple.go generated vendored Normal file
View file

@ -0,0 +1,111 @@
package mempool
import (
"unsafe"
)
// SimplePool provides a type-safe form
// of UnsafePool using generics.
//
// Note it is NOT safe for concurrent
// use, you must protect it yourself!
type SimplePool[T any] struct {
UnsafeSimplePool
// New is an optionally provided
// allocator used when no value
// is available for use in pool.
New func() T
// Reset is an optionally provided
// value resetting function called
// on passed value to Put().
Reset func(T) bool
}
func (p *SimplePool[T]) Get() T {
if ptr := p.UnsafeSimplePool.Get(); ptr != nil {
return *(*T)(ptr)
}
var t T
if p.New != nil {
t = p.New()
}
return t
}
func (p *SimplePool[T]) Put(t T) {
if p.Reset != nil && !p.Reset(t) {
return
}
ptr := unsafe.Pointer(&t)
p.UnsafeSimplePool.Put(ptr)
}
// UnsafeSimplePool provides an incredibly
// simple memory pool implementation
// that stores ptrs to memory values,
// and regularly flushes internal pool
// structures according to CheckGC().
//
// Note it is NOT safe for concurrent
// use, you must protect it yourself!
type UnsafeSimplePool struct {
// Check determines how often to flush
// internal pools based on underlying
// current and victim pool sizes. It gets
// called on every pool Put() operation.
//
// A flush will start a new current
// pool, make victim the old current,
// and drop the existing victim pool.
Check func(current, victim int) bool
current []unsafe.Pointer
victim []unsafe.Pointer
}
func (p *UnsafeSimplePool) Get() unsafe.Pointer {
// First try current list.
if len(p.current) > 0 {
ptr := p.current[len(p.current)-1]
p.current = p.current[:len(p.current)-1]
return ptr
}
// Fallback to victim.
if len(p.victim) > 0 {
ptr := p.victim[len(p.victim)-1]
p.victim = p.victim[:len(p.victim)-1]
return ptr
}
return nil
}
func (p *UnsafeSimplePool) Put(ptr unsafe.Pointer) {
p.current = append(p.current, ptr)
// Get GC check func.
if p.Check == nil {
p.Check = defaultCheck
}
if p.Check(len(p.current), len(p.victim)) {
p.GC() // garbage collection time!
}
}
func (p *UnsafeSimplePool) GC() {
p.victim = p.current
p.current = nil
}
func (p *UnsafeSimplePool) Size() int {
return len(p.current) + len(p.victim)
}
func defaultCheck(current, victim int) bool {
return current-victim > 128 || victim > 256
}

View file

@ -26,14 +26,13 @@ const (
type MutexMap struct { type MutexMap struct {
mapmu sync.Mutex mapmu sync.Mutex
mumap hashmap mumap hashmap
mupool mempool.UnsafePool mupool mempool.UnsafeSimplePool
} }
// checkInit ensures MutexMap is initialized (UNSAFE). // checkInit ensures MutexMap is initialized (UNSAFE).
func (mm *MutexMap) checkInit() { func (mm *MutexMap) checkInit() {
if mm.mumap.m == nil { if mm.mumap.m == nil {
mm.mumap.init(0) mm.mumap.init(0)
mm.mupool.DirtyFactor = 256
} }
} }
@ -175,13 +174,9 @@ func (mu *rwmutex) Lock(lt uint8) bool {
// sleeping goroutines waiting on this mutex. // sleeping goroutines waiting on this mutex.
func (mu *rwmutex) Unlock() bool { func (mu *rwmutex) Unlock() bool {
switch mu.l--; { switch mu.l--; {
case mu.l > 0 && mu.t == lockTypeWrite:
panic("BUG: multiple writer locks")
case mu.l < 0:
panic("BUG: negative lock count")
case mu.l == 0: case mu.l == 0:
// Fully unlocked. // Fully
// unlock.
mu.t = 0 mu.t = 0
// Awake all blocked goroutines and check // Awake all blocked goroutines and check
@ -197,12 +192,16 @@ func (mu *rwmutex) Unlock() bool {
// (before == after) => (waiters = 0) // (before == after) => (waiters = 0)
return (before == after) return (before == after)
default: case mu.l < 0:
panic("BUG: negative lock count")
case mu.t == lockTypeWrite:
panic("BUG: multiple write locks")
}
// i.e. mutex still // i.e. mutex still
// locked by others. // locked by others.
return false return false
} }
}
// WaitRelock expects a mutex to be passed in, already in the // WaitRelock expects a mutex to be passed in, already in the
// locked state. It incr the notifyList waiter count before // locked state. It incr the notifyList waiter count before

View file

@ -4,10 +4,10 @@ import (
"os" "os"
"reflect" "reflect"
"strings" "strings"
"sync"
"unsafe" "unsafe"
"codeberg.org/gruf/go-byteutil" "codeberg.org/gruf/go-byteutil"
"codeberg.org/gruf/go-mempool"
"codeberg.org/gruf/go-xunsafe" "codeberg.org/gruf/go-xunsafe"
) )
@ -371,17 +371,15 @@ type index_entry struct {
key string key string
} }
var index_entry_pool sync.Pool var index_entry_pool mempool.UnsafePool
// new_index_entry returns a new prepared index_entry. // new_index_entry returns a new prepared index_entry.
func new_index_entry() *index_entry { func new_index_entry() *index_entry {
v := index_entry_pool.Get() if ptr := index_entry_pool.Get(); ptr != nil {
if v == nil { return (*index_entry)(ptr)
e := new(index_entry)
e.elem.data = unsafe.Pointer(e)
v = e
} }
entry := v.(*index_entry) entry := new(index_entry)
entry.elem.data = unsafe.Pointer(entry)
return entry return entry
} }
@ -396,7 +394,8 @@ func free_index_entry(entry *index_entry) {
entry.key = "" entry.key = ""
entry.index = nil entry.index = nil
entry.item = nil entry.item = nil
index_entry_pool.Put(entry) ptr := unsafe.Pointer(entry)
index_entry_pool.Put(ptr)
} }
func is_unique(f uint8) bool { func is_unique(f uint8) bool {

View file

@ -2,8 +2,9 @@ package structr
import ( import (
"os" "os"
"sync"
"unsafe" "unsafe"
"codeberg.org/gruf/go-mempool"
) )
type indexed_item struct { type indexed_item struct {
@ -19,17 +20,15 @@ type indexed_item struct {
indexed []*index_entry indexed []*index_entry
} }
var indexed_item_pool sync.Pool var indexed_item_pool mempool.UnsafePool
// new_indexed_item returns a new prepared indexed_item. // new_indexed_item returns a new prepared indexed_item.
func new_indexed_item() *indexed_item { func new_indexed_item() *indexed_item {
v := indexed_item_pool.Get() if ptr := indexed_item_pool.Get(); ptr != nil {
if v == nil { return (*indexed_item)(ptr)
i := new(indexed_item)
i.elem.data = unsafe.Pointer(i)
v = i
} }
item := v.(*indexed_item) item := new(indexed_item)
item.elem.data = unsafe.Pointer(item)
return item return item
} }
@ -43,7 +42,8 @@ func free_indexed_item(item *indexed_item) {
return return
} }
item.data = nil item.data = nil
indexed_item_pool.Put(item) ptr := unsafe.Pointer(item)
indexed_item_pool.Put(ptr)
} }
// drop_index will drop the given index entry from item's indexed. // drop_index will drop the given index entry from item's indexed.

View file

@ -2,8 +2,9 @@ package structr
import ( import (
"os" "os"
"sync"
"unsafe" "unsafe"
"codeberg.org/gruf/go-mempool"
) )
// elem represents an elem // elem represents an elem
@ -27,16 +28,14 @@ type list struct {
len int len int
} }
var list_pool sync.Pool var list_pool mempool.UnsafePool
// new_list returns a new prepared list. // new_list returns a new prepared list.
func new_list() *list { func new_list() *list {
v := list_pool.Get() if ptr := list_pool.Get(); ptr != nil {
if v == nil { return (*list)(ptr)
v = new(list)
} }
list := v.(*list) return new(list)
return list
} }
// free_list releases the list. // free_list releases the list.
@ -48,11 +47,13 @@ func free_list(list *list) {
os.Stderr.WriteString(msg + "\n") os.Stderr.WriteString(msg + "\n")
return return
} }
list_pool.Put(list) ptr := unsafe.Pointer(list)
list_pool.Put(ptr)
} }
// push_front will push the given elem to front (head) of list. // push_front will push the given elem to front (head) of list.
func (l *list) push_front(elem *list_elem) { func (l *list) push_front(elem *list_elem) {
// Set new head. // Set new head.
oldHead := l.head oldHead := l.head
l.head = elem l.head = elem
@ -66,12 +67,14 @@ func (l *list) push_front(elem *list_elem) {
l.tail = elem l.tail = elem
} }
// Incr count // Incr
// count
l.len++ l.len++
} }
// push_back will push the given elem to back (tail) of list. // push_back will push the given elem to back (tail) of list.
func (l *list) push_back(elem *list_elem) { func (l *list) push_back(elem *list_elem) {
// Set new tail. // Set new tail.
oldTail := l.tail oldTail := l.tail
l.tail = elem l.tail = elem
@ -85,7 +88,8 @@ func (l *list) push_back(elem *list_elem) {
l.head = elem l.head = elem
} }
// Incr count // Incr
// count
l.len++ l.len++
} }
@ -131,7 +135,8 @@ func (l *list) insert(elem *list_elem, at *list_elem) {
elem.next = oldNext elem.next = oldNext
} }
// Incr count // Incr
// count
l.len++ l.len++
} }
@ -174,6 +179,7 @@ func (l *list) remove(elem *list_elem) {
prev.next = next prev.next = next
} }
// Decr count // Decr
// count
l.len-- l.len--
} }

View file

@ -146,7 +146,7 @@ func find_field(t xunsafe.TypeIter, names []string) (sfield struct_field, ftype
sfield.mangle = mangler.Get(t) sfield.mangle = mangler.Get(t)
// Calculate zero value string. // Calculate zero value string.
zptr := zero_value_field(o, sfield.offsets) zptr := zero_value_ptr(o, sfield.offsets)
zstr := string(sfield.mangle(nil, zptr)) zstr := string(sfield.mangle(nil, zptr))
sfield.zerostr = zstr sfield.zerostr = zstr
sfield.zero = zptr sfield.zero = zptr
@ -154,7 +154,9 @@ func find_field(t xunsafe.TypeIter, names []string) (sfield struct_field, ftype
return return
} }
// zero_value ... // zero_value iterates the type contained in TypeIter{} along the given
// next_offset{} values, creating new ptrs where necessary, returning the
// zero reflect.Value{} after fully iterating the next_offset{} slice.
func zero_value(t xunsafe.TypeIter, offsets []next_offset) reflect.Value { func zero_value(t xunsafe.TypeIter, offsets []next_offset) reflect.Value {
v := reflect.New(t.Type).Elem() v := reflect.New(t.Type).Elem()
for _, offset := range offsets { for _, offset := range offsets {
@ -175,8 +177,8 @@ func zero_value(t xunsafe.TypeIter, offsets []next_offset) reflect.Value {
return v return v
} }
// zero_value_field ... // zero_value_ptr returns the unsafe pointer address of the result of zero_value().
func zero_value_field(t xunsafe.TypeIter, offsets []next_offset) unsafe.Pointer { func zero_value_ptr(t xunsafe.TypeIter, offsets []next_offset) unsafe.Pointer {
return zero_value(t, offsets).Addr().UnsafePointer() return zero_value(t, offsets).Addr().UnsafePointer()
} }

View file

@ -8,6 +8,8 @@ import (
"strings" "strings"
"sync" "sync"
"unsafe" "unsafe"
"codeberg.org/gruf/go-mempool"
) )
// Direction defines a direction // Direction defines a direction
@ -1133,18 +1135,16 @@ func to_timeline_item(item *indexed_item) *timeline_item {
return to return to
} }
var timeline_item_pool sync.Pool var timeline_item_pool mempool.UnsafePool
// new_timeline_item returns a new prepared timeline_item. // new_timeline_item returns a new prepared timeline_item.
func new_timeline_item() *timeline_item { func new_timeline_item() *timeline_item {
v := timeline_item_pool.Get() if ptr := timeline_item_pool.Get(); ptr != nil {
if v == nil { return (*timeline_item)(ptr)
i := new(timeline_item)
i.elem.data = unsafe.Pointer(i)
i.ck = ^uint(0)
v = i
} }
item := v.(*timeline_item) item := new(timeline_item)
item.elem.data = unsafe.Pointer(item)
item.ck = ^uint(0)
return item return item
} }
@ -1159,5 +1159,6 @@ func free_timeline_item(item *timeline_item) {
} }
item.data = nil item.data = nil
item.pk = nil item.pk = nil
timeline_item_pool.Put(item) ptr := unsafe.Pointer(item)
timeline_item_pool.Put(ptr)
} }

View file

@ -444,20 +444,27 @@ func (c *Conn) Status(op DBStatus, reset bool) (current, highwater int, err erro
// https://sqlite.org/c3ref/table_column_metadata.html // https://sqlite.org/c3ref/table_column_metadata.html
func (c *Conn) TableColumnMetadata(schema, table, column string) (declType, collSeq string, notNull, primaryKey, autoInc bool, err error) { func (c *Conn) TableColumnMetadata(schema, table, column string) (declType, collSeq string, notNull, primaryKey, autoInc bool, err error) {
defer c.arena.mark()() defer c.arena.mark()()
var (
var schemaPtr, columnPtr ptr_t declTypePtr ptr_t
declTypePtr := c.arena.new(ptrlen) collSeqPtr ptr_t
collSeqPtr := c.arena.new(ptrlen) notNullPtr ptr_t
notNullPtr := c.arena.new(ptrlen) primaryKeyPtr ptr_t
autoIncPtr := c.arena.new(ptrlen) autoIncPtr ptr_t
primaryKeyPtr := c.arena.new(ptrlen) columnPtr ptr_t
schemaPtr ptr_t
)
if column != "" {
declTypePtr = c.arena.new(ptrlen)
collSeqPtr = c.arena.new(ptrlen)
notNullPtr = c.arena.new(ptrlen)
primaryKeyPtr = c.arena.new(ptrlen)
autoIncPtr = c.arena.new(ptrlen)
columnPtr = c.arena.string(column)
}
if schema != "" { if schema != "" {
schemaPtr = c.arena.string(schema) schemaPtr = c.arena.string(schema)
} }
tablePtr := c.arena.string(table) tablePtr := c.arena.string(table)
if column != "" {
columnPtr = c.arena.string(column)
}
rc := res_t(c.call("sqlite3_table_column_metadata", stk_t(c.handle), rc := res_t(c.call("sqlite3_table_column_metadata", stk_t(c.handle),
stk_t(schemaPtr), stk_t(tablePtr), stk_t(columnPtr), stk_t(schemaPtr), stk_t(tablePtr), stk_t(columnPtr),

View file

@ -1,7 +1,6 @@
package sqlite3 package sqlite3
import ( import (
"encoding/json"
"errors" "errors"
"math" "math"
"time" "time"
@ -173,21 +172,6 @@ func (ctx Context) ResultPointer(ptr any) {
stk_t(ctx.handle), stk_t(valPtr)) stk_t(ctx.handle), stk_t(valPtr))
} }
// ResultJSON sets the result of the function to the JSON encoding of value.
//
// https://sqlite.org/c3ref/result_blob.html
func (ctx Context) ResultJSON(value any) {
err := json.NewEncoder(callbackWriter(func(p []byte) (int, error) {
ctx.ResultRawText(p[:len(p)-1]) // remove the newline
return 0, nil
})).Encode(value)
if err != nil {
ctx.ResultError(err)
return // notest
}
}
// ResultValue sets the result of the function to a copy of [Value]. // ResultValue sets the result of the function to a copy of [Value].
// //
// https://sqlite.org/c3ref/result_blob.html // https://sqlite.org/c3ref/result_blob.html

View file

@ -608,13 +608,23 @@ type scantype byte
const ( const (
_ANY scantype = iota _ANY scantype = iota
_INT scantype = scantype(sqlite3.INTEGER) _INT
_REAL scantype = scantype(sqlite3.FLOAT) _REAL
_TEXT scantype = scantype(sqlite3.TEXT) _TEXT
_BLOB scantype = scantype(sqlite3.BLOB) _BLOB
_NULL scantype = scantype(sqlite3.NULL) _NULL
_BOOL scantype = iota _BOOL
_TIME _TIME
_NOT_NULL
)
var (
_ [0]struct{} = [scantype(sqlite3.INTEGER) - _INT]struct{}{}
_ [0]struct{} = [scantype(sqlite3.FLOAT) - _REAL]struct{}{}
_ [0]struct{} = [scantype(sqlite3.TEXT) - _TEXT]struct{}{}
_ [0]struct{} = [scantype(sqlite3.BLOB) - _BLOB]struct{}{}
_ [0]struct{} = [scantype(sqlite3.NULL) - _NULL]struct{}{}
_ [0]struct{} = [_NOT_NULL & (_NOT_NULL - 1)]struct{}{}
) )
func scanFromDecl(decl string) scantype { func scanFromDecl(decl string) scantype {
@ -644,8 +654,8 @@ type rows struct {
*stmt *stmt
names []string names []string
types []string types []string
nulls []bool
scans []scantype scans []scantype
dest []driver.Value
} }
var ( var (
@ -675,34 +685,36 @@ func (r *rows) Columns() []string {
func (r *rows) scanType(index int) scantype { func (r *rows) scanType(index int) scantype {
if r.scans == nil { if r.scans == nil {
count := r.Stmt.ColumnCount() count := len(r.names)
scans := make([]scantype, count) scans := make([]scantype, count)
for i := range scans { for i := range scans {
scans[i] = scanFromDecl(strings.ToUpper(r.Stmt.ColumnDeclType(i))) scans[i] = scanFromDecl(strings.ToUpper(r.Stmt.ColumnDeclType(i)))
} }
r.scans = scans r.scans = scans
} }
return r.scans[index] return r.scans[index] &^ _NOT_NULL
} }
func (r *rows) loadColumnMetadata() { func (r *rows) loadColumnMetadata() {
if r.nulls == nil { if r.types == nil {
c := r.Stmt.Conn() c := r.Stmt.Conn()
count := r.Stmt.ColumnCount() count := len(r.names)
nulls := make([]bool, count)
types := make([]string, count) types := make([]string, count)
scans := make([]scantype, count) scans := make([]scantype, count)
for i := range nulls { for i := range types {
var notnull bool
if col := r.Stmt.ColumnOriginName(i); col != "" { if col := r.Stmt.ColumnOriginName(i); col != "" {
types[i], _, nulls[i], _, _, _ = c.TableColumnMetadata( types[i], _, notnull, _, _, _ = c.TableColumnMetadata(
r.Stmt.ColumnDatabaseName(i), r.Stmt.ColumnDatabaseName(i),
r.Stmt.ColumnTableName(i), r.Stmt.ColumnTableName(i),
col) col)
types[i] = strings.ToUpper(types[i]) types[i] = strings.ToUpper(types[i])
scans[i] = scanFromDecl(types[i]) scans[i] = scanFromDecl(types[i])
if notnull {
scans[i] |= _NOT_NULL
}
} }
} }
r.nulls = nulls
r.types = types r.types = types
r.scans = scans r.scans = scans
} }
@ -721,15 +733,13 @@ func (r *rows) ColumnTypeDatabaseTypeName(index int) string {
func (r *rows) ColumnTypeNullable(index int) (nullable, ok bool) { func (r *rows) ColumnTypeNullable(index int) (nullable, ok bool) {
r.loadColumnMetadata() r.loadColumnMetadata()
if r.nulls[index] { nullable = r.scans[index]&^_NOT_NULL == 0
return false, true return nullable, !nullable
}
return true, false
} }
func (r *rows) ColumnTypeScanType(index int) (typ reflect.Type) { func (r *rows) ColumnTypeScanType(index int) (typ reflect.Type) {
r.loadColumnMetadata() r.loadColumnMetadata()
scan := r.scans[index] scan := r.scans[index] &^ _NOT_NULL
if r.Stmt.Busy() { if r.Stmt.Busy() {
// SQLite is dynamically typed and we now have a row. // SQLite is dynamically typed and we now have a row.
@ -772,6 +782,7 @@ func (r *rows) ColumnTypeScanType(index int) (typ reflect.Type) {
} }
func (r *rows) Next(dest []driver.Value) error { func (r *rows) Next(dest []driver.Value) error {
r.dest = nil
c := r.Stmt.Conn() c := r.Stmt.Conn()
if old := c.SetInterrupt(r.ctx); old != r.ctx { if old := c.SetInterrupt(r.ctx); old != r.ctx {
defer c.SetInterrupt(old) defer c.SetInterrupt(old)
@ -790,18 +801,7 @@ func (r *rows) Next(dest []driver.Value) error {
} }
for i := range dest { for i := range dest {
scan := r.scanType(i) scan := r.scanType(i)
switch v := dest[i].(type) { if v, ok := dest[i].([]byte); ok {
case int64:
if scan == _BOOL {
switch v {
case 1:
dest[i] = true
case 0:
dest[i] = false
}
continue
}
case []byte:
if len(v) == cap(v) { // a BLOB if len(v) == cap(v) { // a BLOB
continue continue
} }
@ -816,38 +816,49 @@ func (r *rows) Next(dest []driver.Value) error {
} }
} }
dest[i] = string(v) dest[i] = string(v)
case float64:
break
default:
continue
} }
if scan == _TIME { switch scan {
case _TIME:
t, err := r.tmRead.Decode(dest[i]) t, err := r.tmRead.Decode(dest[i])
if err == nil { if err == nil {
dest[i] = t dest[i] = t
continue }
case _BOOL:
switch dest[i] {
case int64(0):
dest[i] = false
case int64(1):
dest[i] = true
} }
} }
} }
r.dest = dest
return nil return nil
} }
func (r *rows) ScanColumn(dest any, index int) error { func (r *rows) ScanColumn(dest any, index int) (err error) {
// notest // Go 1.26 // notest // Go 1.26
var ptr *time.Time var tm *time.Time
var ok *bool
switch d := dest.(type) { switch d := dest.(type) {
case *time.Time: case *time.Time:
ptr = d tm = d
case *sql.NullTime: case *sql.NullTime:
ptr = &d.Time tm = &d.Time
ok = &d.Valid
case *sql.Null[time.Time]: case *sql.Null[time.Time]:
ptr = &d.V tm = &d.V
ok = &d.Valid
default: default:
return driver.ErrSkip return driver.ErrSkip
} }
if t := r.Stmt.ColumnTime(index, r.tmRead); !t.IsZero() { value := r.dest[index]
*ptr = t *tm, err = r.tmRead.Decode(value)
if ok != nil {
*ok = err == nil
if value == nil {
return nil return nil
} }
return driver.ErrSkip }
return err
} }

View file

@ -1,3 +1,5 @@
//go:build !goexperiment.jsonv2
package util package util
import ( import (

View file

@ -0,0 +1,52 @@
//go:build goexperiment.jsonv2
package util
import (
"encoding/json/v2"
"math"
"strconv"
"time"
"unsafe"
)
type JSON struct{ Value any }
func (j JSON) Scan(value any) error {
var buf []byte
switch v := value.(type) {
case []byte:
buf = v
case string:
buf = unsafe.Slice(unsafe.StringData(v), len(v))
case int64:
buf = strconv.AppendInt(nil, v, 10)
case float64:
buf = AppendNumber(nil, v)
case time.Time:
buf = append(buf, '"')
buf = v.AppendFormat(buf, time.RFC3339Nano)
buf = append(buf, '"')
case nil:
buf = []byte("null")
default:
panic(AssertErr())
}
return json.Unmarshal(buf, j.Value)
}
func AppendNumber(dst []byte, f float64) []byte {
switch {
case math.IsNaN(f):
dst = append(dst, "null"...)
case math.IsInf(f, 1):
dst = append(dst, "9.0e999"...)
case math.IsInf(f, -1):
dst = append(dst, "-9.0e999"...)
default:
return strconv.AppendFloat(dst, f, 'g', -1, 64)
}
return dst
}

View file

@ -1,6 +1,13 @@
//go:build !goexperiment.jsonv2
package sqlite3 package sqlite3
import "github.com/ncruces/go-sqlite3/internal/util" import (
"encoding/json"
"strconv"
"github.com/ncruces/go-sqlite3/internal/util"
)
// JSON returns a value that can be used as an argument to // JSON returns a value that can be used as an argument to
// [database/sql.DB.Exec], [database/sql.Row.Scan] and similar methods to // [database/sql.DB.Exec], [database/sql.Row.Scan] and similar methods to
@ -10,3 +17,77 @@ import "github.com/ncruces/go-sqlite3/internal/util"
func JSON(value any) any { func JSON(value any) any {
return util.JSON{Value: value} return util.JSON{Value: value}
} }
// ResultJSON sets the result of the function to the JSON encoding of value.
//
// https://sqlite.org/c3ref/result_blob.html
func (ctx Context) ResultJSON(value any) {
err := json.NewEncoder(callbackWriter(func(p []byte) (int, error) {
ctx.ResultRawText(p[:len(p)-1]) // remove the newline
return 0, nil
})).Encode(value)
if err != nil {
ctx.ResultError(err)
return // notest
}
}
// BindJSON binds the JSON encoding of value to the prepared statement.
// The leftmost SQL parameter has an index of 1.
//
// https://sqlite.org/c3ref/bind_blob.html
func (s *Stmt) BindJSON(param int, value any) error {
return json.NewEncoder(callbackWriter(func(p []byte) (int, error) {
return 0, s.BindRawText(param, p[:len(p)-1]) // remove the newline
})).Encode(value)
}
// ColumnJSON parses the JSON-encoded value of the result column
// and stores it in the value pointed to by ptr.
// The leftmost column of the result set has the index 0.
//
// https://sqlite.org/c3ref/column_blob.html
func (s *Stmt) ColumnJSON(col int, ptr any) error {
var data []byte
switch s.ColumnType(col) {
case NULL:
data = []byte("null")
case TEXT:
data = s.ColumnRawText(col)
case BLOB:
data = s.ColumnRawBlob(col)
case INTEGER:
data = strconv.AppendInt(nil, s.ColumnInt64(col), 10)
case FLOAT:
data = util.AppendNumber(nil, s.ColumnFloat(col))
default:
panic(util.AssertErr())
}
return json.Unmarshal(data, ptr)
}
// JSON parses a JSON-encoded value
// and stores the result in the value pointed to by ptr.
func (v Value) JSON(ptr any) error {
var data []byte
switch v.Type() {
case NULL:
data = []byte("null")
case TEXT:
data = v.RawText()
case BLOB:
data = v.RawBlob()
case INTEGER:
data = strconv.AppendInt(nil, v.Int64(), 10)
case FLOAT:
data = util.AppendNumber(nil, v.Float())
default:
panic(util.AssertErr())
}
return json.Unmarshal(data, ptr)
}
type callbackWriter func(p []byte) (int, error)
func (fn callbackWriter) Write(p []byte) (int, error) { return fn(p) }

113
vendor/github.com/ncruces/go-sqlite3/json_v2.go generated vendored Normal file
View file

@ -0,0 +1,113 @@
//go:build goexperiment.jsonv2
package sqlite3
import (
"encoding/json/v2"
"strconv"
"github.com/ncruces/go-sqlite3/internal/util"
)
// JSON returns a value that can be used as an argument to
// [database/sql.DB.Exec], [database/sql.Row.Scan] and similar methods to
// store value as JSON, or decode JSON into value.
// JSON should NOT be used with [Stmt.BindJSON], [Stmt.ColumnJSON],
// [Value.JSON], or [Context.ResultJSON].
func JSON(value any) any {
return util.JSON{Value: value}
}
// ResultJSON sets the result of the function to the JSON encoding of value.
//
// https://sqlite.org/c3ref/result_blob.html
func (ctx Context) ResultJSON(value any) {
w := bytesWriter{sqlite: ctx.c.sqlite}
if err := json.MarshalWrite(&w, value); err != nil {
ctx.c.free(w.ptr)
ctx.ResultError(err)
return // notest
}
ctx.c.call("sqlite3_result_text_go",
stk_t(ctx.handle), stk_t(w.ptr), stk_t(len(w.buf)))
}
// BindJSON binds the JSON encoding of value to the prepared statement.
// The leftmost SQL parameter has an index of 1.
//
// https://sqlite.org/c3ref/bind_blob.html
func (s *Stmt) BindJSON(param int, value any) error {
w := bytesWriter{sqlite: s.c.sqlite}
if err := json.MarshalWrite(&w, value); err != nil {
s.c.free(w.ptr)
return err
}
rc := res_t(s.c.call("sqlite3_bind_text_go",
stk_t(s.handle), stk_t(param),
stk_t(w.ptr), stk_t(len(w.buf))))
return s.c.error(rc)
}
// ColumnJSON parses the JSON-encoded value of the result column
// and stores it in the value pointed to by ptr.
// The leftmost column of the result set has the index 0.
//
// https://sqlite.org/c3ref/column_blob.html
func (s *Stmt) ColumnJSON(col int, ptr any) error {
var data []byte
switch s.ColumnType(col) {
case NULL:
data = []byte("null")
case TEXT:
data = s.ColumnRawText(col)
case BLOB:
data = s.ColumnRawBlob(col)
case INTEGER:
data = strconv.AppendInt(nil, s.ColumnInt64(col), 10)
case FLOAT:
data = util.AppendNumber(nil, s.ColumnFloat(col))
default:
panic(util.AssertErr())
}
return json.Unmarshal(data, ptr)
}
// JSON parses a JSON-encoded value
// and stores the result in the value pointed to by ptr.
func (v Value) JSON(ptr any) error {
var data []byte
switch v.Type() {
case NULL:
data = []byte("null")
case TEXT:
data = v.RawText()
case BLOB:
data = v.RawBlob()
case INTEGER:
data = strconv.AppendInt(nil, v.Int64(), 10)
case FLOAT:
data = util.AppendNumber(nil, v.Float())
default:
panic(util.AssertErr())
}
return json.Unmarshal(data, ptr)
}
type bytesWriter struct {
*sqlite
buf []byte
ptr ptr_t
}
func (b *bytesWriter) Write(p []byte) (n int, err error) {
if len(p) > cap(b.buf)-len(b.buf) {
want := int64(len(b.buf)) + int64(len(p))
grow := int64(cap(b.buf))
grow += grow >> 1
want = max(want, grow)
b.ptr = b.realloc(b.ptr, want)
b.buf = util.View(b.mod, b.ptr, want)[:len(b.buf)]
}
b.buf = append(b.buf, p...)
return len(p), nil
}

View file

@ -5,6 +5,7 @@ import (
"context" "context"
"math/bits" "math/bits"
"os" "os"
"strings"
"sync" "sync"
"unsafe" "unsafe"
@ -128,11 +129,10 @@ func (sqlt *sqlite) error(rc res_t, handle ptr_t, sql ...string) error {
var msg, query string var msg, query string
if ptr := ptr_t(sqlt.call("sqlite3_errmsg", stk_t(handle))); ptr != 0 { if ptr := ptr_t(sqlt.call("sqlite3_errmsg", stk_t(handle))); ptr != 0 {
msg = util.ReadString(sqlt.mod, ptr, _MAX_LENGTH) msg = util.ReadString(sqlt.mod, ptr, _MAX_LENGTH)
switch { if msg == "not an error" {
case msg == "not an error":
msg = ""
case msg == util.ErrorCodeString(uint32(rc))[len("sqlite3: "):]:
msg = "" msg = ""
} else {
msg = strings.TrimPrefix(msg, util.ErrorCodeString(uint32(rc))[len("sqlite3: "):])
} }
} }

View file

@ -1,9 +1,7 @@
package sqlite3 package sqlite3
import ( import (
"encoding/json"
"math" "math"
"strconv"
"time" "time"
"github.com/ncruces/go-sqlite3/internal/util" "github.com/ncruces/go-sqlite3/internal/util"
@ -362,16 +360,6 @@ func (s *Stmt) BindPointer(param int, ptr any) error {
return s.c.error(rc) return s.c.error(rc)
} }
// BindJSON binds the JSON encoding of value to the prepared statement.
// The leftmost SQL parameter has an index of 1.
//
// https://sqlite.org/c3ref/bind_blob.html
func (s *Stmt) BindJSON(param int, value any) error {
return json.NewEncoder(callbackWriter(func(p []byte) (int, error) {
return 0, s.BindRawText(param, p[:len(p)-1]) // remove the newline
})).Encode(value)
}
// BindValue binds a copy of value to the prepared statement. // BindValue binds a copy of value to the prepared statement.
// The leftmost SQL parameter has an index of 1. // The leftmost SQL parameter has an index of 1.
// //
@ -598,30 +586,6 @@ func (s *Stmt) columnRawBytes(col int, ptr ptr_t, nul int32) []byte {
return util.View(s.c.mod, ptr, int64(n+nul))[:n] return util.View(s.c.mod, ptr, int64(n+nul))[:n]
} }
// ColumnJSON parses the JSON-encoded value of the result column
// and stores it in the value pointed to by ptr.
// The leftmost column of the result set has the index 0.
//
// https://sqlite.org/c3ref/column_blob.html
func (s *Stmt) ColumnJSON(col int, ptr any) error {
var data []byte
switch s.ColumnType(col) {
case NULL:
data = []byte("null")
case TEXT:
data = s.ColumnRawText(col)
case BLOB:
data = s.ColumnRawBlob(col)
case INTEGER:
data = strconv.AppendInt(nil, s.ColumnInt64(col), 10)
case FLOAT:
data = util.AppendNumber(nil, s.ColumnFloat(col))
default:
panic(util.AssertErr())
}
return json.Unmarshal(data, ptr)
}
// ColumnValue returns the unprotected value of the result column. // ColumnValue returns the unprotected value of the result column.
// The leftmost column of the result set has the index 0. // The leftmost column of the result set has the index 0.
// //
@ -748,7 +712,3 @@ func (s *Stmt) columns(count int64) ([]byte, ptr_t, error) {
return util.View(s.c.mod, typePtr, count), dataPtr, nil return util.View(s.c.mod, typePtr, count), dataPtr, nil
} }
type callbackWriter func(p []byte) (int, error)
func (fn callbackWriter) Write(p []byte) (int, error) { return fn(p) }

View file

@ -94,7 +94,7 @@ func (f TimeFormat) Encode(t time.Time) any {
case TimeFormatUnix: case TimeFormatUnix:
return t.Unix() return t.Unix()
case TimeFormatUnixFrac: case TimeFormatUnixFrac:
return float64(t.Unix()) + float64(t.Nanosecond())*1e-9 return math.FMA(1e-9, float64(t.Nanosecond()), float64(t.Unix()))
case TimeFormatUnixMilli: case TimeFormatUnixMilli:
return t.UnixMilli() return t.UnixMilli()
case TimeFormatUnixMicro: case TimeFormatUnixMicro:

View file

@ -1,9 +1,7 @@
package sqlite3 package sqlite3
import ( import (
"encoding/json"
"math" "math"
"strconv"
"time" "time"
"github.com/ncruces/go-sqlite3/internal/util" "github.com/ncruces/go-sqlite3/internal/util"
@ -162,27 +160,6 @@ func (v Value) Pointer() any {
return util.GetHandle(v.c.ctx, ptr) return util.GetHandle(v.c.ctx, ptr)
} }
// JSON parses a JSON-encoded value
// and stores the result in the value pointed to by ptr.
func (v Value) JSON(ptr any) error {
var data []byte
switch v.Type() {
case NULL:
data = []byte("null")
case TEXT:
data = v.RawText()
case BLOB:
data = v.RawBlob()
case INTEGER:
data = strconv.AppendInt(nil, v.Int64(), 10)
case FLOAT:
data = util.AppendNumber(nil, v.Float())
default:
panic(util.AssertErr())
}
return json.Unmarshal(data, ptr)
}
// NoChange returns true if and only if the value is unchanged // NoChange returns true if and only if the value is unchanged
// in a virtual table update operatiom. // in a virtual table update operatiom.
// //

View file

@ -94,6 +94,10 @@ const (
OPEN_PRIVATECACHE OpenFlag = 0x00040000 /* Ok for sqlite3_open_v2() */ OPEN_PRIVATECACHE OpenFlag = 0x00040000 /* Ok for sqlite3_open_v2() */
OPEN_WAL OpenFlag = 0x00080000 /* VFS only */ OPEN_WAL OpenFlag = 0x00080000 /* VFS only */
OPEN_NOFOLLOW OpenFlag = 0x01000000 /* Ok for sqlite3_open_v2() */ OPEN_NOFOLLOW OpenFlag = 0x01000000 /* Ok for sqlite3_open_v2() */
_FLAG_ATOMIC OpenFlag = 0x10000000
_FLAG_KEEP_WAL OpenFlag = 0x20000000
_FLAG_PSOW OpenFlag = 0x40000000
_FLAG_SYNC_DIR OpenFlag = 0x80000000
) )
// AccessFlag is a flag for the [VFS] Access method. // AccessFlag is a flag for the [VFS] Access method.

View file

@ -51,7 +51,7 @@ func (vfsOS) Delete(path string, syncDir bool) error {
return _OK return _OK
} }
defer f.Close() defer f.Close()
err = osSync(f, false, false) err = osSync(f, 0, SYNC_FULL)
if err != nil { if err != nil {
return _IOERR_DIR_FSYNC return _IOERR_DIR_FSYNC
} }
@ -132,13 +132,15 @@ func (vfsOS) OpenFilename(name *Filename, flags OpenFlag) (File, OpenFlag, error
file := vfsFile{ file := vfsFile{
File: f, File: f,
psow: true, flags: flags | _FLAG_PSOW,
atomic: osBatchAtomic(f),
readOnly: flags&OPEN_READONLY != 0,
syncDir: isUnix && isCreate && isJournl,
delete: !isUnix && flags&OPEN_DELETEONCLOSE != 0,
shm: NewSharedMemory(name.String()+"-shm", flags), shm: NewSharedMemory(name.String()+"-shm", flags),
} }
if osBatchAtomic(f) {
file.flags |= _FLAG_ATOMIC
}
if isUnix && isCreate && isJournl {
file.flags |= _FLAG_SYNC_DIR
}
return &file, flags, nil return &file, flags, nil
} }
@ -146,12 +148,7 @@ type vfsFile struct {
*os.File *os.File
shm SharedMemory shm SharedMemory
lock LockLevel lock LockLevel
readOnly bool flags OpenFlag
keepWAL bool
syncDir bool
atomic bool
delete bool
psow bool
} }
var ( var (
@ -164,7 +161,7 @@ var (
) )
func (f *vfsFile) Close() error { func (f *vfsFile) Close() error {
if f.delete { if !isUnix && f.flags&OPEN_DELETEONCLOSE != 0 {
defer os.Remove(f.Name()) defer os.Remove(f.Name())
} }
if f.shm != nil { if f.shm != nil {
@ -183,21 +180,18 @@ func (f *vfsFile) WriteAt(p []byte, off int64) (n int, err error) {
} }
func (f *vfsFile) Sync(flags SyncFlag) error { func (f *vfsFile) Sync(flags SyncFlag) error {
dataonly := (flags & SYNC_DATAONLY) != 0 err := osSync(f.File, f.flags, flags)
fullsync := (flags & 0x0f) == SYNC_FULL
err := osSync(f.File, fullsync, dataonly)
if err != nil { if err != nil {
return err return err
} }
if isUnix && f.syncDir { if isUnix && f.flags&_FLAG_SYNC_DIR != 0 {
f.syncDir = false f.flags ^= _FLAG_SYNC_DIR
d, err := os.Open(filepath.Dir(f.File.Name())) d, err := os.Open(filepath.Dir(f.File.Name()))
if err != nil { if err != nil {
return nil return nil
} }
defer d.Close() defer d.Close()
err = osSync(d, false, false) err = osSync(f.File, f.flags, flags)
if err != nil { if err != nil {
return _IOERR_DIR_FSYNC return _IOERR_DIR_FSYNC
} }
@ -215,10 +209,10 @@ func (f *vfsFile) SectorSize() int {
func (f *vfsFile) DeviceCharacteristics() DeviceCharacteristic { func (f *vfsFile) DeviceCharacteristics() DeviceCharacteristic {
ret := IOCAP_SUBPAGE_READ ret := IOCAP_SUBPAGE_READ
if f.atomic { if f.flags&_FLAG_ATOMIC != 0 {
ret |= IOCAP_BATCH_ATOMIC ret |= IOCAP_BATCH_ATOMIC
} }
if f.psow { if f.flags&_FLAG_PSOW != 0 {
ret |= IOCAP_POWERSAFE_OVERWRITE ret |= IOCAP_POWERSAFE_OVERWRITE
} }
if runtime.GOOS == "windows" { if runtime.GOOS == "windows" {
@ -250,7 +244,19 @@ func (f *vfsFile) HasMoved() (bool, error) {
} }
func (f *vfsFile) LockState() LockLevel { return f.lock } func (f *vfsFile) LockState() LockLevel { return f.lock }
func (f *vfsFile) PowersafeOverwrite() bool { return f.psow } func (f *vfsFile) PowersafeOverwrite() bool { return f.flags&_FLAG_PSOW != 0 }
func (f *vfsFile) PersistWAL() bool { return f.keepWAL } func (f *vfsFile) PersistWAL() bool { return f.flags&_FLAG_KEEP_WAL != 0 }
func (f *vfsFile) SetPowersafeOverwrite(psow bool) { f.psow = psow }
func (f *vfsFile) SetPersistWAL(keepWAL bool) { f.keepWAL = keepWAL } func (f *vfsFile) SetPowersafeOverwrite(psow bool) {
f.flags &^= _FLAG_PSOW
if psow {
f.flags |= _FLAG_PSOW
}
}
func (f *vfsFile) SetPersistWAL(keepWAL bool) {
f.flags &^= _FLAG_KEEP_WAL
if keepWAL {
f.flags |= _FLAG_KEEP_WAL
}
}

View file

@ -41,7 +41,7 @@ func (f *vfsFile) Lock(lock LockLevel) error {
} }
// Do not allow any kind of write-lock on a read-only database. // Do not allow any kind of write-lock on a read-only database.
if f.readOnly && lock >= LOCK_RESERVED { if lock >= LOCK_RESERVED && f.flags&OPEN_READONLY != 0 {
return _IOERR_LOCK return _IOERR_LOCK
} }

View file

@ -7,3 +7,6 @@ It has some benefits over the C version:
- the memory backing the database needs not be contiguous, - the memory backing the database needs not be contiguous,
- the database can grow/shrink incrementally without copying, - the database can grow/shrink incrementally without copying,
- reader-writer concurrency is slightly improved. - reader-writer concurrency is slightly improved.
[`memdb.TestDB`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/memdb#TestDB)
is the preferred way to setup an in-memory database for testing.

View file

@ -10,6 +10,7 @@
package memdb package memdb
import ( import (
"crypto/rand"
"fmt" "fmt"
"net/url" "net/url"
"sync" "sync"
@ -74,11 +75,27 @@ func Delete(name string) {
// TestDB creates an empty shared memory database for the test to use. // TestDB creates an empty shared memory database for the test to use.
// The database is automatically deleted when the test and all its subtests complete. // The database is automatically deleted when the test and all its subtests complete.
// Returns a URI filename appropriate to call Open with.
// Each subsequent call to TestDB returns a unique database. // Each subsequent call to TestDB returns a unique database.
//
// func Test_something(t *testing.T) {
// t.Parallel()
// dsn := memdb.TestDB(t, url.Values{
// "_pragma": {"busy_timeout(1000)"},
// })
//
// db, err := sql.Open("sqlite3", dsn)
// if err != nil {
// t.Fatal(err)
// }
// defer db.Close()
//
// // ...
// }
func TestDB(tb testing.TB, params ...url.Values) string { func TestDB(tb testing.TB, params ...url.Values) string {
tb.Helper() tb.Helper()
name := fmt.Sprintf("%s_%p", tb.Name(), tb) name := fmt.Sprintf("%s_%s", tb.Name(), rand.Text())
tb.Cleanup(func() { Delete(name) }) tb.Cleanup(func() { Delete(name) })
Create(name, nil) Create(name, nil)

View file

@ -23,12 +23,26 @@ type flocktimeout_t struct {
timeout unix.Timespec timeout unix.Timespec
} }
func osSync(file *os.File, fullsync, _ /*dataonly*/ bool) error { func osSync(file *os.File, open OpenFlag, sync SyncFlag) error {
if fullsync { var cmd int
return file.Sync() if sync&SYNC_FULL == SYNC_FULL {
// For rollback journals all we really need is a barrier.
if open&OPEN_MAIN_JOURNAL != 0 {
cmd = unix.F_BARRIERFSYNC
} else {
cmd = unix.F_FULLFSYNC
} }
}
fd := file.Fd()
for { for {
err := unix.Fsync(int(file.Fd())) err := error(unix.ENOTSUP)
if cmd != 0 {
_, err = unix.FcntlInt(fd, cmd, 0)
}
if err == unix.ENOTSUP {
err = unix.Fsync(int(fd))
}
if err != unix.EINTR { if err != unix.EINTR {
return err return err
} }

View file

@ -10,7 +10,7 @@ import (
"golang.org/x/sys/unix" "golang.org/x/sys/unix"
) )
func osSync(file *os.File, _ /*fullsync*/, _ /*dataonly*/ bool) error { func osSync(file *os.File, _ OpenFlag, _ SyncFlag) error {
// SQLite trusts Linux's fdatasync for all fsync's. // SQLite trusts Linux's fdatasync for all fsync's.
for { for {
err := unix.Fdatasync(int(file.Fd())) err := unix.Fdatasync(int(file.Fd()))

View file

@ -4,6 +4,6 @@ package vfs
import "os" import "os"
func osSync(file *os.File, _ /*fullsync*/, _ /*dataonly*/ bool) error { func osSync(file *os.File, _ OpenFlag, _ SyncFlag) error {
return file.Sync() return file.Sync()
} }

12
vendor/modules.txt vendored
View file

@ -271,11 +271,11 @@ codeberg.org/gruf/go-mangler/v2
# codeberg.org/gruf/go-maps v1.0.4 # codeberg.org/gruf/go-maps v1.0.4
## explicit; go 1.20 ## explicit; go 1.20
codeberg.org/gruf/go-maps codeberg.org/gruf/go-maps
# codeberg.org/gruf/go-mempool v0.0.0-20240507125005-cef10d64a760 # codeberg.org/gruf/go-mempool v0.0.0-20251003110531-b54adae66253
## explicit; go 1.22.2 ## explicit; go 1.24.0
codeberg.org/gruf/go-mempool codeberg.org/gruf/go-mempool
# codeberg.org/gruf/go-mutexes v1.5.3 # codeberg.org/gruf/go-mutexes v1.5.8
## explicit; go 1.22.2 ## explicit; go 1.24.0
codeberg.org/gruf/go-mutexes codeberg.org/gruf/go-mutexes
# codeberg.org/gruf/go-runners v1.6.3 # codeberg.org/gruf/go-runners v1.6.3
## explicit; go 1.19 ## explicit; go 1.19
@ -293,7 +293,7 @@ codeberg.org/gruf/go-storage/disk
codeberg.org/gruf/go-storage/internal codeberg.org/gruf/go-storage/internal
codeberg.org/gruf/go-storage/memory codeberg.org/gruf/go-storage/memory
codeberg.org/gruf/go-storage/s3 codeberg.org/gruf/go-storage/s3
# codeberg.org/gruf/go-structr v0.9.9 # codeberg.org/gruf/go-structr v0.9.12
## explicit; go 1.24.5 ## explicit; go 1.24.5
codeberg.org/gruf/go-structr codeberg.org/gruf/go-structr
# codeberg.org/gruf/go-xunsafe v0.0.0-20250809104800-512a9df57d73 # codeberg.org/gruf/go-xunsafe v0.0.0-20250809104800-512a9df57d73
@ -727,7 +727,7 @@ github.com/modern-go/reflect2
# github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 # github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822
## explicit ## explicit
github.com/munnerz/goautoneg github.com/munnerz/goautoneg
# github.com/ncruces/go-sqlite3 v0.29.0 # github.com/ncruces/go-sqlite3 v0.29.1
## explicit; go 1.24.0 ## explicit; go 1.24.0
github.com/ncruces/go-sqlite3 github.com/ncruces/go-sqlite3
github.com/ncruces/go-sqlite3/driver github.com/ncruces/go-sqlite3/driver