gotosocial/vendor/codeberg.org/gruf/go-structr/runtime.go

295 lines
7 KiB
Go
Raw Normal View History

package structr
import (
"fmt"
"reflect"
2024-10-02 10:58:20 +00:00
"runtime"
"strings"
"unicode"
"unicode/utf8"
"unsafe"
"codeberg.org/gruf/go-mangler/v2"
"codeberg.org/gruf/go-xunsafe"
)
// struct_field contains pre-prepared type
// information about a struct's field member,
// including memory offset and hash function.
type struct_field struct {
// struct field type mangling
// (i.e. fast serializing) fn.
mangle mangler.Mangler
// zero value data, used when
// nil encountered during ptr
// offset following.
zero unsafe.Pointer
// mangled zero value string,
// to check zero value keys.
zerostr string
2024-08-14 12:08:24 +00:00
// offsets defines whereabouts in
// memory this field is located,
// and after how many dereferences.
2024-08-14 12:08:24 +00:00
offsets []next_offset
}
// next_offset defines a next offset location
// in a struct_field, first by the number of
// derefences required, then by offset from
// that final memory location.
type next_offset struct {
derefs int
offset uintptr
}
// get_type_iter returns a prepared xunsafe.TypeIter{} for generic parameter type,
// with flagIndir specifically set as we always take a reference to value type.
func get_type_iter[T any]() xunsafe.TypeIter {
rtype := reflect.TypeOf((*T)(nil)).Elem()
flags := xunsafe.Reflect_flag(xunsafe.Abi_Type_Kind(rtype))
flags |= xunsafe.Reflect_flagIndir // always comes from unsafe ptr
return xunsafe.ToTypeIter(rtype, flags)
}
// find_field will search for a struct field with given set of names,
// where names is a len > 0 slice of names account for struct nesting.
func find_field(t xunsafe.TypeIter, names []string) (sfield struct_field, ftype reflect.Type) {
var (
// is_exported returns whether name is exported
// from a package; can be func or struct field.
is_exported = func(name string) bool {
r, _ := utf8.DecodeRuneInString(name)
return unicode.IsUpper(r)
}
// pop_name pops the next name from
// the provided slice of field names.
pop_name = func() string {
name := names[0]
names = names[1:]
if !is_exported(name) {
[performance] rewrite timelines to rely on new timeline cache type (#3941) * start work rewriting timeline cache type * further work rewriting timeline caching * more work integration new timeline code * remove old code * add local timeline, fix up merge conflicts * remove old use of go-bytes * implement new timeline code into more areas of codebase, pull in latest go-mangler, go-mutexes, go-structr * remove old timeline package, add local timeline cache * remove references to old timeline types that needed starting up in tests * start adding page validation * fix test-identified timeline cache package issues * fix up more tests, fix missing required changes, etc * add exclusion for test.out in gitignore * clarify some things better in code comments * tweak cache size limits * fix list timeline cache fetching * further list timeline fixes * linter, ssssssssshhhhhhhhhhhh please * fix linter hints * reslice the output if it's beyond length of 'lim' * remove old timeline initialization code, bump go-structr to v0.9.4 * continued from previous commit * improved code comments * don't allow multiple entries for BoostOfID values to prevent repeated boosts of same boosts * finish writing more code comments * some variable renaming, for ease of following * change the way we update lo,hi paging values during timeline load * improved code comments for updated / returned lo , hi paging values * finish writing code comments for the StatusTimeline{} type itself * fill in more code comments * update go-structr version to latest with changed timeline unique indexing logic * have a local and public timeline *per user* * rewrite calls to public / local timeline calls * remove the zero length check, as lo, hi values might still be set * simplify timeline cache loading, fix lo/hi returns, fix timeline invalidation side-effects missing for some federated actions * swap the lo, hi values :facepalm: * add (now) missing slice reverse of tag timeline statuses when paging ASC * remove local / public caches (is out of scope for this work), share more timeline code * remove unnecessary change * again, remove more unused code * remove unused function to appease the linter * move boost checking to prepare function * fix use of timeline.lastOrder, fix incorrect range functions used * remove comments for repeat code * remove the boost logic from prepare function * do a maximum of 5 loads, not 10 * add repeat boost filtering logic, update go-structr, general improvements * more code comments * add important note * fix timeline tests now that timelines are returned in page order * remove unused field * add StatusTimeline{} tests * add more status timeline tests * start adding preloading support * ensure repeat boosts are marked in preloaded entries * share a bunch of the database load code in timeline cache, don't clear timelines on relationship change * add logic to allow dynamic clear / preloading of timelines * comment-out unused functions, but leave in place as we might end-up using them * fix timeline preload state check * much improved status timeline code comments * more code comments, don't bother inserting statuses if timeline not preloaded * shift around some logic to make sure things aren't accidentally left set * finish writing code comments * remove trim-after-insert behaviour * fix-up some comments referring to old logic * remove unsetting of lo, hi * fix preload repeatBoost checking logic * don't return on status filter errors, these are usually transient * better concurrency safety in Clear() and Done() * fix test broken due to addition of preloader * fix repeatBoost logic that doesn't account for already-hidden repeatBoosts * ensure edit submodels are dropped on cache insertion * update code-comment to expand CAS accronym * use a plus1hULID() instead of 24h * remove unused functions * add note that public / local timeline requester can be nil * fix incorrect visibility filtering of tag timeline statuses * ensure we filter home timeline statuses on local only * some small re-orderings to confirm query params in correct places * fix the local only home timeline filter func
2025-04-26 09:56:15 +00:00
panic(fmt.Sprintf("field is not exported: %s", name))
}
return name
}
// field is the iteratively searched
// struct field value in below loop.
field reflect.StructField
)
// Take reference
// of parent iter.
o := t
for len(names) > 0 {
// Pop next name.
name := pop_name()
var n int
rtype := t.Type
flags := t.Flag
// Iteratively dereference pointer types.
for rtype.Kind() == reflect.Pointer {
// If this actual indirect memory,
// increase dereferences counter.
if flags&xunsafe.Reflect_flagIndir != 0 {
n++
}
// Get next elem type.
rtype = rtype.Elem()
// Get next set of dereferenced element type flags.
flags = xunsafe.ReflectPointerElemFlags(flags, rtype)
// Update type iter info.
t = t.Child(rtype, flags)
}
// Check for valid struct type.
if rtype.Kind() != reflect.Struct {
panic(fmt.Sprintf("field %s is not struct (or ptr-to): %s", rtype, name))
}
// Set offset info.
var off next_offset
off.derefs = n
var ok bool
// Look for the next field by name.
field, ok = rtype.FieldByName(name)
if !ok {
[performance] rewrite timelines to rely on new timeline cache type (#3941) * start work rewriting timeline cache type * further work rewriting timeline caching * more work integration new timeline code * remove old code * add local timeline, fix up merge conflicts * remove old use of go-bytes * implement new timeline code into more areas of codebase, pull in latest go-mangler, go-mutexes, go-structr * remove old timeline package, add local timeline cache * remove references to old timeline types that needed starting up in tests * start adding page validation * fix test-identified timeline cache package issues * fix up more tests, fix missing required changes, etc * add exclusion for test.out in gitignore * clarify some things better in code comments * tweak cache size limits * fix list timeline cache fetching * further list timeline fixes * linter, ssssssssshhhhhhhhhhhh please * fix linter hints * reslice the output if it's beyond length of 'lim' * remove old timeline initialization code, bump go-structr to v0.9.4 * continued from previous commit * improved code comments * don't allow multiple entries for BoostOfID values to prevent repeated boosts of same boosts * finish writing more code comments * some variable renaming, for ease of following * change the way we update lo,hi paging values during timeline load * improved code comments for updated / returned lo , hi paging values * finish writing code comments for the StatusTimeline{} type itself * fill in more code comments * update go-structr version to latest with changed timeline unique indexing logic * have a local and public timeline *per user* * rewrite calls to public / local timeline calls * remove the zero length check, as lo, hi values might still be set * simplify timeline cache loading, fix lo/hi returns, fix timeline invalidation side-effects missing for some federated actions * swap the lo, hi values :facepalm: * add (now) missing slice reverse of tag timeline statuses when paging ASC * remove local / public caches (is out of scope for this work), share more timeline code * remove unnecessary change * again, remove more unused code * remove unused function to appease the linter * move boost checking to prepare function * fix use of timeline.lastOrder, fix incorrect range functions used * remove comments for repeat code * remove the boost logic from prepare function * do a maximum of 5 loads, not 10 * add repeat boost filtering logic, update go-structr, general improvements * more code comments * add important note * fix timeline tests now that timelines are returned in page order * remove unused field * add StatusTimeline{} tests * add more status timeline tests * start adding preloading support * ensure repeat boosts are marked in preloaded entries * share a bunch of the database load code in timeline cache, don't clear timelines on relationship change * add logic to allow dynamic clear / preloading of timelines * comment-out unused functions, but leave in place as we might end-up using them * fix timeline preload state check * much improved status timeline code comments * more code comments, don't bother inserting statuses if timeline not preloaded * shift around some logic to make sure things aren't accidentally left set * finish writing code comments * remove trim-after-insert behaviour * fix-up some comments referring to old logic * remove unsetting of lo, hi * fix preload repeatBoost checking logic * don't return on status filter errors, these are usually transient * better concurrency safety in Clear() and Done() * fix test broken due to addition of preloader * fix repeatBoost logic that doesn't account for already-hidden repeatBoosts * ensure edit submodels are dropped on cache insertion * update code-comment to expand CAS accronym * use a plus1hULID() instead of 24h * remove unused functions * add note that public / local timeline requester can be nil * fix incorrect visibility filtering of tag timeline statuses * ensure we filter home timeline statuses on local only * some small re-orderings to confirm query params in correct places * fix the local only home timeline filter func
2025-04-26 09:56:15 +00:00
panic(fmt.Sprintf("unknown field: %s", name))
}
// Set next offset value.
off.offset = field.Offset
sfield.offsets = append(sfield.offsets, off)
// Calculate value flags, and set next nested field type.
flags = xunsafe.ReflectStructFieldFlags(t.Flag, field.Type)
t = t.Child(field.Type, flags)
}
// Set final field type.
ftype = t.TypeInfo.Type
// Get mangler from type info.
sfield.mangle = mangler.Get(t)
// Calculate zero value string.
zptr := zero_value_ptr(o, sfield.offsets)
zstr := string(sfield.mangle(nil, zptr))
sfield.zerostr = zstr
sfield.zero = zptr
return
}
// zero_value iterates the type contained in TypeIter{} along the given
// next_offset{} values, creating new ptrs where necessary, returning the
// zero reflect.Value{} after fully iterating the next_offset{} slice.
func zero_value(t xunsafe.TypeIter, offsets []next_offset) reflect.Value {
v := reflect.New(t.Type).Elem()
for _, offset := range offsets {
for range offset.derefs {
if v.IsNil() {
new := reflect.New(v.Type().Elem())
v.Set(new)
}
v = v.Elem()
}
for i := 0; i < v.NumField(); i++ {
if v.Type().Field(i).Offset == offset.offset {
v = v.Field(i)
break
}
}
}
return v
}
// zero_value_ptr returns the unsafe pointer address of the result of zero_value().
func zero_value_ptr(t xunsafe.TypeIter, offsets []next_offset) unsafe.Pointer {
return zero_value(t, offsets).Addr().UnsafePointer()
}
// extract_fields extracts given structfields from the provided value type,
// this is done using predetermined struct field memory offset locations.
func extract_fields(ptr unsafe.Pointer, fields []struct_field) []unsafe.Pointer {
// Prepare slice of field value pointers.
ptrs := make([]unsafe.Pointer, len(fields))
if len(ptrs) != len(fields) {
panic(assert("BCE"))
}
for i, field := range fields {
// loop scope.
fptr := ptr
for _, offset := range field.offsets {
// Dereference any ptrs to offset.
fptr = deref(fptr, offset.derefs)
if fptr == nil {
break
}
// Jump forward by offset to next ptr.
fptr = unsafe.Pointer(uintptr(fptr) +
offset.offset)
}
if fptr == nil {
// Use zero value.
fptr = field.zero
}
[performance] cache more database calls, reduce required database calls overall (#3290) * improvements to caching for lists and relationship to accounts / follows * fix nil panic in AddToList() * ensure list related caches are correctly invalidated * ensure returned ID lists are ordered correctly * bump go-structr to v0.8.9 (returns early if zero uncached keys to be loaded) * remove zero checks in uncached key load functions (go-structr now handles this) * fix issues after rebase on upstream/main * update the expected return order of CSV exports (since list entries are now down by entry creation date) * rename some funcs, allow deleting list entries for multiple follow IDs at a time, fix up more tests * use returning statements on delete to get cache invalidation info * fixes to recent database delete changes * fix broken list entries delete sql * remove unused db function * update remainder of delete functions to behave in similar way, some other small tweaks * fix delete user sql, allow returning on err no entries * uncomment + fix list database tests * update remaining list tests * update envparsing test * add comments to each specific key being invalidated * add more cache invalidation explanatory comments * whoops; actually delete poll votes from database in the DeletePollByID() func * remove added but-commented-out field * improved comment regarding paging being disabled * make cache invalidation comments match what's actually happening * fix up delete query comments to match what is happening * rename function to read a bit better * don't use ErrNoEntries on delete when not needed (it's only needed for a RETURNING call) * update function name in test * move list exclusivity check to AFTER eligibility check. use log.Panic() instead of panic() * use the poll_id column in poll_votes for selecting votes in poll ID * fix function name
2024-09-16 16:46:09 +00:00
// Set field ptr.
ptrs[i] = fptr
}
[performance] cache more database calls, reduce required database calls overall (#3290) * improvements to caching for lists and relationship to accounts / follows * fix nil panic in AddToList() * ensure list related caches are correctly invalidated * ensure returned ID lists are ordered correctly * bump go-structr to v0.8.9 (returns early if zero uncached keys to be loaded) * remove zero checks in uncached key load functions (go-structr now handles this) * fix issues after rebase on upstream/main * update the expected return order of CSV exports (since list entries are now down by entry creation date) * rename some funcs, allow deleting list entries for multiple follow IDs at a time, fix up more tests * use returning statements on delete to get cache invalidation info * fixes to recent database delete changes * fix broken list entries delete sql * remove unused db function * update remainder of delete functions to behave in similar way, some other small tweaks * fix delete user sql, allow returning on err no entries * uncomment + fix list database tests * update remaining list tests * update envparsing test * add comments to each specific key being invalidated * add more cache invalidation explanatory comments * whoops; actually delete poll votes from database in the DeletePollByID() func * remove added but-commented-out field * improved comment regarding paging being disabled * make cache invalidation comments match what's actually happening * fix up delete query comments to match what is happening * rename function to read a bit better * don't use ErrNoEntries on delete when not needed (it's only needed for a RETURNING call) * update function name in test * move list exclusivity check to AFTER eligibility check. use log.Panic() instead of panic() * use the poll_id column in poll_votes for selecting votes in poll ID * fix function name
2024-09-16 16:46:09 +00:00
return ptrs
}
// pkey_field contains pre-prepared type
// information about a primary key struct's
// field member, including memory offset.
type pkey_field struct {
// zero value data, used when
// nil encountered during ptr
// offset following.
zero unsafe.Pointer
// offsets defines whereabouts in
// memory this field is located.
offsets []next_offset
}
// extract_pkey will extract a pointer from 'ptr', to
// the primary key struct field defined by 'field'.
func extract_pkey(ptr unsafe.Pointer, field pkey_field) unsafe.Pointer {
for _, offset := range field.offsets {
// Dereference any ptrs to offset.
ptr = deref(ptr, offset.derefs)
if ptr == nil {
break
}
// Jump forward by offset to next ptr.
ptr = unsafe.Pointer(uintptr(ptr) +
offset.offset)
}
if ptr == nil {
// Use zero value.
ptr = field.zero
}
return ptr
}
// deref will dereference ptr 'n' times (or until nil).
func deref(p unsafe.Pointer, n int) unsafe.Pointer {
for ; n > 0; n-- {
if p == nil {
return nil
}
p = *(*unsafe.Pointer)(p)
}
return p
}
// assert can be called to indicated a block
// of code should not be able to be reached,
// it returns a BUG report with callsite.
func assert(assert string) string {
2024-10-02 10:58:20 +00:00
pcs := make([]uintptr, 1)
_ = runtime.Callers(2, pcs)
funcname := "go-structr" // by default use just our library name
if frames := runtime.CallersFrames(pcs); frames != nil {
frame, _ := frames.Next()
funcname = frame.Function
2024-10-02 10:58:20 +00:00
if i := strings.LastIndexByte(funcname, '/'); i != -1 {
funcname = funcname[i+1:]
}
}
var buf strings.Builder
buf.Grow(32 + len(assert) + len(funcname))
buf.WriteString("BUG: assertion \"")
buf.WriteString(assert)
buf.WriteString("\" failed in ")
buf.WriteString(funcname)
return buf.String()
2024-10-02 10:58:20 +00:00
}