dan/my-log
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

✨ Special handling of json field

Browse source
This commit is contained in:
Dan Jones 2024-01-30 23:40:47 -06:00
commit 16a0a9d20b
4 changed files with 112 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

110
models/entry.go
View file

@ -14,9 +14,10 @@ import (
const DateFormat = "January 02, 2006 at 03:04:05PM -0700"
type Entry struct {
Title string
Date time.Time
Fields []Meta
Title string
Date time.Time
Fields []Meta
skipMissing bool
}
type metaRes struct {
@ -29,13 +30,23 @@ func (e Entry) getFieldMarshalChan() chan metaRes {
ch := make(chan metaRes, size)
var wg sync.WaitGroup
// @todo figure out a way to handle json field
for i := 0; i < size; i++ {
wg.Add(1)
go func(m Meta) {
defer wg.Done()
o, er := m.MarshalText()
ch <- metaRes{o, er}
if m.Key == "json" {
if j, ok := m.Value.(json.RawMessage); ok {
sub := Entry{skipMissing: true}
json.Unmarshal(j, &sub)
for _, subM := range sub.Fields {
o, er := subM.MarshalText()
ch <- metaRes{o, er}
}
}
} else {
o, er := m.MarshalText()
ch <- metaRes{o, er}
}
}(e.Fields[i])
}
@ -137,7 +148,6 @@ func (e *Entry) getFieldUnarshalChan(in []byte) chan Meta {
scan.Split(scanEntry)
scan.Scan() // throw out first line
// @todo figure out a way to handle json field
for scan.Scan() {
wg.Add(1)
go func(field []byte) {
@ -145,7 +155,17 @@ func (e *Entry) getFieldUnarshalChan(in []byte) chan Meta {
m := new(Meta)
err := m.UnmarshalText(field)
if err == nil {
ch <- *m
if m.Key == "json" {
if j, ok := m.Value.(json.RawMessage); ok {
sub := Entry{skipMissing: true}
json.Unmarshal(j, &sub)
for _, subM := range sub.Fields {
ch <- subM
}
}
} else {
ch <- *m
}
}
}(scan.Bytes())
}
@ -170,15 +190,63 @@ func (e Entry) MarshalJSON() ([]byte, error) {
out["date"] = e.Date.Format(time.RFC3339)
for _, f := range e.Fields {
if _, ok := out[f.Key]; !ok {
out[f.Key] = f.Value
if vt, ok := f.Value.(time.Time); ok {
out[f.Key] = vt.Format(time.RFC3339)
if f.Key == "json" {
ob := map[string]any{}
if j, ok := f.Value.(json.RawMessage); ok {
json.Unmarshal(j, &ob)
}
// If we couldn't get valid data from there, this will just be empty
for k, v := range ob {
if k != "title" && k != "date" {
out[k] = v
}
}
} else {
out[f.Key] = f.Value
if vt, ok := f.Value.(time.Time); ok {
out[f.Key] = vt.Format(time.RFC3339)
}
}
}
}
return json.Marshal(out)
}
func (e *Entry) unmarshalJsonChanHelper(m map[string]any, ch chan Meta, wg *sync.WaitGroup) {
for k, v := range m {
wg.Add(1)
go func(key string, value any) {
defer wg.Done()
if key != "json" {
ch <- Meta{key, value}
return
}
subM := map[string]any{}
if s, ok := value.(string); ok {
dec := json.NewDecoder(strings.NewReader(s))
dec.UseNumber()
dec.Decode(&subM)
} else {
subM = value.(map[string]any)
}
e.unmarshalJsonChanHelper(subM, ch, wg)
}(k, v)
}
}
func (e *Entry) getUnmarshalJsonChan(m map[string]any) chan Meta {
ch := make(chan Meta, len(m))
var wg sync.WaitGroup
e.unmarshalJsonChanHelper(m, ch, &wg)
go func() {
wg.Wait()
close(ch)
}()
return ch
}
func (e *Entry) UnmarshalJSON(in []byte) error {
out := map[string]any{}
dec := json.NewDecoder(bytes.NewReader(in))
@ -188,30 +256,30 @@ func (e *Entry) UnmarshalJSON(in []byte) error {
return newParsingError(err)
}
title, ok := out["title"].(string)
if !ok || title == "" {
if (!ok || title == "") && !e.skipMissing {
return ErrorMissingTitle
}
e.Title = title
delete(out, "title")
dates, ok := out["date"].(string)
if !ok || dates == "" {
if (!ok || dates == "") && !e.skipMissing {
return ErrorMissingDate
}
date, err := time.Parse(time.RFC3339, dates)
if err != nil {
if err != nil && !e.skipMissing {
return newParsingError(err)
}
e.Date = date
delete(out, "date")
for k, v := range out {
m := Meta{Key: k}
if vs, ok := v.(string); ok {
ch := e.getUnmarshalJsonChan(out)
for m := range ch {
if m.Key == "title" || m.Key == "date" {
continue
} else if vs, ok := m.Value.(string); ok {
if vd, err := time.Parse(time.RFC3339, vs); err == nil {
m.Value = vd
} else {
m.Value = vs
}
} else if n, ok := v.(json.Number); ok {
} else if n, ok := m.Value.(json.Number); ok {
it, _ := n.Int64()
fl, _ := n.Float64()
if float64(it) == fl {
@ -219,8 +287,6 @@ func (e *Entry) UnmarshalJSON(in []byte) error {
} else {
m.Value = fl
}
} else {
m.Value = v
}
e.Fields = append(e.Fields, m)
}

29
models/entry_test.go
View file

@ -52,7 +52,6 @@ func TestEntryMarshal(t *testing.T) {
[]string{"@age 41", "@cool true", "@name Jim"},
nil,
},
/* uncomment when implemented
{
"json-field",
"Title J",
@ -62,7 +61,6 @@ func TestEntryMarshal(t *testing.T) {
[]string{"@age 41", "@cool true", "@name Jim"},
nil,
},
*/
}
for _, tt := range tests {
@ -72,7 +70,7 @@ func TestEntryMarshal(t *testing.T) {
func getEntryMarshalTestRunner(title string, date time.Time, fields []Meta, first string, lines []string, err error) func(*testing.T) {
return func(t *testing.T) {
en := Entry{title, date, fields}
en := Entry{title, date, fields, false}
o, er := en.MarshalText()
assert.Equal(t, err, er)
if first == "" {
@ -127,16 +125,14 @@ func TestEntryUnmarshal(t *testing.T) {
[]Meta{{"me", json.RawMessage(`{"name":"Dan","coder":true}`)}},
nil,
},
/* uncomment when implemented
{
"json-field",
"@begin " + whens + " - Some Guy\n" + `@json {"name":"Dan","coder":true} @end`,
"A Title",
"Some Guy",
when,
[]Meta{{"name", "Dan"}, {"coder", true}},
nil,
},
*/
}
for _, tt := range tests {
@ -171,7 +167,7 @@ func getEntryUnmarshalTestRunner(in string, title string, date time.Time, fields
break
}
}
assert.Truef(t, got, "Couldn't find field %+v", f)
assert.Truef(t, got, "Couldn't find field %+v. We have %+v", f, e.Fields)
}
}
}
@ -212,6 +208,7 @@ func TestEntryJsonMarshal(t *testing.T) {
{"empty-date", "A Title", time.Time{}, simple, "", ErrorMissingDate},
{"obj-field", "A Title", when, []Meta{{"obj", json.RawMessage(`{"foo":"bar","title":"Sub-title"}`)}}, `{"title":"A Title","date":"` + whens + `","obj":{"foo":"bar","title":"Sub-title"}}`, nil},
{"date-field", "A Title", when, []Meta{{"when", when.Add(time.Hour)}}, `{"title":"A Title","date":"` + whens + `","when":"` + when.Add(time.Hour).Format(time.RFC3339) + `"}`, nil},
{"json-field", "A Title", when, []Meta{{"json", json.RawMessage(`{"age": 41, "cool": true, "name": "Jim"}`)}}, `{"title":"A Title","date":"` + whens + `","age":41,"cool": true, "name": "Jim"}`, nil},
}
for _, tt := range tests {
@ -221,7 +218,7 @@ func TestEntryJsonMarshal(t *testing.T) {
func getEntryJsonMarshalTestRunner(title string, date time.Time, fields []Meta, out string, err error) func(t *testing.T) {
return func(t *testing.T) {
e := Entry{title, date, fields}
e := Entry{title, date, fields, false}
o, er := json.Marshal(e)
if err == nil {
assert.JSONEq(t, out, string(o))
@ -275,6 +272,22 @@ func TestEntryJsonUnmarshal(t *testing.T) {
[]Meta{{"posted", when.Add(-time.Hour)}},
nil,
},
{
"json-field",
`{"title":"A Title","date":"` + whens + `","json":{"age": 41, "cool": true, "name": "Jim"}}`,
"A Title",
when,
[]Meta{{"age", int64(41)}, {"cool", true}, {"name", "Jim"}},
nil,
},
{
"json-field-embed",
`{"title":"A Title","date":"` + whens + `","json":"{\"age\": 41, \"cool\": true, \"name\": \"Jim\"}"}`,
"A Title",
when,
[]Meta{{"age", int64(41)}, {"cool", true}, {"name", "Jim"}},
nil,
},
}
for _, tt := range tests {

2
models/meta.go
View file

@ -33,6 +33,8 @@ func (m Meta) MarshalText() ([]byte, error) {
buff.WriteString(v)
case int:
buff.WriteString(strconv.Itoa(v))
case int64:
buff.WriteString(strconv.FormatInt(v, 10))
case float64:
buff.WriteString(strconv.FormatFloat(v, 'f', -1, 64))
case json.Number:

1
models/meta_test.go
View file

@ -27,6 +27,7 @@ func TestMeta(t *testing.T) {
newVal any
}{
{"int", "num", 42, "@num 42", nil, 42},
{"int64", "num", int64(42), "@num 42", nil, int(42)},
{"float", "num", 42.13, "@num 42.13", nil, 42.13},
{"string", "word", "hello", "@word hello", nil, "hello"},
{"json number", "num", json.Number("42.13"), "@num 42.13", nil, 42.13},