package config
import (
"fmt"
"os"
fp "path/filepath"
"github.com/BurntSushi/toml"
)
const ConfigStr = `# Configuration for my-log
[input]
# Path to where the log files are stored
path = "%s"
# File extension for log files
ext = "txt"
# Whether to look in sub-folders
recurse = true
# config for output types
[output]
# This one just prints the logs to stdout when run
[output.stdout]
enabled = true
[output.stdout.config]
# Whether to output as JSON. Maybe useful to pipe elsewhere.
json = false
`
func DefaultStr() string {
home, _ := os.UserHomeDir()
inDir := fp.Join(home, "my-log")
return fmt.Sprintf(ConfigStr, inDir)
}
func DefaultConfig() (Config, error) {
s := DefaultStr()
c := Config{}
_, err := toml.Decode(s, &c)
return c, err
}
package config
import (
"encoding/json"
"fmt"
"os"
fp "path/filepath"
"time"
"codeberg.org/danjones000/my-log/tools"
"github.com/BurntSushi/toml"
"github.com/caarlos0/env/v10"
mapst "github.com/mitchellh/mapstructure"
)
var ConfigPath string
var Overrides = map[string]string{}
func init() {
conf, _ := os.UserConfigDir()
ConfigPath = fp.Join(conf, "my-log", "config.toml")
}
func Load() (Config, error) {
c, _ := DefaultConfig()
_, err := os.Stat(ConfigPath)
if !os.IsNotExist(err) {
_, err = toml.DecodeFile(ConfigPath, &c)
if err != nil {
return c, err
}
}
env.Parse(&c)
c.Outputs["stdout"] = loadStdout(c.Outputs["stdout"])
l := ""
for k, v := range Overrides {
val := tools.ParseString(v)
if val == nil {
continue
}
if _, isJson := val.(json.RawMessage); isJson {
continue
}
valout := fmt.Sprintf("%v", val)
if vals, isString := val.(string); isString {
valout = fmt.Sprintf(`"%s"`, vals)
}
if valt, isTime := val.(time.Time); isTime {
valout = valt.Format(time.RFC3339)
}
l = l + "\n" + fmt.Sprintf("%s = %s", k, valout)
}
_, err = toml.Decode(l, &c)
return c, err
}
func loadStdout(stdout Output) Output {
st := stdoutEnabled{stdout.Enabled}
env.Parse(&st)
stdout.Enabled = st.Enabled
var std Stdout
mapst.Decode(stdout.Config, &std)
env.Parse(&std)
mapst.Decode(std, &stdout.Config)
return stdout
}
func (oo Outputs) Stdout() (s Stdout, enabled bool) {
o, ok := oo["stdout"]
if !ok {
return s, false
}
enabled = o.Enabled
mapst.Decode(o.Config, &s)
return
}
package files
import (
"fmt"
"os"
fp "path/filepath"
"strings"
"codeberg.org/danjones000/my-log/config"
"codeberg.org/danjones000/my-log/models"
)
func Append(l models.Log) error {
conf, err := config.Load()
if err != nil {
return err
}
filename := fmt.Sprintf("%s.%s", strings.ReplaceAll(l.Name, ".", string(os.PathSeparator)), conf.Input.Ext)
path := fp.Join(conf.Input.Path, filename)
dir := fp.Dir(path)
err = os.MkdirAll(dir, 0750)
if err != nil {
return err
}
f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0640)
if err != nil {
return err
}
defer f.Close()
for _, e := range l.Entries {
by, err := e.MarshalText()
if err != nil {
continue
}
f.Write(by)
}
return nil
}
package models
import (
"bufio"
"bytes"
"encoding/json"
"errors"
"regexp"
"strings"
"sync"
"time"
)
const DateFormat = "January 02, 2006 at 03:04:05PM -0700"
type Entry struct {
Title string
Date time.Time
Fields []Meta
skipMissing bool
}
func PartialEntry() Entry {
return Entry{skipMissing: true}
}
type metaRes struct {
out []byte
err error
}
func (e Entry) getFieldMarshalChan() chan metaRes {
size := len(e.Fields)
ch := make(chan metaRes, size)
var wg sync.WaitGroup
for i := 0; i < size; i++ {
wg.Add(1)
go func(m Meta) {
defer wg.Done()
if m.Key == "json" {
if j, ok := m.Value.(json.RawMessage); ok {
sub := Entry{skipMissing: true}
json.Unmarshal(j, &sub)
for _, subM := range sub.Fields {
o, er := subM.MarshalText()
ch <- metaRes{o, er}
}
}
} else {
o, er := m.MarshalText()
ch <- metaRes{o, er}
}
}(e.Fields[i])
}
go func() {
wg.Wait()
close(ch)
}()
return ch
}
func (e Entry) MarshalText() ([]byte, error) {
e.Title = strings.TrimSpace(e.Title)
if e.Title == "" {
return []byte{}, ErrorMissingTitle
}
if e.Date == (time.Time{}) {
return []byte{}, ErrorMissingDate
}
ch := e.getFieldMarshalChan()
buff := &bytes.Buffer{}
buff.WriteString("\n@begin ")
buff.WriteString(e.Date.Format(DateFormat))
buff.WriteString(" - ")
buff.WriteString(e.Title)
for res := range ch {
if res.err == nil && len(res.out) > 0 {
buff.WriteString("\n")
buff.Write(res.out)
}
}
buff.WriteString(" @end")
return buff.Bytes(), nil
}
func (m *Entry) UnmarshalText(in []byte) error {
re := regexp.MustCompile("(?s)^@begin (.+) - (.+?)[ \n]@")
match := re.FindSubmatch(in)
if len(match) == 0 {
return newParsingError(errors.New("Failed to find title and date"))
}
ch := m.getFieldUnarshalChan(in)
title := bytes.TrimSpace(match[2])
if len(title) == 0 {
return ErrorMissingTitle
}
m.Title = string(title)
date := string(bytes.TrimSpace(match[1]))
if date == "" {
return ErrorMissingDate
}
d, e := time.Parse(time.RFC3339, date)
if e != nil {
d, e = time.Parse(DateFormat, date)
if e != nil {
return newParsingError(e)
}
}
m.Date = d
for meta := range ch {
m.Fields = append(m.Fields, meta)
}
return nil
}
func scanEntry(data []byte, atEOF bool) (advance int, token []byte, err error) {
if atEOF && len(data) == 0 {
return 0, nil, nil
}
if i := bytes.Index(data, []byte{10, 64}); i > 0 {
return i + 1, data[0:i], nil
}
if atEOF {
end := []byte{32, 64, 101, 110, 100}
token = data
if i := bytes.Index(data, end); i >= 0 {
token = data[0:i]
}
return len(data), token, nil
}
// Request more data.
return 0, nil, nil
}
func (e *Entry) getFieldUnarshalChan(in []byte) chan Meta {
size := len(in) / 3 // rough estimation
ch := make(chan Meta, size)
var wg sync.WaitGroup
read := bytes.NewReader(in)
scan := bufio.NewScanner(read)
scan.Split(scanEntry)
scan.Scan() // throw out first line
for scan.Scan() {
wg.Add(1)
go func(field []byte) {
defer wg.Done()
m := new(Meta)
err := m.UnmarshalText(field)
if err == nil {
if m.Key == "json" {
if j, ok := m.Value.(json.RawMessage); ok {
sub := Entry{skipMissing: true}
json.Unmarshal(j, &sub)
for _, subM := range sub.Fields {
ch <- subM
}
}
} else {
ch <- *m
}
}
}(scan.Bytes())
}
go func() {
wg.Wait()
close(ch)
}()
return ch
}
func (e Entry) MarshalJSON() ([]byte, error) {
if e.Title == "" {
return []byte{}, ErrorMissingTitle
}
if e.Date == (time.Time{}) {
return []byte{}, ErrorMissingDate
}
out := map[string]any{}
out["title"] = e.Title
out["date"] = e.Date.Format(time.RFC3339)
for _, f := range e.Fields {
if _, ok := out[f.Key]; !ok {
if f.Key == "json" {
ob := map[string]any{}
if j, ok := f.Value.(json.RawMessage); ok {
json.Unmarshal(j, &ob)
}
// If we couldn't get valid data from there, this will just be empty
for k, v := range ob {
if k != "title" && k != "date" {
out[k] = v
}
}
} else {
out[f.Key] = f.Value
if vt, ok := f.Value.(time.Time); ok {
out[f.Key] = vt.Format(time.RFC3339)
}
}
}
}
return json.Marshal(out)
}
func (e *Entry) unmarshalJsonChanHelper(m map[string]any, ch chan Meta, wg *sync.WaitGroup) {
for k, v := range m {
wg.Add(1)
go func(key string, value any) {
defer wg.Done()
if key != "json" {
ch <- Meta{key, value}
return
}
subM := map[string]any{}
if s, ok := value.(string); ok {
dec := json.NewDecoder(strings.NewReader(s))
dec.UseNumber()
dec.Decode(&subM)
} else {
subM = value.(map[string]any)
}
e.unmarshalJsonChanHelper(subM, ch, wg)
}(k, v)
}
}
func (e *Entry) getUnmarshalJsonChan(m map[string]any) chan Meta {
ch := make(chan Meta, len(m))
var wg sync.WaitGroup
e.unmarshalJsonChanHelper(m, ch, &wg)
go func() {
wg.Wait()
close(ch)
}()
return ch
}
func (e *Entry) UnmarshalJSON(in []byte) error {
out := map[string]any{}
dec := json.NewDecoder(bytes.NewReader(in))
dec.UseNumber()
err := dec.Decode(&out)
if err != nil {
return newParsingError(err)
}
title, ok := out["title"].(string)
if (!ok || title == "") && !e.skipMissing {
return ErrorMissingTitle
}
e.Title = title
dates, ok := out["date"].(string)
if (!ok || dates == "") && !e.skipMissing {
return ErrorMissingDate
}
date, err := time.Parse(time.RFC3339, dates)
if err != nil && !e.skipMissing {
return newParsingError(err)
}
e.Date = date
ch := e.getUnmarshalJsonChan(out)
for m := range ch {
if m.Key == "title" || m.Key == "date" {
continue
} else if vs, ok := m.Value.(string); ok {
if vd, err := time.Parse(time.RFC3339, vs); err == nil {
m.Value = vd
} else {
m.Value = vs
}
} else if n, ok := m.Value.(json.Number); ok {
it, _ := n.Int64()
fl, _ := n.Float64()
if float64(it) == fl {
m.Value = it
} else {
m.Value = fl
}
}
e.Fields = append(e.Fields, m)
}
return nil
}
package models
import (
"errors"
"fmt"
)
var ErrorMissingTitle = errors.New("Missing title")
var ErrorMissingDate = errors.New("Missing date")
var ErrorParsing = errors.New("Parsing Error")
func newParsingError(err error) error {
return fmt.Errorf("%w: %w", ErrorParsing, err)
}
package models
import (
"bufio"
"bytes"
"regexp"
"sync"
)
var reg = regexp.MustCompile("(?sm)^@begin .+?(^| )@end")
type Log struct {
Name string
Entries []Entry
}
func (l *Log) UnmarshalText(in []byte) error {
ch := l.getLogUnarshalChan(in)
for entry := range ch {
l.Entries = append(l.Entries, entry)
}
return nil
}
func scanLog(data []byte, atEOF bool) (advance int, token []byte, err error) {
if atEOF && len(data) == 0 {
// done
return 0, nil, nil
}
m := reg.FindIndex(data)
if len(m) == 0 && atEOF {
// all trash
return len(data), nil, nil
} else if len(m) == 0 && !atEOF {
// get more
return 0, nil, nil
}
return m[1], data[m[0]:m[1]], nil
}
func (l *Log) getLogUnarshalChan(in []byte) chan Entry {
size := len(in) / 10 // rough estimation
ch := make(chan Entry, size)
var wg sync.WaitGroup
read := bytes.NewReader(in)
scan := bufio.NewScanner(read)
scan.Split(scanLog)
for scan.Scan() {
wg.Add(1)
go func(field []byte) {
defer wg.Done()
f := new(Entry)
err := f.UnmarshalText(field)
if err != nil {
return
}
ch <- *f
}(scan.Bytes())
}
go func() {
wg.Wait()
close(ch)
}()
return ch
}
package models
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"regexp"
"strconv"
"time"
"codeberg.org/danjones000/my-log/tools"
)
type Meta struct {
Key string
Value any
}
func (m Meta) MarshalText() ([]byte, error) {
if regexp.MustCompile(`\s`).MatchString(m.Key) {
return []byte{}, fmt.Errorf("whitespace is not allowed in key: %s", m.Key)
}
buff := &bytes.Buffer{}
buff.WriteRune('@')
buff.WriteString(m.Key)
buff.WriteRune(' ')
switch v := m.Value.(type) {
default:
return nil, fmt.Errorf("Unknown type %T", v)
case nil:
return []byte{}, nil
case string:
buff.WriteString(v)
case int:
buff.WriteString(strconv.Itoa(v))
case int64:
buff.WriteString(strconv.FormatInt(v, 10))
case float64:
buff.WriteString(strconv.FormatFloat(v, 'f', -1, 64))
case json.Number:
buff.WriteString(v.String())
case json.RawMessage:
buff.Write(v)
case []byte:
buff.Write(v)
case byte:
buff.WriteByte(v)
case rune:
buff.WriteString(string(v))
case bool:
buff.WriteString(strconv.FormatBool(v))
case time.Time:
buff.WriteString(v.Format(time.RFC3339))
}
return buff.Bytes(), nil
}
func (m *Meta) UnmarshalText(in []byte) error {
if len(in) == 0 {
return newParsingError(errors.New("Unable to Unmarshal empty string"))
}
re := regexp.MustCompile("(?s)^@([^ ]+) (.*)( @end)?$")
match := re.FindSubmatch(in)
if len(match) == 0 {
return newParsingError(fmt.Errorf("Failed to match %s", in))
}
m.Key = string(match[1])
return m.processMeta(match[2])
}
func (m *Meta) processMeta(in []byte) error {
if len(in) == 0 {
return newParsingError(errors.New("No value found"))
}
v := tools.ParseBytes(in)
if v == "" {
return newParsingError(errors.New("No value found"))
}
m.Value = v
return nil
}
package tools
import (
"encoding/json"
"regexp"
"strconv"
"strings"
"time"
)
func ParseBytes(in []byte) any {
return ParseString(string(in))
}
func ParseString(in string) any {
s := strings.TrimSpace(in)
if s == "" {
return s
}
yesno := regexp.MustCompile("^(y|Y|yes|Yes|YES|n|N|no|No|NO|true|True|TRUE|false|False|FALSE|on|On|ON|off|Off|OFF)$")
yes := regexp.MustCompile("^(y|Y|yes|Yes|YES|true|True|TRUE|on|On|ON)$")
null := regexp.MustCompile("^(~|null|Null|NULL|none|None|NONE|nil|Nil|NIL)$")
var j json.RawMessage
if null.MatchString(s) {
return nil
} else if yesno.MatchString(s) {
if yes.MatchString(s) {
return true
} else {
return false
}
} else if i, err := strconv.Atoi(s); err == nil {
return i
} else if f, err := strconv.ParseFloat(s, 64); err == nil {
return f
} else if t, err := time.Parse(time.RFC3339, s); err == nil {
return t
} else if err := json.Unmarshal([]byte(s), &j); err == nil {
return j
}
return s
}
package tools
import (
"time"
dp "github.com/markusmobius/go-dateparser"
"github.com/markusmobius/go-dateparser/date"
)
const (
day = time.Hour * 24
)
// These are somewhat arbitrary, but reasonably useful min and max times
var (
MinTime = time.Unix(-2208988800, 0) // Jan 1, 1900
MaxTime = MinTime.Add(1<<63 - 1)
)
func ParseDate(in string) (t time.Time, err error) {
if in == "min" {
return MinTime, nil
}
if in == "max" {
return MaxTime, nil
}
d, err := dp.Parse(nil, in)
if err != nil {
return
}
t = d.Time.Local()
trunc := time.Second
switch d.Period {
case date.Minute:
trunc = time.Minute
case date.Hour:
trunc = time.Hour
case date.Day:
trunc = day
// @todo Handle other cases separately
}
t = t.Truncate(trunc)
return
}