From f6539ef32ca33d5249a2494b30773a4ebdcf6101 Mon Sep 17 00:00:00 2001 From: mchavez Date: Thu, 29 Aug 2024 08:29:03 -0600 Subject: [PATCH] Adding statistics --- pkg/uhttp/dbcache.go | 200 ++++++++++++++++++++++++++++++++------ pkg/uhttp/dbcache_test.go | 4 +- 2 files changed, 172 insertions(+), 32 deletions(-) diff --git a/pkg/uhttp/dbcache.go b/pkg/uhttp/dbcache.go index fb0fb1e6..ca08e3bb 100644 --- a/pkg/uhttp/dbcache.go +++ b/pkg/uhttp/dbcache.go @@ -52,6 +52,7 @@ const ( errQueryingTable = "Error querying cache table" failRollback = "Failed to rollback transaction" failInsert = "Failed to insert data into cache table" + staticQuery = "UPDATE http_cache SET %s=(%s+1) WHERE key = ?" ) func NewDBCache(ctx context.Context, cfg CacheConfig) (*DBCache, error) { @@ -70,10 +71,18 @@ func NewDBCache(ctx context.Context, cfg CacheConfig) (*DBCache, error) { } // Create cache table and index - _, err = db.Exec(` + _, err = db.ExecContext(ctx, ` CREATE TABLE IF NOT EXISTS http_cache( - id INTEGER PRIMARY KEY, key NVARCHAR, data BLOB, expiration INTEGER, url NVARCHAR, - Hits INTEGER, Misses INTEGER, DelHits INTEGER, DelMisses INTEGER, Collisions INTEGER + id INTEGER PRIMARY KEY, + key NVARCHAR, + data BLOB, + expiration INTEGER, + url NVARCHAR, + hits INTEGER DEFAULT 0, + misses INTEGER DEFAULT 0, + delhits INTEGER DEFAULT 0, + delmisses INTEGER DEFAULT 0, + collisions INTEGER DEFAULT 0 ); CREATE UNIQUE INDEX IF NOT EXISTS idx_cache_key ON http_cache (key);`) if err != nil { @@ -132,7 +141,7 @@ func (d *DBCache) CreateCacheKey(req *http.Request) (string, error) { var headerParts []string for key, values := range req.Header { for _, value := range values { - if key == "Accept" || key == "Authorization" || key == "Cookie" || key == "Range" { + if key == "Accept" || key == "Content-Type" || key == "Cookie" || key == "Range" { headerParts = append(headerParts, fmt.Sprintf("%s=%s", key, value)) } } @@ -168,14 +177,25 @@ func (d *DBCache) Get(ctx context.Context, key string) (*http.Response, error) { return nil, err } + err = d.Hits(ctx, key) + if err != nil { + ctxzap.Extract(ctx).Debug("Failed to update hits", zap.Error(err)) + } + return resp, nil } + err = d.Misses(ctx, key) + if err != nil { + ctxzap.Extract(ctx).Debug("Failed to update misses", zap.Error(err)) + } + return nil, nil } // Set stores and save response in the db. func (d *DBCache) Set(ctx context.Context, key string, value *http.Response) error { + var url string if d.IsNilConnection() { return fmt.Errorf("%s", nilConnection) } @@ -185,7 +205,15 @@ func (d *DBCache) Set(ctx context.Context, key string, value *http.Response) err return err } - err = d.Insert(ctx, key, cacheableResponse) + if value.Request != nil { + url = getFullUrl(value.Request) + } + + err = d.Insert(ctx, + key, + cacheableResponse, + url, + ) if err != nil { return err } @@ -221,7 +249,7 @@ func (d *DBCache) Clear(ctx context.Context) error { } // Insert data into the cache table. -func (d *DBCache) Insert(ctx context.Context, key string, value any) error { +func (d *DBCache) Insert(ctx context.Context, key string, value any, url string) error { var ( bytes []byte err error @@ -247,10 +275,11 @@ func (d *DBCache) Insert(ctx context.Context, key string, value any) error { return err } - _, err = tx.Exec("INSERT INTO http_cache(key, data, expiration) values(?, ?, ?)", + _, err = tx.ExecContext(ctx, "INSERT INTO http_cache(key, data, expiration, url) values(?, ?, ?, ?)", key, bytes, time.Now().UnixNano(), + url, ) if err != nil { if errtx := tx.Rollback(); errtx != nil { @@ -309,7 +338,7 @@ func (d *DBCache) Select(ctx context.Context, key string) ([]byte, error) { } l := ctxzap.Extract(ctx) - rows, err := d.db.Query("SELECT data FROM http_cache where key = ?", key) + rows, err := d.db.QueryContext(ctx, "SELECT data FROM http_cache where key = ?", key) if err != nil { l.Debug(errQueryingTable, zap.Error(err)) return nil, err @@ -333,32 +362,30 @@ func (d *DBCache) Remove(ctx context.Context, key string) error { } l := ctxzap.Extract(ctx) - if ok, _ := d.Has(ctx, key); ok { - tx, err := d.db.Begin() - if err != nil { - l.Debug(failStartTransaction, zap.Error(err)) - return err - } - - _, err = d.db.Exec("DELETE FROM http_cache WHERE key = ?", key) - if err != nil { - if errtx := tx.Rollback(); errtx != nil { - l.Debug(failRollback, zap.Error(errtx)) - } + tx, err := d.db.Begin() + if err != nil { + l.Debug(failStartTransaction, zap.Error(err)) + return err + } - l.Debug("error deleting key", zap.Error(err)) - return err + _, err = d.db.ExecContext(ctx, "DELETE FROM http_cache WHERE key = ?", key) + if err != nil { + if errtx := tx.Rollback(); errtx != nil { + l.Debug(failRollback, zap.Error(errtx)) } - err = tx.Commit() - if err != nil { - if errtx := tx.Rollback(); errtx != nil { - l.Debug(failRollback, zap.Error(errtx)) - } + l.Debug("error deleting key", zap.Error(err)) + return err + } - l.Debug("Failed to remove cache value", zap.Error(err)) - return err + err = tx.Commit() + if err != nil { + if errtx := tx.Rollback(); errtx != nil { + l.Debug(failRollback, zap.Error(errtx)) } + + l.Debug("Failed to remove cache value", zap.Error(err)) + return err } return nil @@ -396,7 +423,7 @@ func (d *DBCache) DeleteExpired(ctx context.Context) error { } l := ctxzap.Extract(ctx) - rows, err := d.db.Query("SELECT key, expiration FROM http_cache") + rows, err := d.db.QueryContext(ctx, "SELECT key, expiration FROM http_cache") if err != nil { l.Debug(errQueryingTable, zap.Error(err)) return err @@ -433,3 +460,116 @@ func (d *DBCache) DeleteExpired(ctx context.Context) error { return nil } + +func getFullUrl(r *http.Request) string { + return fmt.Sprintf("%s://%s%s", r.URL.Scheme, r.Host, r.URL.Path) +} + +func (d *DBCache) Hits(ctx context.Context, key string) error { + if d.IsNilConnection() { + return fmt.Errorf("%s", nilConnection) + } + + strField := "hits" + err := d.Update(ctx, strField, key) + if err != nil { + return err + } + + return nil +} + +func (d *DBCache) DelHits(ctx context.Context, key string) error { + if d.IsNilConnection() { + return fmt.Errorf("%s", nilConnection) + } + + strField := "delhits" + err := d.Update(ctx, strField, key) + if err != nil { + return err + } + + return nil +} + +func (d *DBCache) Misses(ctx context.Context, key string) error { + if d.IsNilConnection() { + return fmt.Errorf("%s", nilConnection) + } + + strField := "misses" + err := d.Update(ctx, strField, key) + if err != nil { + return err + } + + return nil +} + +func (d *DBCache) DelMisses(ctx context.Context, key string) error { + if d.IsNilConnection() { + return fmt.Errorf("%s", nilConnection) + } + + strField := "delmisses" + err := d.Update(ctx, strField, key) + if err != nil { + return err + } + + return nil +} + +func (d *DBCache) Collisions(ctx context.Context, key string) error { + if d.IsNilConnection() { + return fmt.Errorf("%s", nilConnection) + } + + strField := "collisions" + err := d.Update(ctx, strField, key) + if err != nil { + return err + } + + return nil +} + +func (d *DBCache) Update(ctx context.Context, field, key string) error { + l := ctxzap.Extract(ctx) + tx, err := d.db.Begin() + if err != nil { + l.Debug(failStartTransaction, zap.Error(err)) + return err + } + + query, args := d.queryString(field) + _, err = d.db.ExecContext(ctx, fmt.Sprintf(query, args...), key) + if err != nil { + if errtx := tx.Rollback(); errtx != nil { + l.Debug(failRollback, zap.Error(errtx)) + } + + l.Debug("error updating "+field, zap.Error(err)) + return err + } + + err = tx.Commit() + if err != nil { + if errtx := tx.Rollback(); errtx != nil { + l.Debug(failRollback, zap.Error(errtx)) + } + + l.Debug("Failed to update "+field, zap.Error(err)) + return err + } + + return nil +} + +func (d *DBCache) queryString(field string) (string, []interface{}) { + return staticQuery, []interface{}{ + fmt.Sprint(field), + fmt.Sprint(field), + } +} diff --git a/pkg/uhttp/dbcache_test.go b/pkg/uhttp/dbcache_test.go index 9ebcaee2..e4426373 100644 --- a/pkg/uhttp/dbcache_test.go +++ b/pkg/uhttp/dbcache_test.go @@ -45,10 +45,10 @@ func TestDBCache(t *testing.T) { fc, err := getDBCacheForTesting() require.Nil(t, err) - err = fc.Insert(ctx, "url", urlTest) + err = fc.Insert(ctx, "urlTest", urlTest, "http://example.com") require.Nil(t, err) - res, err := fc.Select(ctx, "url") + res, err := fc.Select(ctx, "urlTest") require.Nil(t, err) require.NotNil(t, res)