Skip to content

Commit bba4df6

Browse files
committed
Add postgresql as backend
1 parent 1b059f8 commit bba4df6

File tree

13 files changed

+327
-23
lines changed

13 files changed

+327
-23
lines changed

README.md

+1
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,7 @@ Fields:
202202
## Todo
203203
- atomic local cache updates
204204
- export in hosts file format
205+
- improve scribble add (adding before stored in cache overwrites)
205206

206207

207208
## Changelog

api/README.md

+3-3
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,9 @@ Small, lightweight, api-driven dns server.
1212
| **POST** /records | Adds the domain and full record | json domain object | json domain object |
1313
| **PUT** /records | Update all domains and records (replaces all) | json array of domain objects | json array of domain objects |
1414
| **GET** /records | Returns a list of domains we have records for | nil | string array of domains |
15-
| **PUT** /records/{id} | Update domain's records (replaces all) | json domain object | json domain object |
16-
| **GET** /records/{id} | Returns the records for that domain | nil | json domain object |
17-
| **DELETE** /records/{id} | Delete a domain | nil | success message |
15+
| **PUT** /records/{domain} | Update domain's records (replaces all) | json domain object | json domain object |
16+
| **GET** /records/{domain} | Returns the records for that domain | nil | json domain object |
17+
| **DELETE** /records/{domain} | Delete a domain | nil | success message |
1818

1919
## Usage Example:
2020

api/records.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ func updateRecord(rw http.ResponseWriter, req *http.Request) {
6969
return
7070
}
7171

72-
// "MUST reply 201"(https://www.w3.org/Protocols/rfc2616/rfc2616-sec9.html)
72+
// "MUST reply 201" (https://www.w3.org/Protocols/rfc2616/rfc2616-sec9.html)
7373
writeBody(rw, req, resource, http.StatusCreated)
7474
return
7575
}

build.sh

+2-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,8 @@ MD5=$(which md5 || which md5sum)
2626

2727
# build shaman
2828
echo "Building SHAMAN and uploading it to 's3://tools.nanopack.io/shaman'"
29-
gox -ldflags="-X main.version=${tag} -X main.branch=${branch} -X main.commit=${commit}" -osarch "linux/amd64" -output="./build/{{.OS}}/{{.Arch}}/shaman"
29+
gox -ldflags="-X main.version=${tag} -X main.branch=${branch} -X main.commit=${commit}" \
30+
-osarch "linux/amd64" -output="./build/{{.OS}}/{{.Arch}}/shaman"
3031

3132
# look through each os/arch/file and generate an md5 for each
3233
echo "Generating md5s..."

cache/cache.go

+10-6
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,11 @@ var (
1818
// The cacher interface is what all the backends [will] implement
1919
type cacher interface {
2020
initialize() error
21-
addRecord(resource *shaman.Resource) error
21+
addRecord(resource shaman.Resource) error
2222
getRecord(domain string) (*shaman.Resource, error)
23-
updateRecord(domain string, resource *shaman.Resource) error
23+
updateRecord(domain string, resource shaman.Resource) error
2424
deleteRecord(domain string) error
25-
resetRecords(resources *[]shaman.Resource) error
25+
resetRecords(resources []shaman.Resource) error
2626
listRecords() ([]shaman.Resource, error)
2727
}
2828

@@ -36,6 +36,10 @@ func Initialize() error {
3636
switch u.Scheme {
3737
case "scribble":
3838
storage = &scribbleDb{}
39+
case "postgres":
40+
storage = &postgresDb{}
41+
case "postgresql":
42+
storage = &postgresDb{}
3943
case "none":
4044
storage = nil
4145
default:
@@ -60,7 +64,7 @@ func AddRecord(resource *shaman.Resource) error {
6064
return nil
6165
}
6266
resource.Validate()
63-
return storage.addRecord(resource)
67+
return storage.addRecord(*resource)
6468
}
6569

6670
// GetRecord gets a record to the persistent cache
@@ -80,7 +84,7 @@ func UpdateRecord(domain string, resource *shaman.Resource) error {
8084
}
8185
shaman.SanitizeDomain(&domain)
8286
resource.Validate()
83-
return storage.updateRecord(domain, resource)
87+
return storage.updateRecord(domain, *resource)
8488
}
8589

8690
// DeleteRecord removes a record from the persistent cache
@@ -101,7 +105,7 @@ func ResetRecords(resources *[]shaman.Resource) error {
101105
(*resources)[i].Validate()
102106
}
103107

104-
return storage.resetRecords(resources)
108+
return storage.resetRecords(*resources)
105109
}
106110

107111
// ListRecords lists all records in the persistent cache

cache/cache_test.go

+1
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ var (
1919

2020
func TestMain(m *testing.M) {
2121
// manually configure
22+
// config.Log = lumber.NewConsoleLogger(lumber.LvlInt("trace"))
2223
config.Log = lumber.NewConsoleLogger(lumber.LvlInt("FATAL"))
2324

2425
// run tests

cache/postgres.go

+204
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,204 @@
1+
package cache
2+
3+
import (
4+
"database/sql"
5+
"fmt"
6+
7+
_ "github.com/lib/pq"
8+
9+
"github.com/nanopack/shaman/config"
10+
shaman "github.com/nanopack/shaman/core/common"
11+
)
12+
13+
type postgresDb struct {
14+
pg *sql.DB
15+
}
16+
17+
func (p *postgresDb) connect() error {
18+
// todo: example: config.DatabaseConnection = "postgres://[email protected]?sslmode=disable"
19+
db, err := sql.Open("postgres", config.L2Connect)
20+
if err != nil {
21+
return fmt.Errorf("Failed to connect to postgres - %v", err)
22+
}
23+
err = db.Ping()
24+
if err != nil {
25+
return fmt.Errorf("Failed to ping postgres on connect - %v", err)
26+
}
27+
28+
p.pg = db
29+
return nil
30+
}
31+
32+
func (p postgresDb) createTables() error {
33+
// create records table
34+
_, err := p.pg.Exec(`
35+
CREATE TABLE IF NOT EXISTS records (
36+
recordId SERIAL PRIMARY KEY NOT NULL,
37+
domain TEXT NOT NULL,
38+
address TEXT NOT NULL,
39+
ttl INTEGER,
40+
class TEXT,
41+
type TEXT
42+
)`)
43+
if err != nil {
44+
return fmt.Errorf("Failed to create records table - %v", err)
45+
}
46+
47+
return nil
48+
}
49+
50+
func (p *postgresDb) initialize() error {
51+
err := p.connect()
52+
if err != nil {
53+
return fmt.Errorf("Failed to create new connection - %v", err)
54+
}
55+
56+
// create tables
57+
err = p.createTables()
58+
if err != nil {
59+
return fmt.Errorf("Failed to create tables - %v", err)
60+
}
61+
62+
return nil
63+
}
64+
65+
func (p postgresDb) addRecord(resource shaman.Resource) error {
66+
resources, err := p.listRecords()
67+
if err != nil {
68+
return err
69+
}
70+
71+
for i := range resources {
72+
if resources[i].Domain == resource.Domain {
73+
// if domains match, check address
74+
for k := range resources[i].Records {
75+
next:
76+
for j := range resource.Records {
77+
// check if the record exists...
78+
if resource.Records[j].RType == resources[i].Records[k].RType &&
79+
resource.Records[j].Address == resources[i].Records[k].Address &&
80+
resource.Records[j].Class == resources[i].Records[k].Class {
81+
// if so, skip
82+
config.Log.Trace("Record exists in persistent, skipping...")
83+
resource.Records = append(resource.Records[:i], resource.Records[i+1:]...)
84+
goto next
85+
}
86+
}
87+
}
88+
}
89+
}
90+
91+
// add records
92+
for i := range resource.Records {
93+
config.Log.Trace("Adding record to database...")
94+
_, err = p.pg.Exec(fmt.Sprintf(`
95+
INSERT INTO records(domain, address, ttl, class, type)
96+
VALUES('%v', '%v', '%v', '%v', '%v')`,
97+
resource.Domain, resource.Records[i].Address, resource.Records[i].TTL,
98+
resource.Records[i].Class, resource.Records[i].RType))
99+
if err != nil {
100+
return fmt.Errorf("Failed to insert into records table - %v", err)
101+
}
102+
}
103+
104+
return nil
105+
}
106+
107+
func (p postgresDb) getRecord(domain string) (*shaman.Resource, error) {
108+
// read from records table
109+
rows, err := p.pg.Query(fmt.Sprintf("SELECT address, ttl, class, type FROM records WHERE domain = '%v'", domain))
110+
if err != nil {
111+
return nil, fmt.Errorf("Failed to select from records table - %v", err)
112+
}
113+
defer rows.Close()
114+
115+
records := make([]shaman.Record, 0, 0)
116+
117+
// get data
118+
for rows.Next() {
119+
rcrd := shaman.Record{}
120+
err = rows.Scan(&rcrd.Address, &rcrd.TTL, &rcrd.Class, &rcrd.RType)
121+
if err != nil {
122+
return nil, fmt.Errorf("Failed to save results into record - %v", err)
123+
}
124+
125+
records = append(records, rcrd)
126+
}
127+
128+
// check for errors
129+
if err = rows.Err(); err != nil {
130+
return nil, fmt.Errorf("Error with results - %v", err)
131+
}
132+
133+
if len(records) == 0 {
134+
return nil, errNoRecordError
135+
}
136+
137+
return &shaman.Resource{Domain: domain, Records: records}, nil
138+
}
139+
140+
func (p postgresDb) updateRecord(domain string, resource shaman.Resource) error {
141+
// delete old from records
142+
err := p.deleteRecord(domain)
143+
if err != nil {
144+
return fmt.Errorf("Failed to clean old records - %v", err)
145+
}
146+
147+
return p.addRecord(resource)
148+
}
149+
150+
func (p postgresDb) deleteRecord(domain string) error {
151+
_, err := p.pg.Exec(fmt.Sprintf(`DELETE FROM records WHERE domain = '%v'`, domain))
152+
if err != nil {
153+
return fmt.Errorf("Failed to delete from records table - %v", err)
154+
}
155+
156+
return nil
157+
}
158+
159+
func (p postgresDb) resetRecords(resources []shaman.Resource) error {
160+
// truncate records table
161+
_, err := p.pg.Exec("TRUNCATE records")
162+
if err != nil {
163+
return fmt.Errorf("Failed to truncate records table - %v", err)
164+
}
165+
for i := range resources {
166+
err = p.addRecord(resources[i]) // prevents duplicates
167+
if err != nil {
168+
return fmt.Errorf("Failed to save records - %v", err)
169+
}
170+
}
171+
return nil
172+
}
173+
174+
func (p postgresDb) listRecords() ([]shaman.Resource, error) {
175+
// read from records table
176+
rows, err := p.pg.Query("SELECT DISTINCT domain FROM records")
177+
if err != nil {
178+
return nil, fmt.Errorf("Failed to select from records table - %v", err)
179+
}
180+
defer rows.Close()
181+
182+
resources := make([]shaman.Resource, 0)
183+
184+
// get data
185+
for rows.Next() {
186+
var domain string
187+
err = rows.Scan(&domain)
188+
if err != nil {
189+
return nil, fmt.Errorf("Failed to save domain - %v", err)
190+
}
191+
resource, err := p.getRecord(domain)
192+
if err != nil {
193+
return nil, fmt.Errorf("Failed to get record for domain - %v", err)
194+
}
195+
196+
resources = append(resources, *resource)
197+
}
198+
199+
// check for errors
200+
if err = rows.Err(); err != nil {
201+
return nil, fmt.Errorf("Error with results - %v", err)
202+
}
203+
return resources, nil
204+
}

cache/postgres_test.go

+93
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
package cache_test
2+
3+
import (
4+
"testing"
5+
6+
"github.com/nanopack/shaman/cache"
7+
"github.com/nanopack/shaman/config"
8+
shaman "github.com/nanopack/shaman/core/common"
9+
)
10+
11+
// test postgres cache init
12+
func TestPostgresInitialize(t *testing.T) {
13+
config.L2Connect = "postgres://[email protected]?sslmode=disable" // default
14+
err := cache.Initialize()
15+
config.L2Connect = "postgresql://[email protected]:9999?sslmode=disable" // unable to init?
16+
err2 := cache.Initialize()
17+
if err != nil || err2 != nil {
18+
t.Errorf("Failed to initalize postgres cacher - %v%v", err, err2)
19+
}
20+
}
21+
22+
// test postgres cache addRecord
23+
func TestPostgresAddRecord(t *testing.T) {
24+
postgresReset()
25+
err := cache.AddRecord(&nanopack)
26+
if err != nil {
27+
t.Errorf("Failed to add record to postgres cacher - %v", err)
28+
}
29+
30+
err = cache.AddRecord(&nanopack)
31+
if err != nil {
32+
t.Errorf("Failed to add record to postgres cacher - %v", err)
33+
}
34+
}
35+
36+
// test postgres cache getRecord
37+
func TestPostgresGetRecord(t *testing.T) {
38+
postgresReset()
39+
cache.AddRecord(&nanopack)
40+
_, err := cache.GetRecord("nanobox.io.")
41+
_, err2 := cache.GetRecord("nanopack.io")
42+
if err == nil || err2 != nil {
43+
t.Errorf("Failed to get record from postgres cacher - %v%v", err, err2)
44+
}
45+
}
46+
47+
// test postgres cache updateRecord
48+
func TestPostgresUpdateRecord(t *testing.T) {
49+
postgresReset()
50+
err := cache.UpdateRecord("nanobox.io", &nanopack)
51+
err2 := cache.UpdateRecord("nanopack.io", &nanopack)
52+
if err != nil || err2 != nil {
53+
t.Errorf("Failed to update record in postgres cacher - %v%v", err, err2)
54+
}
55+
}
56+
57+
// test postgres cache deleteRecord
58+
func TestPostgresDeleteRecord(t *testing.T) {
59+
postgresReset()
60+
err := cache.DeleteRecord("nanobox.io")
61+
cache.AddRecord(&nanopack)
62+
err2 := cache.DeleteRecord("nanopack.io")
63+
if err != nil || err2 != nil {
64+
t.Errorf("Failed to delete record from postgres cacher - %v%v", err, err2)
65+
}
66+
}
67+
68+
// test postgres cache resetRecords
69+
func TestPostgresResetRecords(t *testing.T) {
70+
postgresReset()
71+
err := cache.ResetRecords(&nanoBoth)
72+
if err != nil {
73+
t.Errorf("Failed to reset records in postgres cacher - %v", err)
74+
}
75+
}
76+
77+
// test postgres cache listRecords
78+
func TestPostgresListRecords(t *testing.T) {
79+
postgresReset()
80+
_, err := cache.ListRecords()
81+
cache.ResetRecords(&nanoBoth)
82+
_, err2 := cache.ListRecords()
83+
if err != nil || err2 != nil {
84+
t.Errorf("Failed to list records in postgres cacher - %v%v", err, err2)
85+
}
86+
}
87+
88+
func postgresReset() {
89+
config.L2Connect = "postgres://[email protected]?sslmode=disable"
90+
cache.Initialize()
91+
blank := make([]shaman.Resource, 0, 0)
92+
cache.ResetRecords(&blank)
93+
}

0 commit comments

Comments
 (0)