-
Notifications
You must be signed in to change notification settings - Fork 13
/
GNUmakefile
339 lines (285 loc) · 11.9 KB
/
GNUmakefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
PG_PREFAULTER ?= pg_prefaulter
GOCOVER_TMPFILE?= $(GOCOVER_FILE).tmp
GOCOVER_FILE?= .cover.out
GOCOVERHTML?= coverage.html
GOFMT_FILES?=$$(find . -name '*.go' |grep -v vendor)
FIND=`/usr/bin/which 2> /dev/null gfind find | /usr/bin/grep -v ^no | /usr/bin/head -n 1`
XARGS=`/usr/bin/which 2> /dev/null gxargs xargs | /usr/bin/grep -v ^no | /usr/bin/head -n 1`
default:: help
.PHONY: build
build:: $(PG_PREFAULTER) ## 10 Build pg_prefaulter binary
.PHONY: pg_prefaulter
pg_prefaulter::
go build -ldflags "-X main.commit=`git describe --tags --always` -X main.date=`date +%Y-%m-%d_%H:%d`" -o $@ main.go
.PHONY: test
test:: ## 10 Run go test
go test -v ./...
cover:: coverage_report ## 10 Generate a coverage report
$(GOCOVER_FILE)::
@${FIND} . -type d ! -path '*vendor*' ! -path '*.git*' ! -path '*pgdata*' -print0 | ${XARGS} -0 -P4 -I % sh -ec "cd % && rm -f $(GOCOVER_TMPFILE) && go test -coverprofile=$(GOCOVER_TMPFILE) || true"
@echo 'mode: set' > $(GOCOVER_FILE)
@${FIND} . -type f ! -path '*vendor*' ! -path '*.git*' ! -path '*pgdata*' -name "$(GOCOVER_TMPFILE)" -print0 | ${XARGS} -0 -n1 cat $(GOCOVER_TMPFILE) | grep -v '^mode: ' >> ${PWD}/$(GOCOVER_FILE)
$(GOCOVERHTML): $(GOCOVER_FILE)
go tool cover -html=$(GOCOVER_FILE) -o $(GOCOVERHTML)
coverage_report:: $(GOCOVER_FILE)
go tool cover -html=$(GOCOVER_FILE)
install_audit_tools:: ## 10 Install static analysis tools
@go get -u github.com/golang/lint/golint && echo "Installed golint"
@go get -u github.com/fzipp/gocyclo && echo "Installed gocyclo"
@go get -u github.com/remyoudompheng/go-misc/deadcode && echo "Installed deadcode"
@go get -u github.com/client9/misspell/cmd/misspell && echo "Installed misspell"
@go get -u github.com/gordonklaus/ineffassign && echo "Installed ineffassig:"
@go get -u golang.org/x/tools/cover && echo "Installed cover"
audit:: ## 10 Run static analysis tools over the code
deadcode
go tool vet -all *.go
go tool vet -shadow=true *.go
golint *.go
ineffassign .
gocyclo -over 65 *.go
misspell *.go
.PHONY: vet
vet:: ## 10 vet the binary (excluding dependencies)
@echo "go vet ."
@go vet $$(go list ./... | grep -v vendor/) ; if [ $$? -eq 1 ]; then \
echo ""; \
echo "Vet found suspicious constructs. Please check the reported constructs"; \
echo "and fix them if necessary before submitting the code for review."; \
exit 1; \
fi
.PHONY: fmt
fmt: ## 10 fmt and simplify the code
gofmt -s -w $(GOFMT_FILES)
.PHONY: vendor-status
vendor-status: ## 10 Display the vendor/ status
@dep status
.PHONY: release
release: ## 10 Build a release
#goreleaser --release-notes=release_notes.md
@goreleaser
.PHONY: release-snapshot
release-snapshot: ## 10 Build a snapshot release
@goreleaser --snapshot --skip-validate --rm-dist
### PostgreSQL-specific targets
PGVERSION?=96
POSTGRES?=$(wildcard /usr/local/bin/postgres /opt/local/lib/postgresql$(PGVERSION)/bin/postgres /opt/local/bin/postgres)
PSQL?=$(wildcard /usr/local/bin/psql /opt/local/lib/postgresql$(PGVERSION)/bin/psql /opt/local/bin/psql)
PG_BASEBACKUP?=$(wildcard /usr/local/bin/pg_basebackup /opt/local/lib/postgresql$(PGVERSION)/bin/pg_basebackup /opt/local/bin/pg_basebackup)
INITDB?=$(wildcard /usr/local/bin/initdb /opt/local/lib/postgresql$(PGVERSION)/bin/initdb /opt/local/bin/initdb)
PG_CONTROLDATA?=$(wildcard /usr/local/bin/pg_controldata /opt/local/lib/postgresql$(PGVERSION)/bin/pg_controldata /opt/local/bin/pg_controldata)
PWFILE?=.pwfile
PGBENCH?=$(wildcard /usr/local/bin/pgbench /opt/local/lib/postgresql$(PGVERSION)/bin/pgbench /opt/local/bin/pgbench)
PGBENCH_ARGS?=-j 64 -P 60 -r -T 900 --no-vacuum --protocol=prepared
PGBENCH_INIT_ARGS?=-i -s 10 -F 90
GOPATH?=$(shell go env GOPATH)
PGDATA_PRIMARY?=$(GOPATH)/src/github.com/joyent/pg_prefaulter/.pgdata_primary
PGDATA_FOLLOWER?=$(GOPATH)/src/github.com/joyent/pg_prefaulter/.pgdata_follower
PGPRIMARYPORT?=5442
PGFOLLOWPORT?=5452
.PHONY: check-pg_controldata
check-pg_controldata::
@if [ -z "$(PG_CONTROLDATA)" ]; then \
printf "pg_controldata(1) not found. Check PostgreSQL installation or set PG_CONTROLDATA=/path/to/pg_controldata"; \
exit 1; \
fi
.PHONY: check-initdb
check-initdb::
@if [ -z "$(INITDB)" ]; then \
printf "initdb(1) not found. Check PostgreSQL installation or set INITDB=/path/to/initdb"; \
exit 1; \
fi
.PHONY: check-pg_basebackup
check-pg_basebackup::
@if [ -z "$(PG_BASEBACKUP)" ]; then \
printf "pg_basebackup(1) not found. Check PostgreSQL installation or set PG_BASEBACKUP=/path/to/pg_basebackup"; \
exit 1; \
fi
.PHONY: check-pgbench
check-pgbench::
@if [ -z "$(PGBENCH)" ]; then \
printf "pgbench(1) not found. Check PostgreSQL installation or set PGBENCH=/path/to/pgbench"; \
exit 1; \
fi
.PHONY: check-psql
check-psql::
@if [ -z "$(PSQL)" ]; then \
printf "psql(1) not found. Check PostgreSQL installation or set PSQL=/path/to/psql"; \
exit 1; \
fi
.PHONY: check-postgres
check-postgres::
@if [ -z "$(POSTGRES)" ]; then \
printf "postgres(1) not found. Check PostgreSQL installation or set POSTGRES=/path/to/postgres"; \
exit 1; \
fi
$(PWFILE):
-cat /dev/urandom | strings | grep -o '[[:alnum:]]' | head -n 32 | tr -d '\n' > $@
.PHONY: freshdb-primary
freshdb-primary:: cleandb-primary initdb-primary startdb-primary ## 30 Drops and recreates the primary database
.PHONY: initdb-primary
initdb-primary:: $(PWFILE) check-initdb ## 30 initdb(1) a primary database
$(INITDB) --no-locale -U postgres -A md5 --pwfile="$(PWFILE)" -D "$(PGDATA_PRIMARY)"
mkdir -p $(PGDATA_PRIMARY) $(PGDATA_FOLLOWER) || true
echo "local replication postgres md5" >> $(PGDATA_PRIMARY)/pg_hba.conf
echo "host replication postgres 127.0.0.1/32 md5" >> $(PGDATA_PRIMARY)/pg_hba.conf
echo "host replication postgres ::1/128 md5" >> $(PGDATA_PRIMARY)/pg_hba.conf
.PHONY: initdb-follower
initdb-follower:: $(PWFILE) check-pg_basebackup ## 40 initdb(1) a follower database
env PGPASSWORD="`cat \"$(PWFILE)\"`" $(PG_BASEBACKUP) -R -h localhost -p $(PGPRIMARYPORT) -D $(PGDATA_FOLLOWER) -P -U postgres -X stream
mkdir -p $(PGDATA_FOLLOWER)/archive || true
.PHONY: startdb-primary
startdb-primary:: check-postgres ## 30 Start the primary database
2>&1 \
exec $(POSTGRES) \
-D "$(PGDATA_PRIMARY)" \
-p "$(PGPRIMARYPORT)" \
-c log_connections=off \
-c log_disconnections=off \
-c log_duration=off \
-c log_statement=ddl \
-c wal_level=hot_standby \
-c wal_log_hints=on \
-c full_page_writes=on \
-c archive_mode=on \
-c max_wal_senders=5 \
-c wal_keep_segments=50 \
-c hot_standby=on \
-c archive_command="exit 0" \
-c synchronous_commit=off \
-c fsync=off \
| tee postgresql-primary.log
.PHONY: startdb-primary-debug
startdb-primary-debug:: check-postgres ## 30 Start the primary database with debug-level logging
2>&1 \
exec $(POSTGRES) \
-D "$(PGDATA_PRIMARY)" \
-c log_connections=on \
-c log_disconnections=on \
-c log_duration=on \
-c log_statement=all \
-c wal_level=hot_standby \
-c wal_log_hints=on \
-c full_page_writes=on \
-c archive_mode=on \
-c max_wal_senders=5 \
-c wal_keep_segments=50 \
-c hot_standby=on \
-c archive_command="exit 0" \
-c synchronous_commit=off \
-c fsync=off \
| tee postgresql-primary.log
# Note, the follower config is deliberately slower than the primary
.PHONY: startdb-follower
startdb-follower:: check-postgres ## 40 Start the follower database
2>&1 \
exec nice -n 20 \
$(POSTGRES) \
-D "$(PGDATA_FOLLOWER)" \
-p "$(PGFOLLOWPORT)" \
-c log_connections=on \
-c log_disconnections=on \
-c log_duration=off \
-c log_statement=ddl \
-c wal_level=hot_standby \
-c wal_log_hints=on \
-c full_page_writes=on \
-c archive_mode=on \
-c max_wal_senders=5 \
-c wal_keep_segments=50 \
-c hot_standby=on \
-c archive_command="exit 0" \
-c synchronous_commit=on \
-c fsync=on \
| tee postgresql-follower.log
# Note, the follower config is deliberately slower than the primary
.PHONY: startdb-follower-debug
startdb-follower-debug:: check-postgres ## 40 Start the follower database with debug-level logging
2>&1 \
exec nice -n 20 \
$(POSTGRES) \
-D "$(PGDATA_FOLLOWER)" \
-p "$(PGFOLLOWPORT)" \
-c log_connections=on \
-c log_disconnections=on \
-c log_duration=on \
-c log_statement=all \
-c wal_level=hot_standby \
-c wal_log_hints=on \
-c full_page_writes=on \
-c archive_mode=on \
-c max_wal_senders=5 \
-c wal_keep_segments=50 \
-c hot_standby=on \
-c archive_command="exit 0" \
-c synchronous_commit=on \
-c fsync=on \
| tee postgresql-follower.log
.PHONY: clean
clean:: cleandb-shard ## 90 Clean target
find . -name "$(GOCOVER_FILE).tmp" -o -name "$(GOCOVER_FILE)" -delete
.PHONY: cleandb-shard
cleandb-shard:: cleandb-primary cleandb-follower ## 90 Clean entire shard
rm -f "$(PWFILE)"
.PHONY: cleandb-primary
cleandb-primary:: ## 30 Clean primary database
rm -rf "$(PGDATA_PRIMARY)"
.PHONY: cleandb-follower
cleandb-follower:: ## 40 Clean follower database
rm -rf "$(PGDATA_FOLLOWER)"
.PHONY: freshdb-follower
freshdb-follower:: cleandb-follower initdb-follower startdb-follower ## 40 Drops and recreates the follower database
.PHONY: testdb
testdb:: test resetdb ## 50 Run database tests
.PHONY: pgbench-init
pgbench-init:: check-pgbench ## 60 Initialize pgbench
2>&1 env PGSSLMODE=disable PGHOST=/tmp PGUSER=postgres PGPASSWORD="`cat \"$(PWFILE)\"`" "$(PGBENCH)" -i $(PGBENCH_INIT_ARGS)
.PHONY: pgbench
pgbench:: check-pgbench ## 60 Run pgbench(1)
2>&1 env PGSSLMODE=disable PGHOST=/tmp PGUSER=postgres PGPASSWORD="`cat \"$(PWFILE)\"`" "$(PGBENCH)" $(PGBENCH_ARGS)
.PHONY: resetdb
resetdb:: check-psql dropdb createdb gendata ## 50 Drop and recreate the database
2>&1 PGSSLMODE=disable PGHOST=/tmp PGUSER=postgres PGPASSWORD="`cat \"$(PWFILE)\"`" "$(PSQL)" postgres -c 'DROP DATABASE test' | tee -a test.log
.PHONY: dropdb
dropdb:: check-psql ## 50 Reset the test database
2>&1 PGSSLMODE=disable PGHOST=/tmp PGUSER=postgres PGPASSWORD="`cat \"$(PWFILE)\"`" "$(PSQL)" postgres -c 'DROP DATABASE test' | tee -a test.log
.PHONY: createdb
createdb: check-psql ## 50 Create the test database
2>&1 PGSSLMODE=disable PGHOST=/tmp PGUSER=postgres PGPASSWORD="`cat \"$(PWFILE)\"`" "$(PSQL)" postgres -c 'CREATE DATABASE test' | tee -a test.log
2>&1 PGSSLMODE=disable PGHOST=/tmp PGUSER=postgres PGPASSWORD="`cat \"$(PWFILE)\"`" "$(PSQL)" test -c 'CREATE TABLE garbage (s INT, md5 TEXT)' | tee -a test.log
.PHONY: controldata
controldata:: check-pg_controldata ## 70 Display pg_controldata(1) of the primary
$(PG_CONTROLDATA) -D "$(PGDATA_PRIMARY)"
.PHONY: gendata
gendata:: check-psql ## 50 Generate data in the primary
2>&1 PGSSLMODE=disable PGHOST=/tmp PGUSER=postgres PGPASSWORD="`cat \"$(PWFILE)\"`" PGOPTIONS="-c synchronous_commit=off" "$(PSQL)" test -c 'INSERT INTO garbage SELECT s, md5(random()::text) FROM generate_series(1,1000000) s' | tee -a test.log
.PHONY: psql
psql:: psql-primary ## 70 Open a psql(1) shell to the primary
.PHONY: psql-both
psql-both:: ## 70 Send a psql(1) command to both using -c
@if [ -z "$(PSQL_ARGS)" ]; then \
printf "PSQL_ARGS not set."; \
exit 1; \
fi
@$(MAKE) psql-primary PSQL_ARGS="$(PSQL_ARGS)"
@$(MAKE) psql-follower PSQL_ARGS="$(PSQL_ARGS)"
.PHONY: psql-primary
psql-primary:: check-psql ## 30 Open a psql(1) shell to the primary
@exec env PGPASSWORD="`cat \"$(PWFILE)\"`" "$(PSQL)" -p $(PGPRIMARYPORT) -E postgres postgres $(PSQL_ARGS)
.PHONY: psql-follower
psql-follower:: check-psql ## 40 Open a psql(1) shell to the follower
@exec env PGPASSWORD="`cat \"$(PWFILE)\"`" "$(PSQL)" -p $(PGFOLLOWPORT) -E postgres postgres $(PSQL_ARGS)
.PHONY: help
help:: ## 99 This help message
@echo "pg_prefaulter make(1) targets:"
@grep -E '^[a-zA-Z\_\-]+:[:]?.*?## [0-9]+ .*$$' $(MAKEFILE_LIST) | \
sort -n -t '#' -k3,1 | awk ' \
BEGIN { FS = ":[:]?.*?## "; section = 10; }; \
{ \
newSect = int($$2); \
if (section != newSect) { \
section = newSect; \
printf "\n"; \
} \
sub("^[0-9]+", "", $$2); \
printf "\033[36m%-15s\033[0m %s\n", $$1, $$2; \
}'