-
Notifications
You must be signed in to change notification settings - Fork 42
/
properties.ex
394 lines (322 loc) · 12 KB
/
properties.ex
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
defmodule PropCheck.Properties do
@moduledoc """
This module defines the `property/4` and `property/1` macros. It is automatically available
by `use PropCheck`.
"""
alias PropCheck.CounterStrike
require Logger
@doc """
Defines a property as part of an ExUnit test.
The property macro takes at minimum a name and a `do`-block containing
the code of the property to be tested. The property code is encapsulated
as an `ExUnit` test case of category `property`, which is released as
part of Elixir 1.3 and allows a nice mix of regular unit test and property
based testing. This is the reason for the third parameter taking an
environment of variables defined in a test setup function. In `ExUnit`, this
is referred to as a test's "context".
The second parameter sets options for Proper (see `PropCheck` ). The default
is `:quiet` such that execution during ExUnit runs are silent, as normal
unit tests are. You can change it e.g. to `:verbose` or setting the
maximum size of the test data generated or what ever may be helpful. For
seeing the result of wrapper functions `PropCheck.aggregate/2` etc., the
verbose mode is required.
## Counter Examples
If a property fails, the counter example is in a file. The next time this
property is checked again, only the counter example is used to ensure that
the property now behaves correctly. Additionally, a property with an existing
counter example is embellished with the tag `failing_prop`. You can skip all
other tests and property by running `mix test --only failing_prop`. In this case
only the properties with counter example are run. Another option is to use
the `--stale` option of `ExUnit` to reduce the amount of tests and properties
while fixing the code tested by a property.
After a property was ran successfully against a previous counter example, PropCheck will
run the property again to check if other counter examples can be found.
### Disable Storing Counter Examples
Storing counter examples can be disabled using the `:store_counter_example` tag. This
can be done in three different scopes: module-wide scope, describe-wide scope or for
a single property.
**NOTE** that this facility is meant for properties which cannot run with a value generated
in a previous test run. This should usually not be the case, and `:store_counter_example`
should only be used after careful consideration.
Disable for all properties in a module:
```
defmodule Test do
# ...
@moduletag store_counter_example: false
#...
end
```
Disable for all properties in a describe block:
```
defmodule Test do
# ...
describe "describe block" do
@describetag store_counter_example: false
# ...
end
end
```
Disable for a single property:
```
@tag store_counter_example: false
property "a property" do
# ...
end
```
"""
defmacro property(name, opts \\ [], var \\ quote(do: _), do: p_block) do
block =
quote do
unquote(p_block)
end
var = Macro.escape(var)
block = Macro.escape(block, unquote: true)
quote bind_quoted: [name: name, block: block, var: var, opts: opts] do
ExUnit.plural_rule("property", "properties")
%{module: module} = __ENV__
module_default_opts = Module.get_attribute(module, :propcheck_default_opts) || [:quiet]
# Get the attributes and allow using the Keyword module by filtering for tuple entries in the tags
moduletag =
Module.get_attribute(module, :moduletag) |> List.flatten() |> Enum.filter(&is_tuple/1)
describetag =
Module.get_attribute(module, :describetag) |> List.flatten() |> Enum.filter(&is_tuple/1)
tag = Module.get_attribute(module, :tag) |> List.flatten() |> Enum.filter(&is_tuple/1)
# intended precedence: tag > describetag > moduletag
store_counter_example =
moduletag
|> Keyword.merge(describetag)
|> Keyword.merge(tag)
|> Keyword.get(:store_counter_example, true)
# @tag failing_prop: tag_property({module, prop_name, []})
tags = [[failing_prop: tag_property({module, name, []})]]
prop_name =
ExUnit.Case.register_test(
__ENV__.module,
__ENV__.file,
__ENV__.line,
:property,
name,
tags
)
def unquote(prop_name)(unquote(var)) do
{:ok, output_agent} = PropCheck.OutputAgent.start_link()
opts = [{:output_agent, output_agent} | unquote(opts)]
merged_opts =
opts
|> PropCheck.Properties.merge_opts(unquote(module_default_opts))
|> PropCheck.Utils.merge_global_opts()
|> PropCheck.Utils.put_opts()
p = unquote(block)
mfa = {unquote(module), unquote(prop_name), []}
execute_property(p, mfa, merged_opts, unquote(store_counter_example))
:ok
end
end
end
@doc false
def merge_opts(opts, module_default_opts) do
module_default_opts =
case is_function(module_default_opts) do
true -> module_default_opts.()
false -> module_default_opts
end
case {is_list(opts), is_list(module_default_opts)} do
{true, true} -> opts ++ module_default_opts
{true, false} -> opts ++ [module_default_opts]
{false, true} -> [opts | module_default_opts]
{false, false} -> [opts, module_default_opts]
end
end
@doc false
# Returns the `failing_prop` tag value for the property. The `property_`
# prefix is added to the function name. The value is determined by
# looking up the `counter_example` in `CounterStrike` for the property.
@spec tag_property(mfa) :: boolean
def tag_property({m, f, a}) do
mfa = {m, String.to_atom("property_#{f}"), a}
case CounterStrike.counter_example(mfa) do
{:ok, _} ->
# Logger.debug "Found failing property #{inspect mfa}"
true
_ ->
false
end
end
@doc false
# Executes the body `p` of property `name` with PropEr options `opts`
# by ExUnit.
def execute_property(p, name, opts, store_counter_example?) do
should_fail = is_tuple(p) and elem(p, 0) == :fails
# Logger.debug "Execute property #{inspect name} "
proper_opts = PropCheck.Utils.to_proper_opts(opts)
case CounterStrike.counter_example(name) do
:none ->
PropCheck.quickcheck(p, [:long_result] ++ proper_opts)
:others ->
# since the tag is set, we execute everything. You can limit
# the amount of checks by using either --stale or --only failing_prop
qc(p, proper_opts)
{:ok, counter_example} ->
# Logger.debug "Found counter example #{inspect counter_example}"
result = PropCheck.check(p, counter_example, [:long_result] ++ proper_opts)
case result do
true -> qc(p, proper_opts)
false -> {:rerun_failed, counter_example}
e = {:error, _} -> e
end
end
|> handle_check_results(%{
name: name,
opts: opts,
should_fail: should_fail,
store_counter_example?: store_counter_example?
})
end
defp qc(p, opts), do: PropCheck.quickcheck(p, [:long_result] ++ opts)
# Handles the result of executing quick check or a re-check of a counter example.
# In this method a new found counter example is added to `CounterStrike`. Note that
# some macros such as exists/2 do not return counter examples when they fail.
defp handle_check_results(true, %{should_fail: false}) do
true
end
defp handle_check_results(true, args = %{should_fail: true}) do
raise ExUnit.AssertionError,
message:
"Property #{mfa_to_string(args.name)} should fail, but succeeded for all test data :-(",
expr: nil
end
defp handle_check_results(error = {:error, _}, args = %{}) do
raise ExUnit.AssertionError,
message: "Property #{mfa_to_string(args.name)} failed with an error: #{inspect(error)}",
expr: nil
end
defp handle_check_results(counter_example, %{should_fail: true})
when is_list(counter_example) do
true
end
defp handle_check_results(counter_example, args = %{}) when is_list(counter_example) do
counter_example_message =
if args.store_counter_example? do
CounterStrike.add_counter_example(args.name, counter_example)
"Counter example stored."
else
"Counter example NOT stored, :store_counter_example is set to false."
end
raise ExUnit.AssertionError,
message:
"""
Property #{mfa_to_string(args.name)} failed. Counter-Example is:
#{counter_example_inspect(counter_example)}
#{counter_example_message}
"""
|> add_additional_output(args.opts),
expr: nil
end
defp handle_check_results({:rerun_failed, counter_example}, args = %{})
when is_list(counter_example) do
CounterStrike.add_counter_example(args.name, counter_example)
raise ExUnit.AssertionError,
message:
"""
Property #{mfa_to_string(args.name)} failed. Counter-Example is:
#{counter_example_inspect(counter_example)}
Consider running `MIX_ENV=test mix propcheck.clean` if a bug in a generator was
identified and fixed. PropCheck cannot identify changes to generators. See
https://github.com/alfert/propcheck/issues/30 for more details.
"""
|> add_additional_output(args.opts),
expr: nil
end
defp handle_check_results(_, args) do
raise ExUnit.AssertionError,
message: """
Property #{mfa_to_string(args.name)} failed. There is no counter-example available.
"""
end
defp counter_example_inspect(counter_example) do
alias PropCheck.StateM.Reporter
case is_statem_commands(counter_example) do
:data ->
inspect(counter_example, pretty: true)
:commands ->
[cmds] = counter_example
for cmd <- cmds, do: Reporter.pretty_print_counter_example_cmd(cmd)
:parallel_commands ->
[par_cmd] = counter_example
Reporter.pretty_print_counter_example_parallel(par_cmd)
end
end
defp is_statem_commands([[]]) do
:data
end
defp is_statem_commands([counter_example]) when is_list(counter_example) do
is_command_list =
counter_example
|> Enum.all?(fn term ->
match?({:init, _}, term) or
match?(
{:set, {:var, _}, {:call, mod, fun, args}}
when is_atom(mod) and is_atom(fun) and is_list(args),
term
)
end)
case is_command_list do
true -> :commands
false -> :data
end
end
defp is_statem_commands([{seq_commands, _par_commands}]) do
case is_statem_commands([seq_commands]) do
:commands -> :parallel_commands
:data -> :parallel_commands
_ -> :data
end
end
defp is_statem_commands(_), do: :data
# Add additional output to a message
defp add_additional_output(message, opts) do
{:ok, additional_output} =
opts |> PropCheck.Utils.output_agent() |> PropCheck.OutputAgent.close()
if additional_output != "" do
"""
#{message}
#{additional_output}
"""
else
message
end
end
defp mfa_to_string({m, f, _}) do
"#{m}.#{f}()"
end
@doc false
def print_mod_as_erlang(mod) when is_atom(mod) do
{_m, beam, _file} = :code.get_object_code(mod)
{:ok, {_, [{:abstract_code, {_, ac}}]}} = :beam_lib.chunks(beam, [:abstract_code])
ac |> Enum.map(&:erl_pp.form/1) |> List.flatten() |> IO.puts()
end
@doc """
Defines a not yet implemented property.
This convenient macro provides a property which will always flunk. It resembles
the `test/1` macro from ExUnit. Similarly to `ExUnit`, it also tags the test with
`:not_implemented`, allowing to filter it when running `mix test`.
## Example
property "This property will be implemented in the future"
"""
defmacro property(message) do
quote bind_quoted: [message: message] do
prop_name =
ExUnit.Case.register_test(
__ENV__.module,
__ENV__.file,
__ENV__.line,
:property,
message,
[:not_implemented]
)
def unquote(prop_name)(_) do
flunk("Not implemented")
end
end
end
end