Generative Testing in Elixir

Transcript

1. Don’t write tests;
   Generate them
   Chris Keathley / @ChrisKeathley / [email protected]
   

   View Slide

2. What are we gunna
   talk about?
   

   View Slide

3. Testing today
   What are property tests
   Basic Example
   Real world(ish) example
   

   View Slide

4. Why Elixir?
   

   View Slide

5. View Slide

6. Elixir is a functional, dynamic
   language that targets the Erlang VM
   (BEAM)
   

   View Slide

7. Testing
   

   View Slide

8. TDD
   

   View Slide

9. 1. Write a failing test
   2. Write enough code to make that test pass
   3. Refactor
   Test Driven Development
   

   View Slide

10. Validation
    Protection From Regression
    Design
    Test Driven Development
    

    View Slide

11. Career
    Happiness
    TDD
    quiet contemplation
    Trough of disillusionment
    Property Tests!!!
    

    View Slide

12. Tests provide guard rails
    

    View Slide

13. Tests directly couple your implementation
    Test Api
    v1
    

    View Slide

14. Tests directly couple your implementation
    Test Api
    v1.3
    

    View Slide

15. Tests directly couple your implementation
    Test Api
    v1.3
    

    View Slide

16. Write as few tests
    as possible
    

    View Slide

17. Warning: Contrived
    Strawman Argument
    incoming!
    

    View Slide

18. Lets TDD Addition!
    x + y = ?
    

    View Slide

19. test "adding 2 numbers" do
    end
    

    View Slide

20. test "adding 2 numbers" do
    assert add(1, 1) == 2
    assert add(0, 2) == 2
    end
    

    View Slide

21. test "adding 2 numbers" do
    assert add(1, 1) == 2
    assert add(0, 2) == 2
    end
    def add(x, y) do
    end
    

    View Slide

22. View Slide

23. test "adding 2 numbers" do
    assert add(1, 1) == 2
    assert add(0, 2) == 2
    end
    def add(x, y) do
    2
    end
    

    View Slide

24. test "adding 2 numbers" do
    assert add(1, 1) == 2
    assert add(0, 2) == 2
    end
    def add(_x, _y) do
    2
    end
    

    View Slide

25. View Slide

26. But wait…
    

    View Slide

27. View Slide

28. test "adding 2 numbers" do
    assert add(1, 1) == 2
    assert add(0, 2) == 2
    assert add(3, 4) == 7
    end
    def add(_x, _y) do
    2
    end
    

    View Slide

29. Pattern Matching
    

    View Slide

30. Pattern Matching
    

    View Slide

31. Pattern Matching
    x = 3
    

    View Slide

32. Pattern Matching
    x = 3
    y = x
    

    View Slide

33. Pattern Matching
    x = 3
    y = x
    3 = y
    

    View Slide

34. Pattern Matching is
    an assertion
    

    View Slide

35. Pattern Matching
    x = 3
    3 = x
    

    View Slide

36. Pattern Matching
    3 = 3
    

    View Slide

37. Pattern Matching
    %{name: "Chris", hobbies: ["Coffee", "Pinball", "Lego"]}
    

    View Slide

38. Pattern Matching
    %{name: user_name} = %{name: "Chris", hobbies: ["Coffee", "Pinball", "Lego"]}
    

    View Slide

39. Pattern Matching
    user_name = "Chris"
    

    View Slide

40. Pattern Matching
    def user_name(user_map) do
    %{name: name} = user_map
    name
    end
    

    View Slide

41. Pattern Matching
    def user_name(%{name: name}) do
    name
    end
    

    View Slide

42. Pattern Matching
    def user_name(%{name: name}) do
    name
    end
    def user_name(_), do: "Default User"
    

    View Slide

43. test "adding 2 numbers" do
    assert add(1, 1) == 2
    assert add(0, 2) == 2
    assert add(3, 4) == 7
    end
    def add(_x, _y) do
    2
    end
    

    View Slide

44. test "adding 2 numbers" do
    assert add(1, 1) == 2
    assert add(0, 2) == 2
    assert add(3, 4) == 7
    end
    def add(_x, _y), do: 2
    

    View Slide

45. test "adding 2 numbers" do
    assert add(1, 1) == 2
    assert add(0, 2) == 2
    assert add(3, 4) == 7
    end
    def add(3, _), do: 7
    def add(_x, _y), do: 2
    

    View Slide

46. View Slide

47. But wait…
    

    View Slide

48. View Slide

49. test "adding 2 numbers" do
    assert add(1, 1) == 2
    assert add(0, 2) == 2
    assert add(3, 4) == 7
    end
    def add(3, _), do: 7
    def add(_x, _y), do: 2
    

    View Slide

50. test "adding 2 numbers" do
    assert add(1, 1) == 2
    assert add(0, 2) == 2
    assert add(3, 4) == 7
    assert add(-1, 4) == 3
    end
    def add(3, _), do: 7
    def add(_x, _y), do: 2
    

    View Slide

51. View Slide

52. test "adding 2 numbers" do
    assert add(1, 1) == 2
    assert add(0, 2) == 2
    assert add(3, 4) == 7
    assert add(-1, 4) == 3
    end
    def add(3, _), do: 7
    def add(_x, _y), do: 2
    

    View Slide

53. Guard clauses
    

    View Slide

54. Guard clauses
    def user_name(%{name: name}) do
    name
    end
    

    View Slide

55. Guard clauses
    def user_name(%{name: name}) when is_binary(name) do
    name
    end
    

    View Slide

56. Guard clauses
    def user_name(%{name: name}) when is_binary(name) do
    name
    end
    def user_name(%{name: name, age: age}) when age < 20
    

    View Slide

57. test "adding 2 numbers" do
    assert add(1, 1) == 2
    assert add(0, 2) == 2
    assert add(3, 4) == 7
    assert add(-1, 4) == 3
    end
    def add(3, _), do: 7
    def add(_x, _y), do: 2
    

    View Slide

58. test "adding 2 numbers" do
    assert add(1, 1) == 2
    assert add(0, 2) == 2
    assert add(3, 4) == 7
    assert add(-1, 4) == 3
    end
    def add(x, _) when x < 0, do: 3
    def add(3, _), do: 7
    def add(_x, _y), do: 2
    

    View Slide

59. View Slide

60. View Slide

61. View Slide

62. View Slide

63. What have we gained?
    

    View Slide

64. Validation
    Protection From Regression
    Design
    TDD
    

    View Slide

65. Validation
    Protection From Regression
    Design
    TDD
    

    View Slide

66. Validation
    Protection From Regression
    Design
    TDD
    

    View Slide

67. Validation
    Protection From Regression
    Design ?
    TDD
    

    View Slide

68. There are bugs in your code
    

    View Slide

69. Writing tests for one feature
    

    View Slide

70. Writing tests for one feature
    o(n)
    

    View Slide

71. Writing tests for two features
    o(n)
    o(n^2)
    

    View Slide

72. Writing tests for three features
    o(n)
    o(n^2)
    o(n^3)
    

    View Slide

73. Race conditions?
    

    View Slide

74. Property tests
    

    View Slide

75. View Slide

76. Expected == Actual
    

    View Slide

77. Expected == Actual
    Overly Specific
    

    View Slide

78. View Slide

79. Property Tests
    

    View Slide

80. Property Tests
    Int
    

    View Slide

81. Property Tests
    Int
    p(x)
    

    View Slide

82. Property Tests
    Int
    p(x) ?
    

    View Slide

83. Property Tests
    Int
    p(x) ?
    Invariant
    

    View Slide

84. Invariant: “a function, quantity,
    or property that remains
    unchanged when a specified
    transformation is applied.”
    

    View Slide

85. Basic Property
    Tests
    

    View Slide

86. What is true about addition?
    

    View Slide

87. x + 0 == x
    

    View Slide

88. test "addition with zero returns the same number" do
    end
    def add(_x, _y) do
    end
    

    View Slide

89. test "addition with zero returns the same number" do
    ptest do
    end
    end
    def add(_x, _y) do
    end
    

    View Slide

90. test "addition with zero returns the same number" do
    ptest x: int() do
    end
    end
    def add(_x, _y) do
    end
    

    View Slide

91. test "addition with zero returns the same number" do
    ptest x: int() do
    assert add(x, 0) == x
    end
    end
    def add(_x, _y) do
    end
    

    View Slide

92. View Slide

93. test "addition with zero returns the same number" do
    ptest x: int() do
    assert add(x, 0) == x
    end
    end
    def add(_x, _y) do
    end
    

    View Slide

94. test "addition with zero returns the same number" do
    ptest x: int() do
    assert add(x, 0) == x
    end
    end
    def add(x, _y) do
    x
    end
    

    View Slide

95. x + y == y + x
    

    View Slide

96. test "addition is commutative" do
    end
    def add(x, _y) do
    x
    end
    

    View Slide

97. test "addition is commutative" do
    ptest x: int(), y: int() do
    end
    end
    def add(x, _y) do
    x
    end
    

    View Slide

98. test "addition is commutative" do
    ptest x: int(), y: int() do
    assert add(x, y) == add(y, x)
    end
    end
    def add(x, _y) do
    x
    end
    

    View Slide

99. View Slide

100. test "addition is commutative" do
     ptest x: int(), y: int() do
     assert add(x, y) == add(y, x)
     end
     end
     def add(x, _y) do
     x
     end
     

     View Slide

101. test "addition is commutative" do
     ptest x: int(), y: int() do
     assert add(x, y) == add(y, x)
     end
     end
     def add(x, y) do
     x * y
     end
     

     View Slide

102. View Slide

103. test "addition is commutative" do
     ptest x: int(), y: int() do
     assert add(x, y) == add(y, x)
     end
     end
     def add(x, y) do
     x * y
     end
     

     View Slide

104. test "addition is commutative" do
     ptest x: int(), y: int() do
     assert add(x, y) == add(y, x)
     end
     end
     def add(x, y) do
     x * y
     end
     def add(x, 0), do: x
     

     View Slide

105. View Slide

106. (1 + x) + y == x + (1 + y)
     

     View Slide

107. def add(x, y) do
     x * y
     end
     def add(x, 0), do: x
     test "addition is asociative" do
     end
     

     View Slide

108. def add(x, y) do
     x * y
     end
     def add(x, 0), do: x
     test "addition is asociative" do
     ptest x: int(), y: int(), z: int() do
     end
     end
     

     View Slide

109. def add(x, y) do
     x * y
     end
     def add(x, 0), do: x
     test "addition is asociative" do
     ptest x: int(), y: int(), z: int() do
     assert add(x, add(y, z)) == add(add(x, y), z)
     end
     end
     

     View Slide

110. def add(x, y) do
     x * y
     end
     def add(x, 0), do: x
     test "addition is asociative" do
     ptest x: int(), y: int(), z: int() do
     assert add(x, add(y, z)) == add(add(x, y), z)
     end
     end
     

     View Slide

111. View Slide

112. def add(x, 0), do: x
     def add(x, y) do
     x * y
     end
     

     View Slide

113. def add(x, y) do
     x + y
     end
     

     View Slide

114. A Real-ish
     Example
     

     View Slide

115. View Slide

116. Modeling the application
     ?
     p(x) ?
     

     View Slide

117. Modeling the application
     ?
     p(x) ?
     What is the input domain?
     

     View Slide

118. Vote Vote Vote
     User
     

     View Slide

119. Vote Vote Vote
     User
     

     View Slide

120. Vote Vote Vote
     User
     

     View Slide

121. Vote Vote Vote
     User
     

     View Slide

122. Modeling Users as FSMs
     logged_out logged_in
     login
     logout
     vote
     

     View Slide

123. Generate Commands
     

     View Slide

124. Generate Commands
     

     View Slide

125. Generate Commands
     

     View Slide

126. Generate Commands
     

     View Slide

127. Generate Commands
     

     View Slide

128. Generate Commands
     

     View Slide

129. Generated Commands
     [{:vote, "chris", 1},
     {:vote, "chris", 2},
     {:vote, "jane", 1},
     {:vote, "jane", 1},
     {:vote, "jane", 3}
     {:vote, "chris", 2}]
     

     View Slide

130. Generated Commands
     [{:vote, "chris", 1},
     {:vote, "chris", 2},
     {:vote, "jane", 1},
     {:vote, "jane", 1},
     {:vote, "jane", 3}
     {:vote, "chris", 2}]
     [{:vote, "chris", 1},
     {:vote, "jane", 1}]
     

     View Slide

131. Property: Users votes should increase
     

     View Slide

132. Property: Users votes should increase
     test “users votes increase after voting" do
     end
     

     View Slide

133. Property: Users votes should increase
     test “users votes increase after voting" do
     ptest [commands: gen_commands("chris")] do
     end
     end
     

     View Slide

134. Property: Users votes should increase
     test “users votes increase after voting" do
     ptest [commands: gen_commands("chris")] do
     VoteCounter.reset()
     end
     end
     

     View Slide

135. Property: Users votes should increase
     test “users votes increase after voting" do
     ptest [commands: gen_commands("chris")] do
     VoteCounter.reset()
     {_state, result} = run_commands(commands, Client)
     end
     end
     

     View Slide

136. Property: Users votes should increase
     test “users votes increase after voting" do
     ptest [commands: gen_commands("chris")] do
     VoteCounter.reset()
     {_state, result} = run_commands(commands, Client)
     assert result
     end
     end
     

     View Slide

137. Property: Users votes should increase
     test “users votes increase after voting" do
     ptest [commands: gen_commands("chris")] do
     VoteCounter.reset()
     {_state, result} = run_commands(commands, Client)
     assert result
     end
     end
     def run_commands(commands, module) do
     Enum.reduce(
     commands,
     {0, true},
     & run_command(module, &1, &2) )
     end
     

     View Slide

138. def gen_commands(name) do
     end
     Command Generators
     

     View Slide

139. def gen_commands(name) do
     list(of: gen_vote(name), max: 20)
     end
     Command Generators
     

     View Slide

140. def gen_commands(name) do
     list(of: gen_vote(name), max: 20)
     end
     def gen_vote(name) do
     end
     Command Generators
     

     View Slide

141. def gen_commands(name) do
     list(of: gen_vote(name), max: 20)
     end
     def gen_vote(name) do
     tuple(like: {
     )})
     end
     Command Generators
     

     View Slide

142. def gen_commands(name) do
     list(of: gen_vote(name), max: 20)
     end
     def gen_vote(name) do
     tuple(like: {
     value(:vote),
     )})
     end
     Command Generators
     

     View Slide

143. def gen_commands(name) do
     list(of: gen_vote(name), max: 20)
     end
     def gen_vote(name) do
     tuple(like: {
     value(:vote),
     value(name),
     )})
     end
     Command Generators
     

     View Slide

144. def gen_commands(name) do
     list(of: gen_vote(name), max: 20)
     end
     def gen_vote(name) do
     tuple(like: {
     value(:vote),
     value(name),
     choose(from: [value(1), value(2), value(3)])})
     end
     Command Generators
     

     View Slide

145. defmodule ClientStateMachine do
     end
     

     View Slide

146. defmodule ClientStateMachine do
     def vote(name, id) do
     %{"votes" => new_votes} = post(id, name)
     {:ok, new_votes}
     end
     end
     

     View Slide

147. defmodule ClientStateMachine do
     def vote(name, id) do
     %{"votes" => new_votes} = post(id, name)
     {:ok, new_votes}
     end
     def vote_next(state, [id, name], _result) do
     {:ok, update_in(state, [name, to_string(id)], &(&1 + 1))}
     end
     end
     

     View Slide

148. defmodule ClientStateMachine do
     def vote(name, id) do
     %{"votes" => new_votes} = post(id, name)
     {:ok, new_votes}
     end
     def vote_next(state, [id, name], _result) do
     {:ok, update_in(state, [name, to_string(id)], &(&1 + 1))}
     end
     def vote_post(state, [id, name], actual_result) do
     expected_result = get_in(state, [name, to_string(id)]) + 1
     {:ok, actual_result == expected_result}
     end
     end
     

     View Slide

149. Property: Users votes should increase
     test “users votes increase after voting" do
     ptest [commands: gen_commands("chris")] do
     VoteCounter.reset()
     {_state, result} = run_commands(commands, Client)
     assert result
     end
     end
     

     View Slide

150. View Slide

151. Property: Users shouldn’t effect
     other users votes
     

     View Slide

152. Property: Users should get the correct votes
     test "users don't effect each others votes" do
     end
     

     View Slide

153. Property: Users should get the correct votes
     test "users don't effect each others votes" do
     ptest [chris: gen_commands("chris"), jane: gen_commands("jane")] do
     end
     end
     

     View Slide

154. Property: Users should get the correct votes
     test "users don't effect each others votes" do
     ptest [chris: gen_commands("chris"), jane: gen_commands("jane")] do
     VoteCounter.reset()
     end
     end
     

     View Slide

155. Property: Users should get the correct votes
     test "users don't effect each others votes" do
     ptest [chris: gen_commands("chris"), jane: gen_commands("jane")] do
     VoteCounter.reset()
     {_state, result} = run_commands([chris, jane], Client)
     end
     end
     

     View Slide

156. Property: Users should get the correct votes
     test "users don't effect each others votes" do
     ptest [chris: gen_commands("chris"), jane: gen_commands("jane")] do
     VoteCounter.reset()
     {_state, result} = run_commands([chris, jane], Client)
     assert result
     end
     end
     

     View Slide

157. Property: Users should get the correct votes
     test "users don't effect each others votes" do
     ptest [chris: gen_commands("chris"), jane: gen_commands("jane")] do
     VoteCounter.reset()
     {_state, result} = run_parallel_commands([chris, jane], Client)
     assert result
     end
     end
     

     View Slide

158. Running parallel tests
     def run_parallel_commands([l1, l2], module) do
     t1 = Task.async(fn -> run_commands(l1, module) end)
     t2 = Task.async(fn -> run_commands(l2, module) end)
     {_, ra} = Task.await(t1)
     {_, rb} = Task.await(t2)
     {:ok, ra && rb}
     end
     

     View Slide

159. View Slide

160. View Slide

161. The Bug
     def new(conn, %{"id" => id, "name" => name}) do
     {:ok, current_votes} = VoteCounter.get(id)
     new_votes = [name | current_votes]
     VoteCounter.put(id, new_votes)
     # Other nonsense
     end
     

     View Slide

162. The Bug
     Votes
     

     View Slide

163. The Bug
     Votes
     3
     3
     

     View Slide

164. The Bug
     Votes
     3
     3
     

     View Slide

165. The Bug
     Votes
     4
     4
     

     View Slide

166. The Bug
     Votes
     4
     4
     

     View Slide

167. The Bug
     3 + 1 + 1 == 4
     ?
     

     View Slide

168. The Bug
     def new(conn, %{"id" => id, "name" => name}) do
     {:ok, current_votes} = VoteCounter.get(id)
     new_votes = [name | current_votes]
     VoteCounter.put(id, new_votes)
     # Other nonsense
     end
     

     View Slide

169. The Bug
     def new(conn, %{"id" => id, "name" => name}) do
     {:ok, new_votes} = VoteCounter.incr(id, name)
     # Other nonsense
     end
     

     View Slide

170. View Slide

171. Conclusion
     

     View Slide

172. How to think in properties
     Generating data
     Generate commands
     Model users as FSMs
     

     View Slide

173. Resources:
     “Finding Race conditions in Erlang with QuickCheck and PULSE”
     http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.724.3518&rep=rep1&type=pdf
     Testing Async apis with QuickCheck
     https://www.youtube.com/watch?v=iW2J7Of8jsE&t=272s
     “QuickCheck: A lightweight tool for Random Testing of Haskell Programs”
     http://www.cs.tufts.edu/~nr/cs257/archive/john-hughes/quick.pdf
     Composing Test Generators
     https://www.youtube.com/watch?v=4-sPhFtGwZk
     Property based testing for better code
     https://www.youtube.com/watch?v=shngiiBfD80
     

     View Slide

174. If you could write less code and find
     more bugs would you do that?
     

     View Slide

175. Thanks
     Chris Keathley / @ChrisKeathley / [email protected]
     

     View Slide