This was my attempt at describing a methodology for building systems in elixir that can handle failures at all levels. It touches on technology solutions as well as how to engage humans in those solutions.
Complex systems run in degraded mode. “…complex systems run as broken systems. The system continues to function because it contains so many redundancies and because people can make it function, despite the presence of many flaws… System operations are dynamic, with components (organizational, human, technical) failing and being replaced continuously.”
Lets talk about… Booting the runtime & Configuration Starting dependencies Connecting to external systems Alarms and feedback Communicating with services we don’t control
Lets talk about… Booting the runtime & Configuration Starting dependencies Connecting to external systems Alarms and feedback Communicating with services we don’t control
defmodule Jenga.Application do use Application def start(_type, _args) do children = [ ] opts = [strategy: :one_for_one, name: Jenga.Supervisor] Supervisor.start_link(children, opts) end end
defmodule Jenga.Config do use GenServer def start_link(desired_config) do GenServer.start_link(__MODULE__, desired_config, name: __MODULE__) end def init(desired) do :jenga_config = :ets.new(:jenga_config, [:set, :protected, :named_table]) end end
defmodule Jenga.Config do use GenServer def start_link(desired_config) do GenServer.start_link(__MODULE__, desired_config, name: __MODULE__) end def init(desired) do :jenga_config = :ets.new(:jenga_config, [:set, :protected, :named_table]) case load_config(:jenga_config, desired) do :ok -> {:ok, %{table: :jenga_config, desired: desired}} end end end
defmodule Jenga.Config do use GenServer def start_link(desired_config) do GenServer.start_link(__MODULE__, desired_config, name: __MODULE__) end def init(desired) do :jenga_config = :ets.new(:jenga_config, [:set, :protected, :named_table]) case load_config(:jenga_config, desired) do :ok -> {:ok, %{table: :jenga_config, desired: desired}} :error -> {:stop, :could_not_load_config} end end end
** (Mix) Could not start application jenga: Jenga.Application.start(:normal, []) returned an error: shutdown: failed to start child: Jenga.Config ** (EXIT) :could_not_load_config
Lets talk about… Booting the runtime & Configuration Starting dependencies Connecting to external systems Alarms and feedback Communicating with services we don’t control
Lets talk about… Booting the runtime & Configuration Starting dependencies Connecting to external systems Alarms and feedback Communicating with services we don’t control
defmodule JengaWeb.Endpoint do use Phoenix.Endpoint, otp_app: :jenga def init(_key, config) do port = Jenga.Config.get(:port) {:ok, Keyword.put(config, :http, [:inet6, port: port])} end end
defmodule JengaWeb.UpController do use JengaWeb, :controller def up(conn, _params) do {code, message} = status() conn |> Plug.Conn.put_status(code) |> json(message) end defp status do {500, %{status: “LOADING”}} end end
Lets talk about… Booting the runtime & Configuration Starting dependencies Connecting to external systems Alarms and feedback Communicating with services we don’t control
Lets talk about… Booting the runtime & Configuration Starting dependencies Connecting to external systems Alarms and feedback Communicating with services we don’t control
defmodule Jenga.DemoConnection do use GenServer def init(opts) do wait_for = 3_000 + backoff() + jitter() Process.send_after(self(), {:try_connect, opts}, wait_for) {:ok, %{state: :disconnected}} end end
defmodule Jenga.DemoConnection do use GenServer def init(opts) do wait_for = 3_000 + backoff() + jitter() Process.send_after(self(), {:try_connect, opts}, wait_for) {:ok, %{state: :disconnected}} end def handle_info({:try_connect, opts}, _) do do_connect(opts) {:noreply, state} end end
Lets talk about… Booting the runtime & Configuration Starting dependencies Connecting to external systems Alarms and feedback Communicating with services we don’t control
Lets talk about… Booting the runtime & Configuration Starting dependencies Connecting to external systems Alarms and feedback Communicating with services we don’t control
defmodule JengaWeb.UpController do use JengaWeb, :controller def up(conn, _params) do {code, message} = status() conn |> Plug.Conn.put_status(code) |> json(message) end defp status do {500, %{status: “LOADING”}} end end
defmodule JengaWeb.UpController do use JengaWeb, :controller def up(conn, _params) do {code, message} = status() conn |> Plug.Conn.put_status(code) |> json(message) end defp status do case Database.check_status() do :ok -> {200, %{status: "OK"}} _ -> {500, %{status: "LOADING"}} end end end
defmodule Jenga.Database.Watchdog do use GenServer def init(:ok) do schedule_check() {:ok, %{status: :degraded, passing_checks: 0}} end def handle_info(:check_db, state) do status = Jenga.Database.check_status() state = change_state(status, state) schedule_check() {:noreply, state} end end
defmodule Jenga.Database.Watchdog do use GenServer def init(:ok) do schedule_check() {:ok, %{status: :degraded, passing_checks: 0}} end def handle_info(:check_db, state) do status = Jenga.Database.check_status() state = change_state(status, state) schedule_check() {:noreply, state} end defp change_state(result, %{status: status, passing_checks: count}) do end end
defmodule Jenga.Database.Watchdog do use GenServer def init(:ok) do schedule_check() {:ok, %{status: :degraded, passing_checks: 0}} end def handle_info(:check_db, state) do status = Jenga.Database.check_status() state = change_state(status, state) schedule_check() {:noreply, state} end defp change_state(result, %{status: status, passing_checks: count}) do case {result, status, count} do {:ok, :connected, count} -> if count == 3 do :alarm_handler.clear_alarm(@alarm_id) end %{status: :connected, passing_checks: count + 1} {:ok, :degraded, _} -> %{status: :connected, passing_checks: 0} end end end
defmodule Jenga.Database.Watchdog do use GenServer def init(:ok) do schedule_check() {:ok, %{status: :degraded, passing_checks: 0}} end def handle_info(:check_db, state) do status = Jenga.Database.check_status() state = change_state(status, state) schedule_check() {:noreply, state} end defp change_state(result, %{status: status, passing_checks: count}) do case {result, status, count} do {:ok, :connected, count} -> if count == 3 do :alarm_handler.clear_alarm(@alarm_id) end %{status: :connected, passing_checks: count + 1} {:ok, :degraded, _} -> %{status: :connected, passing_checks: 0} {:error, :connected, _} -> :alarm_handler.set_alarm({@alarm_id, "We cannot connect to the database”}) %{status: :degraded, passing_checks: 0} {:error, :degraded, _} -> %{status: :degraded, passing_checks: 0} end end end
defmodule Jenga.AlarmHandler do require Logger def init({:ok, {:alarm_handler, _old_alarms}}) do Logger.info("Installing alarm handler") {:ok, %{}} end def handle_event({:set_alarm, :database_disconnected}, alarms) do send_alert_to_slack(database_alarm()) {:ok, alarms} end def handle_event({:clear_alarm, :database_disconnected}, alarms) do send_recovery_to_slack(database_alarm()) {:ok, alarms} end def handle_event(event, state) do Logger.info("Unhandled alarm event: #{inspect(event)}") {:ok, state} end end
Lets talk about… Booting the runtime & Configuration Starting dependencies Connecting to external systems Alarms and feedback Communicating with services we don’t control
Lets talk about… Booting the runtime & Configuration Starting dependencies Connecting to external systems Alarms and feedback Communicating with services we don’t control
defmodule Jenga.ExternalService do def fetch(params) do with :ok <- :fuse.ask(@fuse, :async_dirty), {:ok, result} <- make_call(params) do {:ok, result} else {:error, e} -> :ok = :fuse.melt(@fuse) {:error, e} :blown -> {:error, :service_is_down} end end end
Lets talk about… Booting the runtime & Configuration Starting dependencies Connecting to external systems Alarms and feedback Communicating with services we don’t control
keathley
kazuhitoyokoi
antonicg
cohalz
sji
kilometer
memory1994
shogogg
manfredsteyer
77web
ennael
excid3
mfunaki
aki_iinuma
hannesfritz
panda_program
paulrobertlloyd
eileencodes
lauravandoore
addyosmani
bkeepers
pauljervisheath
zakiyama
pedronauck
smashingmag