diff --git a/lua/ivy/config.lua b/lua/ivy/config.lua
new file mode 100644
index 0000000..8ff1e6a
--- /dev/null
+++ b/lua/ivy/config.lua
@@ -0,0 +1,33 @@
+local config_mt = {}
+config_mt.__index = config_mt
+
+function config_mt:get_in(config, key_table)
+  local current_value = config
+  for _, key in ipairs(key_table) do
+    if current_value == nil then
+      return nil
+    end
+
+    current_value = current_value[key]
+  end
+
+  return current_value
+end
+
+function config_mt:get(key_table)
+  return self:get_in(self.user_config, key_table) or self:get_in(self.default_config, key_table)
+end
+
+local config = { user_config = {} }
+
+config.default_config = {
+  backends = {
+    "ivy.backends.buffers",
+    "ivy.backends.files",
+    "ivy.backends.lines",
+    "ivy.backends.rg",
+    "ivy.backends.lsp-workspace-symbols",
+  },
+}
+
+return setmetatable(config, config_mt)
diff --git a/lua/ivy/config_spec.lua b/lua/ivy/config_spec.lua
new file mode 100644
index 0000000..7ef8cd8
--- /dev/null
+++ b/lua/ivy/config_spec.lua
@@ -0,0 +1,27 @@
+local config = require "ivy.config"
+
+describe("config", function()
+  before_each(function()
+    config.user_config = {}
+  end)
+
+  it("gets the first item when there is only default values", function()
+    local first_backend = config:get { "backends", 1 }
+    assert.is_equal("ivy.backends.buffers", first_backend)
+  end)
+
+  it("returns nil if we access a key that is not a valid config item", function()
+    assert.is_nil(config:get { "not", "a", "thing" })
+  end)
+
+  it("returns the users overridden config value", function()
+    config.user_config = { backends = { "ivy.my.backend" } }
+    local first_backend = config:get { "backends", 1 }
+    assert.is_equal("ivy.my.backend", first_backend)
+  end)
+
+  it("returns a nested value", function()
+    config.user_config = { some = { nested = "value" } }
+    assert.is_equal(config:get { "some", "nested" }, "value")
+  end)
+end)
diff --git a/lua/ivy/init.lua b/lua/ivy/init.lua
index f62f6fc..58672ff 100644
--- a/lua/ivy/init.lua
+++ b/lua/ivy/init.lua
@@ -1,28 +1,32 @@
 local controller = require "ivy.controller"
+local config = require "ivy.config"
 local register_backend = require "ivy.register_backend"
 
--- Local variable to check if ivy has been setup, this is to prevent multiple
--- setups of ivy
-local has_setup = false
-
 local ivy = {}
 ivy.run = controller.run
 ivy.register_backend = register_backend
 
+-- Private variable to check if ivy has been setup, this is to prevent multiple
+-- setups of ivy. This is only exposed for testing purposes.
+---@private
+ivy.has_setup = false
+
 ---@class IvySetupOptions
 ---@field backends (IvyBackend | { ["1"]: string, ["2"]: IvyBackendOptions} | string)[]
 
----@param config IvySetupOptions
-function ivy.setup(config)
-  if has_setup then
+---@param user_config IvySetupOptions
+function ivy.setup(user_config)
+  if ivy.has_setup then
     return
   end
 
-  for _, backend in ipairs(config.backends) do
+  config.user_config = user_config or {}
+
+  for _, backend in ipairs(config:get { "backends" } or {}) do
     register_backend(backend)
   end
 
-  has_setup = true
+  ivy.has_setup = true
 end
 
 return ivy
diff --git a/lua/ivy/init_spec.lua b/lua/ivy/init_spec.lua
new file mode 100644
index 0000000..f8ea91b
--- /dev/null
+++ b/lua/ivy/init_spec.lua
@@ -0,0 +1,30 @@
+local ivy = require "ivy"
+local config = require "ivy.config"
+
+describe("ivy.setup", function()
+  before_each(function()
+    ivy.has_setup = false
+    config.user_config = {}
+  end)
+
+  it("sets the users config options", function()
+    ivy.setup { backends = { "ivy.backends.files" } }
+    assert.is_equal("ivy.backends.files", config:get { "backends", 1 })
+  end)
+
+  it("will not reconfigure if its called twice", function()
+    ivy.setup { backends = { "ivy.backends.files" } }
+    ivy.setup { backends = { "some.backend" } }
+    assert.is_equal("ivy.backends.files", config:get { "backends", 1 })
+  end)
+
+  it("does not crash if you don't pass in any params to the setup function", function()
+    ivy.setup()
+    assert.is_equal("ivy.backends.buffers", config:get { "backends", 1 })
+  end)
+
+  it("will fallback if the key is not set at all in the users config", function()
+    ivy.setup { some_key = "some_value" }
+    assert.is_equal("ivy.backends.buffers", config:get { "backends", 1 })
+  end)
+end)