python - How to test that a model was used in a FastAPI route?

Question

I'm trying to check if a specific model was used as an input parser for a FastAPI route. However, I'm not sure how to patch (or spy on) it.

I have the following file structure:

.
└── roo
    ├── __init__.py
    ├── main.py
    └── test_demo.py

main.py:

from fastapi import FastAPI
from pydantic import BaseModel

app = FastAPI()

class ItemModel(BaseModel):
    name: str

@app.post("/")
async def read_main(item: ItemModel):
    return {"msg": f"Item: {item.name}"}

test_demo.py:

from fastapi.testclient import TestClient
from unittest.mock import patch
from roo.main import app, ItemModel

client = TestClient(app)

def test_can_creating_new_item_users_proper_validation_model():
    with patch('roo.main.ItemModel', wraps=ItemModel) as patched_model:
        response = client.post("/", json={'name': 'good'})
    assert response.status_code == 200
    assert response.json() == {"msg": "Item: good"}
    assert patched_model.called

However, patched_model is never called (other asserts pass). I don't want to change the functionality or replace ItemModel in main.py, I just want to check if it was used.

question from:https://stackoverflow.com/questions/65876503/how-to-test-that-a-model-was-used-in-a-fastapi-route

Answer 1

My first approach to this was to wrap the read_main method and check that the item passed into the function is indeed an instance of ItemModel. But that was a dead-end approach because of the way FastAPI endpoints are prepared and stored: FastAPI stores a copy of the endpoint function objects in a list: (see fastapi/routing.py), then evaluates at request-time which endpoint to call.

from roo.main import app

def test_read_main():
    assert 'read_main' in [r.endpoint.__name__ for r in app.routes]

    # check that read_main was called *and* received an ItemModel instance?

My second approach involves spying or "breaking" the initialization of ItemModel, such that if the endpoint does indeed use that model, then a "broken" ItemModel would cause a request that hits that endpoint to fail. We "break" ItemModel by making use of the fact that (1) FastAPI calls the __init__ of your model during the request-response cycle, and (2) a 422 error response is propagated by default when the endpoint is unable to serialize a model properly:

class ItemModel(BaseModel):
    name: str

    def __init__(__pydantic_self__, **data: Any) -> None:
        print("Make a POST request and confirm that this is printed out")
        super().__init__(**data)

So in tests, just mock the __init__ method:

• Example for pytest

  import pytest
  from fastapi.testclient import TestClient
  from roo.main import app, ItemModel
  
  def test_read_main(monkeypatch: pytest.MonkeyPatch):
      client = TestClient(app)
  
      def broken_init(self, **data):
          pass  # `name` and other fields won't be set
  
      monkeypatch.setattr(ItemModel, '__init__', broken_init)
      with pytest.raises(AttributeError) as exc:
          client.post("/", json={'name': 'good'})
          assert 422 == response.status_code
      assert "'ItemModel' object has no attribute" in str(exc.value)
  

• Example for pytest + pytest-mock's mocker.spy

  from fastapi.testclient import TestClient
  from pytest_mock import MockerFixture
  from roo.main import app, ItemModel
  
  def test_read_main(mocker: MockerFixture):
      client = TestClient(app)
      spy = mocker.spy(ItemModel, '__init__')
  
      client.post("/", json={'name': 'good'})
      spy.assert_called()
      spy.assert_called_with(**{'name': 'good'})
  

• Example for unittest

  from fastapi.testclient import TestClient
  from roo.main import app, ItemModel
  from unittest.mock import patch
  
  def test_read_main():
      client = TestClient(app)
  
      # Wrapping __init__ like this isn't really correct, but serves the purpose
      with patch.object(ItemModel, '__init__', wraps=ItemModel.__init__) as mocked_init:
          response = client.post("/", json={'name': 'good'})
          assert 422 == response.status_code
          mocked_init.assert_called()
          mocked_init.assert_called_with(**{'name': 'good'})
  

Again, the tests check that the endpoint fails in either serializing into an ItemModel or in accessing item.name, which will only happen if the endpoint is indeed using ItemModel.

If you modify the endpoint from item: ItemModel into item: OtherModel:

class OtherModel(BaseModel):
    name: str

class ItemModel(BaseModel):
    name: str

@app.post("/")
async def read_main(item: OtherModel):  # <----
    return {"msg": f"Item: {item.name}"}

then running the tests should now fail because the endpoint is now creating the wrong object:

    def test_read_main(mocker: MockerFixture):
        client = TestClient(app)
        spy = mocker.spy(ItemModel, '__init__')
    
        client.post("/", json={'name': 'good'})
>       spy.assert_called()
E       AssertionError: Expected '__init__' to have been called.

test_demo_spy.py:11: AssertionError

        with pytest.raises(AttributeError) as exc:
            response = client.post("/", json={'name': 'good'})
>           assert 422 == response.status_code
E           assert 422 == 200
E             +422
E             -200

test_demo_pytest.py:15: AssertionError

The assertion errors for 422 == 200 is a bit confusing, but it basically means that even though we "broke" ItemModel, we still got a 200/OK response.. which means ItemModel is not being used.

Likewise, if you modified the tests first and mocked-out the __init__ of OtherModel' instead of ItemModel`, then running the tests without modifying the endpoint will result in similar failing tests:

    def test_read_main(mocker: MockerFixture):
        client = TestClient(app)
        spy = mocker.spy(OtherModel, '__init__')
    
        client.post("/", json={'name': 'good'})
>       spy.assert_called()
E       AssertionError: Expected '__init__' to have been called.

    def test_read_main():
        client = TestClient(app)
    
        with patch.object(OtherModel, '__init__', wraps=OtherModel.__init__) as mocked_init:
            response = client.post("/", json={'name': 'good'})
            # assert 422 == response.status_code
    
>           mocked_init.assert_called()
E           AssertionError: Expected '__init__' to have been called.

The assertion here is less confusing because it says we expected that the endpoint will call OtherModel's __init__, but it wasn't called. It should pass after modifying the endpoint to use item: OtherModel.

One last thing to note is that since we are manipulating the __init__, then it can cause the "happy path" to fail, so it should now be tested separately. Make sure to undo/revert the mocks and patches:

• Example for pytest

  def test_read_main(monkeypatch: pytest.MonkeyPatch):
      client = TestClient(app)
  
      def broken_init(self, **data):
          pass
  
      # Are we really using ItemModel?
      monkeypatch.setattr(ItemModel, '__init__', broken_init)
      with pytest.raises(AttributeError) as exc:
          response = client.post("/", json={'name': 'good'})
          assert 422 == response.status_code
      assert "'ItemModel' object has no attribute" in str(exc.value)
  
      # Okay, really using ItemModel. Does it work correctly?
      monkeypatch.undo()
      response = client.post("/", json={'name': 'good'})
      assert response.status_code == 200
      assert response.json() == {"msg": "Item: good"}
  

• Example for pytest + pytest-mock's mocker.spy

  from pytest_mock import MockerFixture
  from fastapi.testclient import TestClient
  from roo.main import app, ItemModel
  
  
  def test_read_main(mocker: MockerFixture):
      client = TestClient(app)
  
      # Are we really using ItemModel?
      spy = mocker.spy(ItemModel, '__init__')
      client.post("/", json={'name': 'good'})
      spy.assert_called()
      spy.assert_called_with(**{'name': 'good'})
  
      # Okay, really using ItemModel. Does it work correctly?
      mocker.stopall()
      response = client.post("/", json={'name': 'good'})
      assert response.status_code == 200
      assert response.json() == {"msg": "Item: good"}
  

• Example for unittest

  def test_read_main():
      client = TestClient(app)
  
      # Are we really using ItemModel?
      with patch.object(ItemModel, '__init__', wraps=ItemModel.__init__) as mocked_init:
          response = client.post("/", json={'name': 'good'})
          assert 422 == response.status_code
          mocked_init.assert_called()
          mocked_init.assert_called_with(**{'name': 'good'})
  
      # Okay, really using ItemModel. Does it work correctly?
      response = client.post("/", json={'name': 'good'})
      assert response.status_code == 200
      assert response.json() == {"msg": "Item: good"}
  

All in all, you might want to consider if/why it's useful to check for which model is exactly used. Normally, I just check that passing-in valid request params returns the expected valid response, and likewise, that invalid requests returns an error response.