diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..fe5f950 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,9 @@ +FROM mcr.microsoft.com/devcontainers/python:3.10 + +COPY zscaler-bundle.pem /tmp/zscaler.pem +RUN if [ -s /tmp/zscaler.pem ]; then \ + cp /tmp/zscaler.pem /usr/local/share/ca-certificates/zscaler.crt && \ + update-ca-certificates; \ + fi && rm -f /tmp/zscaler.pem + +RUN find /etc/apt /usr/share/keyrings -name "*yarn*" -delete 2>/dev/null || true diff --git a/.devcontainer/devcontainer-lock.json b/.devcontainer/devcontainer-lock.json new file mode 100644 index 0000000..fbc79c1 --- /dev/null +++ b/.devcontainer/devcontainer-lock.json @@ -0,0 +1,9 @@ +{ + "features": { + "ghcr.io/devcontainers/features/docker-in-docker:3": { + "version": "3.0.1", + "resolved": "ghcr.io/devcontainers/features/docker-in-docker@sha256:ca2508495b01ba29eba93e8153772a2daa65eaa86471cc6863fe2a3d21933df9", + "integrity": "sha256:ca2508495b01ba29eba93e8153772a2daa65eaa86471cc6863fe2a3d21933df9" + } + } +} diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 14a6eb7..4de17f2 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,7 +1,7 @@ { "name": "MLE Interview", - // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile - "image": "mcr.microsoft.com/devcontainers/python:1-3.10", + "build": { "dockerfile": "Dockerfile" }, + "initializeCommand": "cp ~/.ssl/zscaler-bundle.pem .devcontainer/zscaler-bundle.pem 2>/dev/null || touch .devcontainer/zscaler-bundle.pem", "customizations": { "vscode": { "settings": { @@ -14,7 +14,7 @@ "source.organizeImports.ruff": true, "source.organizeImports": true } - }, + } }, "extensions": [ "charliermarsh.ruff", @@ -29,7 +29,10 @@ } }, "features": { - "ghcr.io/devcontainers/features/docker-in-docker": {} + "ghcr.io/devcontainers/features/docker-in-docker:3": { + "moby": false, + "disableIp6tables": true + } }, "postCreateCommand": "make setup" -} \ No newline at end of file +} diff --git a/.gitignore b/.gitignore index 3f83ee9..8d5a0af 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# Zscaler cert (machine-specific, copied by initializeCommand) +.devcontainer/zscaler-bundle.pem + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/INTERVIEW.md b/INTERVIEW.md index 534694b..b14ecdc 100644 --- a/INTERVIEW.md +++ b/INTERVIEW.md @@ -1,28 +1,153 @@ - ## Coding Challenge -You can refer to Google or relevant documentation to guide you in addressing the issues below. -You are not allowed to use ChatGPT, Copilot, or any similar tools for assistance. +You are working on a user interest recommendation system. It consists of: + +- A **TensorFlow model** trained on user content interactions, served via TF Serving (Docker) +- A **Flask API** that fetches user features, calls the model, and returns ranked interests +- A **feature store** backed by CSV files + +There are **7 tasks** to complete. Tasks 1, 3, 4, 5 and 6 each have a failing test that tells you what to fix. Tasks 2 and 7 are verified by running the app. You can refer to Google or relevant documentation — no AI-assisted coding tools. + +--- + +### Running the tests + +**Every task has its own command** — `make test-1` through `make test-7`. Run only the one for the task you are on: + +``` +make test-4 +``` + +`make test` runs everything at once. Avoid it until the end: tests for tasks you have not reached yet will fail and bury the output you care about. + +**Read the last few lines of the output.** Each test failure ends with the assertion message that names exactly what is wrong, for example: + +``` +E AssertionError: Incorrect number of interests +``` + +That message is the whole signal — you do not need to read the traceback above it. + +**The tests are correct.** Nearly every fix belongs in the source code, in `api/` or `model/` — not in `tests/`. **Task 4 is the only exception:** `top_k` has to be passed into the request as a query parameter, so that test does change. + +--- + +### Terminal setup (do this before task 4) + +Once the model is trained and served, you'll need **two terminals** running simultaneously inside the container: + +| Terminal | Command | Purpose | +|---|---|---| +| 1 | `make serve-model` | TF Serving on port 8501 | +| 2 | `make serve-api` | Flask API on port 5005 | + +Keep both running while working on tasks 4–7. + +--- + +### Tasks + +**1. Fix model training** + +Run `make test-1`. The test fails. + +Find and fix the bug in the model code, then verify: +``` +make test-1 +``` + +Once passing, train the model and start the model server (leave it running): +``` +make train +make serve-model +``` + +--- + +**2. Fix the API startup** + +Run `make serve-api`. The Flask application fails to start. + +Find and fix the bug, then start the server again and leave it running in its terminal: +``` +make serve-api +``` + +From a second terminal, verify: +``` +make test-2 +``` + +--- + +**3. Fix the probability scores** + +The model's serving function returns raw logit scores, not probabilities. These values are not bounded between 0 and 1 and cannot be meaningfully compared or filtered. + +Find where the issue originates and fix it so that the scores returned represent proper probability values. There are two valid approaches — both are acceptable. + +Verify: +``` +make test-3 +``` + +This test exercises the model in memory. For the **served** model to return probabilities too, re-run `make train` and restart `make serve-model`. + +--- + +**4. Fix the interests count** + +With both servers running, run `make test-4`. It fails with: +``` +AssertionError: Incorrect number of interests +``` + +The test asks for 15 interests but never tells the API how many it wants. Make the request send `top_k`, and make sure the API honours it. Verify: +``` +make test-4 +``` + +This is the one task where you edit a test. + +--- + +**5. Fix response time** + +With both servers running, run `make test-5`. It fails with: +``` +AssertionError: Request greater than 1 second +``` + +Find the performance bottleneck and fix it. Verify: +``` +make test-5 +``` + +--- + +**6. Add probability to the response** + +With both servers running, run `make test-6`. It fails with: +``` +AssertionError: Interest object missing 'probability': ... +``` -1. **Model Training Error Resolution:** - - Begin by executing `make test` to identify any issues during model training. - - After passing the `TestModel::test_model_build` test, use `make train` followed by `make serve-model` to train and serve the model, preparing it for upcoming steps. +Update the API response so each interest includes a `probability` field. Verify: +``` +make test-6 +``` -2. **API Startup Issue Fix:** - - Resolve the Flask API startup issue with `make serve-api`. - - Run `make serve-api` to start a development Flask server, then open a new terminal window without shutting down the server. +--- -3. **Data Return and Interests Length Correction:** - - Execute `make test` and resolve the `TestInterestsAPI::test_basic_response - AssertionError: Incorrect number of interests` test. - - Ensure the return data structure and interests array length meet the specifications, adjusting the API as necessary. +**7. Add probability threshold filtering** -4. **Response Time Optimization:** - - Run `make test` and fix the `TestInterestsAPI::test_basic_response - AssertionError: Request greater than 1 second` test. - - Enhance the API to achieve a response time of one second or less. +Implement a feature that lets API clients filter results by a minimum probability score. For example: -5. **Probability Field Inclusion:** - - Perform `make test` and correct the `TestInterestsAPI::test_with_probability` test. - - Update the API response to incorporate a 'probability' field. +``` +GET /interests/?min_probability=0.5 +``` -6. **Probability Score Filtering Implementation:** - - Create a feature allowing API clients to filter results by probability score, enabling users to specify a probability threshold for more targeted results. \ No newline at end of file +Should return only interests with probability ≥ 0.5. There is no automated test for this task — check the response yourself: +``` +make test-7 +``` diff --git a/Makefile b/Makefile index 80daa2d..76b8e42 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,66 @@ +PYTEST = TF_CPP_MIN_LOG_LEVEL=3 pytest +HANDLE ?= e337a675-46f5-437e-aa72-5d43643b5461 + +.PHONY: help setup serve-api serve-model ping-model train test \ + test-1 test-2 test-3 test-4 test-5 test-6 test-7 + +help: + @printf 'Setup and servers\n' + @printf ' make setup Install dependencies\n' + @printf ' make train Train the model and export it to serving/\n' + @printf ' make serve-model Start TF Serving on port 8501\n' + @printf ' make serve-api Start the Flask API on port 5005\n' + @printf ' make ping-model Check that TF Serving has loaded the model\n' + @printf '\nInterview tasks (run only the target for the task you are on)\n' + @printf ' make test-1 Task 1: model build\n' + @printf ' make test-2 Task 2: API starts up\n' + @printf ' make test-3 Task 3: probability scores\n' + @printf ' make test-4 Task 4: interests count\n' + @printf ' make test-5 Task 5: response time\n' + @printf ' make test-6 Task 6: probability in the response\n' + @printf ' make test-7 Task 7: min_probability filtering\n' + @printf ' make test Every test at once (expect noise until all tasks are done)\n' + setup: pip install -r requirements.txt serve-api: python -m flask --app api/app run --debug --port 5005 --host 0.0.0.0 - + serve-model: docker compose up tf-serving +ping-model: + curl -s http://localhost:8501/v1/models/interests-model | python3 -m json.tool + train: TF_CPP_MIN_LOG_LEVEL=3 python -m model.main test: - TF_CPP_MIN_LOG_LEVEL=3 pytest ./tests -p no:warnings \ No newline at end of file + $(PYTEST) + +test-1: + $(PYTEST) tests/test_model.py::TestModel::test_model_build + +# No automated test — the API only starts once the bug is fixed. +test-2: + @curl -fs http://localhost:5005/ \ + && printf '\nPASS: the API is up\n' \ + || printf 'FAIL: the API is not reachable on port 5005 (see task 2)\n' + +test-3: + $(PYTEST) tests/test_model.py::TestModel::test_probability_scores + +test-4: + $(PYTEST) tests/test_app.py::TestInterestsAPI::test_interests_count + +test-5: + $(PYTEST) tests/test_app.py::TestInterestsAPI::test_response_time + +test-6: + $(PYTEST) tests/test_app.py::TestInterestsAPI::test_with_probability_threshold + +# No automated test — read the response and check every probability is >= 0.5. +test-7: + @curl -fsS "http://localhost:5005/interests/$(HANDLE)?min_probability=0.5" \ + | python3 -m json.tool diff --git a/README.md b/README.md index 5306772..236dbb0 100644 --- a/README.md +++ b/README.md @@ -1,120 +1,106 @@ -# MLE Interview: - +# MLE Interview ## Setup -1. Install [Visual Studio Code](https://code.visualstudio.com/). Also the following plugins: - - [Dev Containers](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) -2. Clone this repo -3. Open the repository in VSCode (`code .`) -4. VSCode will prompt you to install recommended extensions. Select `Yes` - - If you missed this step, you can install recommended extensions using the extensions sidebar (`⇧⌘X`) -5. VSCode will prompt you to reopen the workspace in a Dev Container. Select `Yes`. - - If you missed this step, open the command prompt palette (`⇧⌘P`) and run the command `Remote-Containers: Rebuild and Reopen in Container` +### Option A — GitHub Codespaces (recommended) -## Make Commands +Click **Code → Open with Codespaces** on the repository page. The environment builds automatically — no local installation needed. -Before you start, ensure you have all the necessary dependencies installed: +### Option B — Local Dev Container (VS Code) -``` -make setup -``` +1. Install [Visual Studio Code](https://code.visualstudio.com/) and the [Dev Containers](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) extension +2. Install [Docker Desktop](https://www.docker.com/products/docker-desktop/) +3. Clone this repo and open it in VS Code (`code .`) +4. VS Code will prompt you to reopen in a Dev Container — select **Yes** + - If you miss the prompt: `⇧⌘P` → **Dev Containers: Rebuild and Reopen in Container** -This command will install the Python dependencies listed in `requirements.txt`. +> **Note (corporate networks / Zscaler):** If your machine uses Zscaler SSL inspection, ensure your Zscaler certificate bundle exists at `~/.ssl/zscaler-bundle.pem` before building the container. The build handles this automatically. -### Serve API +Once the container starts, dependencies are installed automatically via `make setup`. You do not need to run it manually. -To start the Flask API, use the following command: -``` -make serve-api -``` +## Make Commands -This command launches the Flask application defined in `api/app` on port 5005, accessible via `0.0.0.0`. The `--debug` flag is included to enable debug mode, which provides useful debugging information in case of errors and automatically reloads the server on code changes. +### Test -### Serve Model +Each interview task has its own target, so you only see the output for the task you are working on: -If you're using a TensorFlow model served via Docker, you can start the TensorFlow serving using: +| Command | Task | +|---|---| +| `make test-1` | 1 — model build | +| `make test-2` | 2 — API starts up | +| `make test-3` | 3 — probability scores | +| `make test-4` | 4 — interests count | +| `make test-5` | 5 — response time | +| `make test-6` | 6 — probability in the response | +| `make test-7` | 7 — `min_probability` filtering | ``` -make serve-model +make test ``` -This command uses Docker Compose to spin up a container defined in the Docker configuration, specifically targeting the TensorFlow serving service (`tf-serving`). Ensure Docker is installed and running on your system before executing this command. +Runs every test in `./tests` at once. Prefer the per-task targets while working — tests for tasks you have not reached yet will fail and make the output hard to read. -### Train +Run `make` on its own to list every available command. -To train the model, execute: +### Train ``` make train ``` -This command runs the main training script located at `model.main`. +Trains the model using `model/main.py` and exports a SavedModel to `serving/interests-model/1/`. -### Test +### Serve Model -Finally, to run the tests for your project, use: +``` +make serve-model +``` + +Starts TF Serving via Docker Compose on port 8501. Run this after `make train` — the model must be trained first. + +### Serve API ``` -make test +make serve-api ``` -This command runs all the tests in the `./tests` directory using pytest. +Starts the Flask development server on port 5005. Requires the model server to be running for inference endpoints to work. ## API -Route: GET: http://localhost:5005/interests/:userHandle +``` +GET http://localhost:5005/interests/ +``` + +Optional query parameters: +- `top_k` (int) — number of interests to return (default: 10) +- `min_probability` (float) — filter results below this probability score -API Response: +Example response: -``` +```json { "user_handle": "e337a675-46f5-437e-aa72-5d43643b5461", + "name": "Kisiza", + "type": "B2B", "interests": [ { "id": "29e020bb-7a02-41db-b21f-527a8ef4dfdf", - "label": "Accounting technician" + "label": "Accounting technician", + "probability": 0.94 }, { "id": "012abcd1-3a6a-4803-a47e-42f46b402024", - "label": "Field seismologist" + "label": "Field seismologist", + "probability": 0.87 }, { "id": "a1b028dd-8464-4c63-85e8-ae29ea184fc7", - "label": "Designer, industrial/product" - }, - { - "id": "686af7e4-6d58-4148-b227-3bf65ff10273", - "label": "Materials engineer" - }, - { - "id": "8d1526b9-a7bf-4972-be43-7b912f149667", - "label": "Fashion designer" - }, - { - "id": "d83a55a3-0143-4318-91c1-88f44ad59390", - "label": "Journalist, newspaper" - }, - { - "id": "cda4441f-dba6-495c-9e2e-7429bd5e0465", - "label": "Therapist, music" - }, - { - "id": "e9b23ad4-753b-4331-9cfa-525965fcf281", - "label": "Secretary, company" - }, - { - "id": "ca4b03b2-0ae2-440a-afc8-5469510b19cb", - "label": "Commissioning editor" - }, - { - "id": "fd3c41b8-8c15-47e2-a80d-cf3683b2d0da", - "label": "Copywriter, advertising" + "label": "Designer, industrial/product", + "probability": 0.81 } - ], - "name": "Kisiza", - "type": "B2B" + ] } -``` \ No newline at end of file +``` diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..5db2bee --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +testpaths = tests +addopts = -q --no-header --tb=short -p no:warnings -p no:cacheprovider diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..a8b2d31 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,31 @@ +import pytest +import requests + +API_URL = "http://localhost:5005/" +MODEL_URL = "http://localhost:8501/v1/models/interests-model" + + +def _is_up(url): + try: + requests.get(url, timeout=2).raise_for_status() + except requests.RequestException: + return False + return True + + +@pytest.fixture(scope="session") +def _servers_running(): + """Fail with a readable message instead of a ConnectionError traceback.""" + if not _is_up(MODEL_URL): + pytest.fail( + "TF Serving is not reachable on port 8501 — " + "run `make train`, then `make serve-model`.", + pytrace=False, + ) + + if not _is_up(API_URL): + pytest.fail( + "The Flask API is not reachable on port 5005 — " + "run `make serve-api` (see task 2).", + pytrace=False, + ) diff --git a/tests/test_app.py b/tests/test_app.py index 86bf5f6..673e36c 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -1,24 +1,24 @@ import time +import pytest import requests +HTTP_OK = 200 + +@pytest.mark.usefixtures("_servers_running") class TestInterestsAPI: BASE_URL = "http://localhost:5005/interests" - def make_request(self, user_handle): - return requests.get(f"{self.BASE_URL}/{user_handle}") + def make_request(self, user_handle, **params): + return requests.get(f"{self.BASE_URL}/{user_handle}", params=params) def test_basic_response(self): user_handle = "e337a675-46f5-437e-aa72-5d43643b5461" - top_k = 15 - start_time = time.time() response = self.make_request(user_handle=user_handle) - end_time = time.time() - request_time = end_time - start_time - assert response.status_code == 200, "Status code is not 200" + assert response.status_code == HTTP_OK, "Status code is not 200" response = response.json() @@ -26,9 +26,29 @@ def test_basic_response(self): assert isinstance(response["interests"], list), "Interests is not a list" + def test_interests_count(self): + user_handle = "e337a675-46f5-437e-aa72-5d43643b5461" + top_k = 15 + + response = self.make_request(user_handle=user_handle) + + assert response.status_code == HTTP_OK, "Status code is not 200" + + response = response.json() + assert len(response["interests"]) == top_k, "Incorrect number of interests" - assert request_time < 1, "Request greater than 1 seconds" + def test_response_time(self): + user_handle = "e337a675-46f5-437e-aa72-5d43643b5461" + + start_time = time.time() + response = self.make_request(user_handle=user_handle) + end_time = time.time() + request_time = end_time - start_time + + assert response.status_code == HTTP_OK, "Status code is not 200" + + assert request_time < 1, "Request greater than 1 second" def test_with_probability_threshold(self): user_handle = "e337a675-46f5-437e-aa72-5d43643b5461" diff --git a/tests/test_model.py b/tests/test_model.py index 5469061..541552c 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -32,6 +32,21 @@ def setup_method(self): hyper_params=hyper_params, ) + def test_probability_scores(self): + input_record = { + "USER_HANDLE": tf.constant([self.users[0]]), + "USER_TYPE": tf.constant(["B2B"]), + "CONTENT_TITLES_JOINED": tf.constant([self.content_titles[0]]), + } + top_k = tf.constant(1, dtype=tf.int64) + + result = self.model.serving_predict(input_record, top_k) + probabilities = result["probabilities"].numpy().flatten() + + assert all(0 <= p <= 1 for p in probabilities), ( + "Probabilities must be between 0 and 1" + ) + def test_model_build(self): interests_input = [self.interests_vocab[0], self.interests_vocab[3]]