Get started

Up and running in four steps.

You need a Mac with Apple Silicon on macOS 27 or 26, and Python 3.12. The steps are for release 0.4.1.

Your macOS:

1
Download and check
Get the engine for your macOS from the v0.4.1 release on GitHub, and check the file.
```
cd ~/Downloads && shasum -a 256 -c inferd-0.4.1-macos27-arm64.tar.gz.sha256   # prints OKtar xzf inferd-0.4.1-macos27-arm64.tar.gz && cd inferd-0.4.1
```
```
cd ~/Downloads && shasum -a 256 -c inferd-0.4.1-macos26-arm64.tar.gz.sha256   # prints OKtar xzf inferd-0.4.1-macos26-arm64.tar.gz && cd inferd-0.4.1
```
Download the file and its .sha256 from the v0.4.1 release. No Xcode needed. Tested on Mac Studio M5 Ultra, Mac mini M6, M5 Pro and M4 Pro.
Beta: the macOS 26 file from the v0.4.1 release was built on macOS 26 itself, with the same engine and toolchain as 0.4.0, whose macOS 26 file passed the full release gate on two macOS 26 Macs so far (external testers' 128 GB Mac on macOS 26.6 and MacBook Pro M4 Max, 36 GB, on macOS 26.6.2). To check it on yours, after step 2 run the release gate once (a few minutes, no other model running): PY=~/.venvs/inferd/bin/python tools/release_gate.sh ~/models/Qwen3.8-27B-4bit ~/models/Qwen3.8-27B-DFlash2. Never use this file on macOS 27. Or build from source (needs Xcode and its Metal compiler): git clone https://github.com/Incept5-AI/inferd && cd inferd && git checkout v0.4.1 && cd engine && ./build.sh; a source build passed the same checks on the 128 GB Mac.
2
Install
One command sets up the small Python front end and downloads Qwen3.8-27B with its draft model into ~/models. Safe to run again.
```
./install.sh --model 27b
```
You need Python 3.12: python.org or brew install python@3.12. The download is about 17 GB. A 32 GB Mac mini runs this model. Already have it in LM Studio or the Hugging Face cache? install.sh finds it there and skips the download (known equivalent builds count too, with a note); ./install.sh --list shows what it found.
3
Start it
The engine and its OpenAI-compatible server start on port 8300, sized for your Mac's memory.
```
./run.sh start --model 27b
```
./run.sh status and ./run.sh stop manage it. On Macs under 256 GB, agents' contexts are kept on the SSD after a stop and come back after a restart; ./run.sh clean deletes them; --dry-run shows what it will run, --help lists the options. Settings for each Mac: docs/MACHINES.md.

4 · Talk to it, from anything that speaks the OpenAI API

just for fun

curl localhost:8300/v1/chat/completions -H 'Content-Type: application/json' \
  -d '{"model":"qwen3.8-27b","messages":[{"role":"user","content":"Hello"}]}'

Run: Any terminal.

# pip install openai
from openai import OpenAI

client = OpenAI(base_url="http://localhost:8300/v1", api_key="not-needed")
reply = client.chat.completions.create(
    model="qwen3.8-27b",
    messages=[{"role": "user", "content": "Hello"}],
)
print(reply.choices[0].message.content)

Run: pip install openai; python3 client.py

// main.mts: npx tsx main.mts (or bun main.mts)
const reply = await fetch("http://localhost:8300/v1/chat/completions", {
  method: "POST",
  headers: { "Content-Type": "application/json" },
  body: JSON.stringify({ model: "qwen3.8-27b", messages: [{ role: "user", content: "Hello" }] }),
});
const data = await reply.json();
console.log(data.choices[0].message.content);

Run: npx tsx main.mts, or bun main.mts

// Cargo.toml:
//   async-openai = { version = "0.43", features = ["chat-completion"] }
//   tokio = { version = "1", features = ["full"] }

use async_openai::{config::OpenAIConfig, types::chat::{ChatCompletionRequestUserMessage, CreateChatCompletionRequestArgs}, Client};

#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
    let client = Client::with_config(OpenAIConfig::new().with_api_base("http://localhost:8300/v1"));
    let request = CreateChatCompletionRequestArgs::default()
        .model("qwen3.8-27b")
        .messages([ChatCompletionRequestUserMessage::from("Hello").into()])
        .build()?;
    let reply = client.chat().create(request).await?;
    println!("{}", reply.choices[0].message.content.as_deref().unwrap_or(""));
    Ok(())
}

Run: cargo run

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
)

func main() {
	body := []byte(`{"model": "qwen3.8-27b", "messages": [{"role": "user", "content": "Hello"}]}`)
	resp, err := http.Post("http://localhost:8300/v1/chat/completions", "application/json", bytes.NewReader(body))
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()
	var reply struct {
		Choices []struct {
			Message struct{ Content string } `json:"message"`
		} `json:"choices"`
	}
	if err := json.NewDecoder(resp.Body).Decode(&reply); err != nil {
		panic(err)
	}
	fmt.Println(reply.Choices[0].Message.Content)
}

Run: go run main.go

import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;

public class Main {
    public static void main(String[] args) throws Exception {
        String body = """
            {"model": "qwen3.8-27b", "messages": [{"role": "user", "content": "Hello"}]}""";
        HttpRequest request = HttpRequest.newBuilder(URI.create("http://localhost:8300/v1/chat/completions"))
            .header("Content-Type", "application/json")
            .POST(HttpRequest.BodyPublishers.ofString(body))
            .build();
        HttpResponse<String> reply = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString());
        System.out.println(reply.body());   // the JSON reply; parse it with your favourite JSON library
    }
}

Run: javac Main.java && java Main (Java 15+)

import java.net.URI
import java.net.http.HttpClient
import java.net.http.HttpRequest
import java.net.http.HttpResponse

fun main() {
    val body = """{"model": "qwen3.8-27b", "messages": [{"role": "user", "content": "Hello"}]}"""
    val request = HttpRequest.newBuilder(URI.create("http://localhost:8300/v1/chat/completions"))
        .header("Content-Type", "application/json")
        .POST(HttpRequest.BodyPublishers.ofString(body))
        .build()
    val reply = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString())
    println(reply.body())   // the JSON reply
}

Run: kotlinc main.kt -include-runtime -d hello.jar && java -jar hello.jar

// cc main.c -lcurl
#include <curl/curl.h>

int main(void) {
    CURL *curl = curl_easy_init();
    struct curl_slist *headers = curl_slist_append(NULL, "Content-Type: application/json");
    curl_easy_setopt(curl, CURLOPT_URL, "http://localhost:8300/v1/chat/completions");
    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
    curl_easy_setopt(curl, CURLOPT_POSTFIELDS,
        "{\"model\": \"qwen3.8-27b\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}");
    CURLcode rc = curl_easy_perform(curl);   // writes the JSON reply to stdout
    curl_slist_free_all(headers);
    curl_easy_cleanup(curl);
    return rc == CURLE_OK ? 0 : 1;
}

Run: cc main.c -lcurl && ./a.out

// c++ -std=c++17 main.cpp -lcurl   (brew install nlohmann-json)
#include <curl/curl.h>
#include <nlohmann/json.hpp>
#include <iostream>
#include <string>

static size_t collect(char *p, size_t size, size_t n, void *out) {
    static_cast<std::string *>(out)->append(p, size * n);
    return size * n;
}

int main() {
    nlohmann::json request = {{"model", "qwen3.8-27b"},
                              {"messages", {{{"role", "user"}, {"content", "Hello"}}}}};
    std::string body = request.dump(), reply;
    CURL *curl = curl_easy_init();
    curl_slist *headers = curl_slist_append(nullptr, "Content-Type: application/json");
    curl_easy_setopt(curl, CURLOPT_URL, "http://localhost:8300/v1/chat/completions");
    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
    curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str());
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, collect);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, &reply);
    curl_easy_perform(curl);
    curl_slist_free_all(headers);
    curl_easy_cleanup(curl);
    std::cout << nlohmann::json::parse(reply)["choices"][0]["message"]["content"].get<std::string>() << "\n";
}

Run: c++ -std=c++17 main.cpp -lcurl (brew install nlohmann-json)

import Foundation

var request = URLRequest(url: URL(string: "http://localhost:8300/v1/chat/completions")!)
request.httpMethod = "POST"
request.setValue("application/json", forHTTPHeaderField: "Content-Type")
request.httpBody = try JSONSerialization.data(withJSONObject: [
    "model": "qwen3.8-27b",
    "messages": [["role": "user", "content": "Hello"]],
])
let (data, _) = try await URLSession.shared.data(for: request)
let reply = try JSONSerialization.jsonObject(with: data) as! [String: Any]
let message = (reply["choices"] as! [[String: Any]])[0]["message"] as! [String: Any]
print(message["content"] as! String)

Run: swiftc main.swift && ./main

require "net/http"
require "json"

uri = URI("http://localhost:8300/v1/chat/completions")
body = { model: "qwen3.8-27b", messages: [{ role: "user", content: "Hello" }] }
reply = Net::HTTP.post(uri, body.to_json, "Content-Type" => "application/json")
puts JSON.parse(reply.body)["choices"][0]["message"]["content"]

Run: ruby main.rb (macOS includes Ruby)

use strict; use warnings;
use HTTP::Tiny; use JSON::PP;

my $body = encode_json({ model => "qwen3.8-27b", messages => [{ role => "user", content => "Hello" }] });
my $reply = HTTP::Tiny->new->post("http://localhost:8300/v1/chat/completions",
    { headers => { "Content-Type" => "application/json" }, content => $body });
print decode_json($reply->{content})->{choices}[0]{message}{content}, "\n";

Run: perl main.pl (core modules only)

-- osascript main.applescript   (the Mac's own scripting language, with a little help from curl)
set reply to do shell script "curl -s localhost:8300/v1/chat/completions -H 'Content-Type: application/json' -d '{\"model\":\"qwen3.8-27b\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | /usr/bin/jq -r '.choices[0].message.content'"
display notification reply with title "inferd"
return reply

Run: osascript main.applescript: the answer also pops up as a notification

! gfortran main.f90 -o hello   (Fortran 2008: yes, really)
program hello
  implicit none
  call execute_command_line("curl -s localhost:8300/v1/chat/completions " // &
    "-H 'Content-Type: application/json' " // &
    "-d '{""model"":""qwen3.8-27b"",""messages"":[{""role"":""user"",""content"":""Hello""}]}'")
end program hello

Run: gfortran main.f90 && ./a.out

Every example sends the same standard chat request, and every one was compiled and run against a stand-in server before it went on this page (14 languages). Agent frameworks that talk to OpenAI connect the same way: change the base URL to http://localhost:8300/v1.

inferd on GitHub Release 0.4.1