Build a full-featured AI assistant in Python using OpenAI, speech recognition, text-to-speech, and a tkinter GUI. Create a voice-enabled, context-aware assistant with memory, tools, and a polished interface.
Build "Nova" — a full-featured AI assistant with voice input, speech output, tool use, and a polished GUI, powered by OpenAI's API.
Project Overview
What you'll build: A voice-enabled AI assistant with tools and GUI
Skills practiced: OpenAI API, speech recognition, TTS, tkinter, function calling
Estimated time: 6–8 hours
Installation
pip install openai python-dotenv
pip install pyttsx3 # Text-to-speech
pip install SpeechRecognition # Speech recognition
pip install pyaudio # Microphone access
pip install requests # For tools (weather, etc.)
# .env
OPENAI_API_KEY=your_key_here
OPENWEATHER_API_KEY=your_weather_key_here
Step 1: Voice Engine
# voice_engine.py — Speech recognition and text-to-speech
import speech_recognition as sr
import pyttsx3
import threading
import queue
import os
class VoiceEngine:
"""Handle speech recognition and synthesis."""
def __init__(self, voice_id=None, rate=175, volume=0.9):
# TTS Engine
self.tts = pyttsx3.init()
self.tts.setProperty('rate', rate)
self.tts.setProperty('volume', volume)
# Set voice
voices = self.tts.getProperty('voices')
if voice_id is not None and voice_id < len(voices):
self.tts.setProperty('voice', voices[voice_id].id)
elif voices:
# Try to find a female voice
for voice in voices:
if 'female' in voice.name.lower() or 'zira' in voice.name.lower():
self.tts.setProperty('voice', voice.id)
break
# STT Engine
self.recognizer = sr.Recognizer()
self.recognizer.energy_threshold = 300
self.recognizer.dynamic_energy_threshold = True
self.recognizer.pause_threshold = 0.8
# State
self.is_speaking = False
self.speech_queue = queue.Queue()
self._start_speech_worker()
def _start_speech_worker(self):
"""Background thread for non-blocking speech."""
def worker():
while True:
text = self.speech_queue.get()
if text is None:
break
self.is_speaking = True
self.tts.say(text)
self.tts.runAndWait()
self.is_speaking = False
self.speech_thread = threading.Thread(target=worker, daemon=True)
self.speech_thread.start()
def speak(self, text, block=False):
"""Convert text to speech."""
# Clean text for TTS (remove markdown)
clean = text.replace('**', '').replace('*', '').replace('`', '')
clean = clean.replace('#', '').replace('_', ' ')
if block:
self.is_speaking = True
self.tts.say(clean)
self.tts.runAndWait()
self.is_speaking = False
else:
self.speech_queue.put(clean)
def stop_speaking(self):
"""Stop current speech."""
self.tts.stop()
self.is_speaking = False
def listen(self, timeout=10, phrase_limit=15, language='en-US'):
"""
Listen for voice input.
Returns:
Transcribed text string, or None if nothing heard
"""
with sr.Microphone() as source:
print(" 🎤 Listening...")
self.recognizer.adjust_for_ambient_noise(source, duration=0.5)
try:
audio = self.recognizer.listen(
source,
timeout=timeout,
phrase_time_limit=phrase_limit
)
except sr.WaitTimeoutError:
return None
print(" ⚙️ Processing speech...")
try:
text = self.recognizer.recognize_google(audio, language=language)
print(f" 📝 Recognized: {text}")
return text
except sr.UnknownValueError:
return None
except sr.RequestError as e:
print(f" ❌ Speech service error: {e}")
return None
def list_voices(self):
"""List available TTS voices."""
voices = self.tts.getProperty('voices')
for i, voice in enumerate(voices):
print(f" {i}: {voice.name} ({voice.id})")
return voices
# assistant_tools.py — Functions the AI can call
import json
import math
import datetime
import requests
import os
import subprocess
import platform
# Tool definitions for OpenAI function calling
TOOL_DEFINITIONS = [
{
"type": "function",
"function": {
"name": "get_current_time",
"description": "Get the current date and time",
"parameters": {"type": "object", "properties": {}, "required": []},
}
},
{
"type": "function",
"function": {
"name": "calculate",
"description": "Perform a mathematical calculation",
"parameters": {
"type": "object",
"properties": {
"expression": {"type": "string", "description": "Math expression to evaluate, e.g. '2 + 2', 'sqrt(16)'"}
},
"required": ["expression"]
},
}
},
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get current weather for a city",
"parameters": {
"type": "object",
"properties": {
"city": {"type": "string", "description": "City name, e.g. 'London', 'New York'"}
},
"required": ["city"]
},
}
},
{
"type": "function",
"function": {
"name": "search_wikipedia",
"description": "Search Wikipedia for information about a topic",
"parameters": {
"type": "object",
"properties": {
"query": {"type": "string", "description": "Search query"}
},
"required": ["query"]
},
}
},
{
"type": "function",
"function": {
"name": "set_timer",
"description": "Set a countdown timer",
"parameters": {
"type": "object",
"properties": {
"seconds": {"type": "integer", "description": "Duration in seconds"}
},
"required": ["seconds"]
},
}
},
{
"type": "function",
"function": {
"name": "get_system_info",
"description": "Get information about the current system",
"parameters": {"type": "object", "properties": {}, "required": []},
}
},
]
def get_current_time():
now = datetime.datetime.now()
return {
"time": now.strftime('%I:%M:%S %p'),
"date": now.strftime('%A, %B %d, %Y'),
"weekday": now.strftime('%A'),
"timestamp": now.isoformat(),
}
def calculate(expression):
"""Safely evaluate math expressions."""
try:
# Allow basic math and math functions
safe_env = {
'__builtins__': {},
'sqrt': math.sqrt, 'sin': math.sin, 'cos': math.cos,
'tan': math.tan, 'log': math.log, 'log10': math.log10,
'pi': math.pi, 'e': math.e, 'abs': abs, 'round': round,
'pow': pow, 'factorial': math.factorial,
}
result = eval(expression, safe_env)
return {"expression": expression, "result": result, "formatted": f"{expression} = {result}"}
except Exception as e:
return {"error": str(e), "expression": expression}
def get_weather(city):
"""Get weather from OpenWeatherMap."""
api_key = os.environ.get('OPENWEATHER_API_KEY')
if not api_key:
return {"error": "No weather API key configured"}
try:
url = f"https://api.openweathermap.org/data/2.5/weather"
resp = requests.get(url, params={'q': city, 'appid': api_key, 'units': 'metric'}, timeout=8)
resp.raise_for_status()
data = resp.json()
return {
"city": data['name'],
"country": data['sys']['country'],
"temperature": f"{data['main']['temp']:.1f}°C",
"feels_like": f"{data['main']['feels_like']:.1f}°C",
"description": data['weather'][0]['description'],
"humidity": f"{data['main']['humidity']}%",
"wind_speed": f"{data['wind']['speed'] * 3.6:.1f} km/h",
}
except requests.exceptions.HTTPError:
return {"error": f"City '{city}' not found"}
except Exception as e:
return {"error": str(e)}
def search_wikipedia(query):
"""Fetch Wikipedia summary."""
try:
url = f"https://en.wikipedia.org/api/rest_v1/page/summary/{query.replace(' ', '_')}"
resp = requests.get(url, timeout=8)
if resp.status_code == 200:
data = resp.json()
return {
"title": data.get('title'),
"summary": data.get('extract', '')[:500],
"url": data.get('content_urls', {}).get('desktop', {}).get('page', ''),
}
return {"error": f"No Wikipedia page found for '{query}'"}
except Exception as e:
return {"error": str(e)}
def set_timer(seconds):
"""Set a simple timer (returns info — actual timer runs in background)."""
import threading
def timer_callback():
import time
time.sleep(seconds)
print(f"\n ⏰ Timer finished! ({seconds} seconds)")
t = threading.Thread(target=timer_callback, daemon=True)
t.start()
mins, secs = divmod(seconds, 60)
return {"message": f"Timer set for {mins}m {secs}s", "duration_seconds": seconds}
def get_system_info():
"""Get system information."""
import psutil
try:
return {
"os": platform.system(),
"os_version": platform.version()[:50],
"machine": platform.machine(),
"python_version": platform.python_version(),
"cpu_cores": os.cpu_count(),
"cpu_percent": psutil.cpu_percent(interval=1),
"memory_total_gb": round(psutil.virtual_memory().total / 1e9, 1),
"memory_used_gb": round(psutil.virtual_memory().used / 1e9, 1),
"disk_free_gb": round(psutil.disk_usage('/').free / 1e9, 1),
}
except ImportError:
return {
"os": platform.system(),
"os_version": platform.version()[:50],
"python_version": platform.python_version(),
}
TOOL_FUNCTIONS = {
"get_current_time": get_current_time,
"calculate": calculate,
"get_weather": get_weather,
"search_wikipedia": search_wikipedia,
"set_timer": set_timer,
"get_system_info": get_system_info,
}
Step 3: AI Core
# ai_core.py — OpenAI-powered AI brain
import json
import os
from openai import OpenAI
from assistant_tools import TOOL_DEFINITIONS, TOOL_FUNCTIONS
from dotenv import load_dotenv
load_dotenv()
SYSTEM_PROMPT = """You are Nova, a helpful, witty, and knowledgeable AI assistant.
You have a friendly personality and speak naturally and concisely.
You have access to tools: time, calculator, weather, Wikipedia, timers, and system info.
Use tools proactively when they would help answer the user's question.
Keep responses concise (2-4 sentences) unless asked for detailed explanations.
Always be helpful, accurate, and engaging."""
class NovaAI:
def __init__(self, model="gpt-3.5-turbo"):
self.client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))
self.model = model
self.messages = [{"role": "system", "content": SYSTEM_PROMPT}]
self.max_history = 20
def chat(self, user_message, use_tools=True):
"""Send a message and get a response (with tool use)."""
self.messages.append({"role": "user", "content": user_message})
# Trim history to prevent token overflow
if len(self.messages) > self.max_history + 1:
self.messages = [self.messages[0]] + self.messages[-self.max_history:]
tools = TOOL_DEFINITIONS if use_tools else None
try:
response = self.client.chat.completions.create(
model=self.model,
messages=self.messages,
tools=tools,
tool_choice="auto" if tools else None,
max_tokens=500,
temperature=0.7,
)
msg = response.choices[0].message
# Handle tool calls
if msg.tool_calls:
self.messages.append(msg) # Append assistant's tool call request
for tool_call in msg.tool_calls:
func_name = tool_call.function.name
func_args = json.loads(tool_call.function.arguments)
print(f" 🔧 Using tool: {func_name}({func_args})")
if func_name in TOOL_FUNCTIONS:
result = TOOL_FUNCTIONS[func_name](**func_args)
else:
result = {"error": f"Unknown tool: {func_name}"}
self.messages.append({
"role": "tool",
"tool_call_id": tool_call.id,
"content": json.dumps(result)
})
# Get final response after tool use
final_response = self.client.chat.completions.create(
model=self.model,
messages=self.messages,
max_tokens=500,
temperature=0.7,
)
assistant_text = final_response.choices[0].message.content
else:
assistant_text = msg.content
self.messages.append({"role": "assistant", "content": assistant_text})
return assistant_text
except Exception as e:
error_msg = f"I encountered an error: {str(e)}"
return error_msg
def reset(self):
"""Reset conversation history."""
self.messages = [self.messages[0]]
return "Conversation reset. How can I help you?"
Step 4: Main CLI Interface
# main_cli.py — Text-based CLI for Nova
import os
from ai_core import NovaAI
from voice_engine import VoiceEngine
def run_nova_cli(voice_enabled=False):
print("\n" + "=" * 55)
print(" 🤖 Nova — AI Assistant")
print(" Commands: 'voice', 'reset', 'history', 'quit'")
print("=" * 55)
nova = NovaAI()
voice = VoiceEngine() if voice_enabled else None
# Greeting
greeting = "Hi! I'm Nova, your AI assistant. How can I help you today?"
print(f"\n Nova: {greeting}")
if voice:
voice.speak(greeting)
while True:
# Input
if voice and voice_enabled:
user_input = voice.listen()
if not user_input:
print(" (Nothing heard, please speak clearly)")
continue
else:
user_input = input("\n You: ").strip()
if not user_input:
continue
if user_input.lower() in ['quit', 'exit', 'bye', 'goodbye']:
farewell = "Goodbye! Have a wonderful day!"
print(f"\n Nova: {farewell}")
if voice:
voice.speak(farewell, block=True)
break
if user_input.lower() == 'reset':
msg = nova.reset()
print(f"\n Nova: {msg}")
continue
if user_input.lower() == 'voice':
voice_enabled = not voice_enabled
status = "Voice input enabled!" if voice_enabled else "Voice input disabled."
print(f" ✅ {status}")
continue
if user_input.lower() == 'history':
print(f"\n 📋 Conversation History:")
for msg in nova.messages[1:]:
role = "You" if msg['role'] == 'user' else "Nova"
print(f" {role}: {msg['content'][:80]}...")
continue
# Get AI response
print(" Nova is thinking...", end='\r')
response = nova.chat(user_input)
print(f"\n Nova: {response}")
if voice:
voice.speak(response)
if __name__ == '__main__':
api_key = os.environ.get("OPENAI_API_KEY")
if not api_key:
print("❌ OPENAI_API_KEY not found in .env file!")
print(" Create a .env file and add: OPENAI_API_KEY=your_key")
else:
run_nova_cli(voice_enabled=False)
Step 5: Full tkinter GUI
# nova_gui.py — Full-featured tkinter GUI for Nova
import tkinter as tk
from tkinter import ttk, scrolledtext, messagebox
import threading
import os
from datetime import datetime
class NovaGUI:
def __init__(self, root):
self.root = root
self.root.title("🤖 Nova — AI Assistant")
self.root.geometry("700x750")
self.root.configure(bg='#1a1a2e')
self.nova = None
self.voice = None
self.is_processing = False
self._init_ai()
self._build_ui()
self._send_greeting()
def _init_ai(self):
try:
from ai_core import NovaAI
from voice_engine import VoiceEngine
self.nova = NovaAI()
self.voice = VoiceEngine()
print("✅ Nova AI initialized")
except Exception as e:
messagebox.showerror("Error", f"Could not initialize AI: {e}")
def _build_ui(self):
# Header
header = tk.Frame(self.root, bg='#16213e', pady=10)
header.pack(fill='x')
tk.Label(header, text="🤖 NOVA", font=('Arial', 22, 'bold'),
bg='#16213e', fg='#4fc3f7').pack(side='left', padx=20)
# Status indicator
self.status_dot = tk.Label(header, text="●", font=('Arial', 14),
bg='#16213e', fg='#4CAF50')
self.status_dot.pack(side='right', padx=5)
tk.Label(header, text="Online", font=('Arial', 10), bg='#16213e', fg='#888').pack(side='right')
# Chat area
chat_frame = tk.Frame(self.root, bg='#1a1a2e')
chat_frame.pack(fill='both', expand=True, padx=15, pady=10)
self.chat_display = scrolledtext.ScrolledText(
chat_frame, wrap=tk.WORD, font=('Consolas', 11),
bg='#0f3460', fg='white', relief='flat',
padx=10, pady=10, state='disabled',
insertbackground='white'
)
self.chat_display.pack(fill='both', expand=True)
# Configure tags
self.chat_display.tag_config('user', foreground='#81d4fa', font=('Consolas', 11, 'bold'))
self.chat_display.tag_config('nova', foreground='#a5d6a7', font=('Consolas', 11))
self.chat_display.tag_config('tool', foreground='#ffcc80', font=('Consolas', 10, 'italic'))
self.chat_display.tag_config('time', foreground='#546e7a', font=('Consolas', 9))
self.chat_display.tag_config('error', foreground='#ef9a9a')
# Input area
input_frame = tk.Frame(self.root, bg='#1a1a2e', padx=15, pady=10)
input_frame.pack(fill='x')
self.input_field = tk.Text(input_frame, height=2, font=('Arial', 12),
bg='#0f3460', fg='white', relief='flat',
insertbackground='white', padx=10, pady=8)
self.input_field.pack(side='left', fill='x', expand=True, padx=(0, 10))
self.input_field.bind('<Return>', self._on_enter)
self.input_field.bind('<Shift-Return>', lambda e: None) # Allow newline with Shift+Enter
btn_frame = tk.Frame(input_frame, bg='#1a1a2e')
btn_frame.pack(side='right')
self.send_btn = tk.Button(btn_frame, text='Send', command=self._send_message,
font=('Arial', 11, 'bold'), bg='#1976D2', fg='white',
relief='flat', cursor='hand2', padx=15, pady=5)
self.send_btn.pack(pady=(0, 5))
self.voice_btn = tk.Button(btn_frame, text='🎤', command=self._voice_input,
font=('Arial', 14), bg='#388E3C', fg='white',
relief='flat', cursor='hand2', width=3)
self.voice_btn.pack()
# Quick commands bar
cmd_frame = tk.Frame(self.root, bg='#0f3460', pady=5)
cmd_frame.pack(fill='x')
quick_cmds = [
("⏰ Time", "What time is it?"),
("🌤️ Weather", "What's the weather in London?"),
("📖 Explain", "Explain quantum computing"),
("🔢 Math", "Calculate 15% of 850"),
("ℹ️ System", "Show system info"),
("🗑️ Clear", "reset"),
]
for label, cmd in quick_cmds:
btn = tk.Button(cmd_frame, text=label, command=lambda c=cmd: self._quick_command(c),
font=('Arial', 9), bg='#1565C0', fg='white',
relief='flat', cursor='hand2', padx=8, pady=3)
btn.pack(side='left', padx=3)
def _add_message(self, role, text):
"""Add a message to the chat display."""
self.chat_display.config(state='normal')
timestamp = datetime.now().strftime('%H:%M')
if role == 'user':
self.chat_display.insert(tk.END, f"\n You [{timestamp}]\n ", 'time')
self.chat_display.insert(tk.END, text + "\n", 'user')
elif role == 'nova':
self.chat_display.insert(tk.END, f"\n Nova [{timestamp}]\n ", 'time')
self.chat_display.insert(tk.END, text + "\n", 'nova')
elif role == 'tool':
self.chat_display.insert(tk.END, f" 🔧 {text}\n", 'tool')
self.chat_display.see(tk.END)
self.chat_display.config(state='disabled')
def _send_greeting(self):
greeting = "Hello! I'm Nova, your AI assistant. I can help with questions, calculations, weather, Wikipedia searches, and more! What can I do for you today?"
self._add_message('nova', greeting)
if self.voice:
threading.Thread(target=lambda: self.voice.speak(greeting), daemon=True).start()
def _on_enter(self, event):
if event.state == 0: # No modifier keys
self._send_message()
return 'break'
def _send_message(self):
text = self.input_field.get('1.0', tk.END).strip()
if not text or self.is_processing:
return
self.input_field.delete('1.0', tk.END)
self._add_message('user', text)
self._process_in_background(text)
def _quick_command(self, cmd):
if cmd == 'reset':
if self.nova:
self.nova.reset()
self.chat_display.config(state='normal')
self.chat_display.delete('1.0', tk.END)
self.chat_display.config(state='disabled')
self._send_greeting()
return
self.input_field.delete('1.0', tk.END)
self.input_field.insert('1.0', cmd)
self._send_message()
def _voice_input(self):
if not self.voice:
messagebox.showinfo("Voice", "Install SpeechRecognition and PyAudio for voice input!")
return
self.voice_btn.config(bg='#f44336', text='🔴')
self.root.update()
def listen():
text = self.voice.listen()
self.root.after(0, lambda: self.voice_btn.config(bg='#388E3C', text='🎤'))
if text:
self.root.after(0, lambda: self.input_field.insert('1.0', text))
self.root.after(100, self._send_message)
else:
self.root.after(0, lambda: messagebox.showinfo("Voice", "Nothing heard. Try again."))
threading.Thread(target=listen, daemon=True).start()
def _process_in_background(self, text):
self.is_processing = True
self.send_btn.config(state='disabled', text='...')
self.status_dot.config(fg='#FF9800')
def process():
if self.nova:
response = self.nova.chat(text)
else:
response = "AI not initialized. Check your API key."
self.root.after(0, lambda: self._on_response(response))
threading.Thread(target=process, daemon=True).start()
def _on_response(self, response):
self._add_message('nova', response)
if self.voice:
threading.Thread(target=lambda: self.voice.speak(response), daemon=True).start()
self.is_processing = False
self.send_btn.config(state='normal', text='Send')
self.status_dot.config(fg='#4CAF50')
if __name__ == '__main__':
root = tk.Tk()
app = NovaGUI(root)
root.mainloop()
Summary
In this project, you built:
- ✅ Speech recognition (voice input)
- ✅ Text-to-speech (voice output)
- ✅ OpenAI GPT integration with function calling
- ✅ 6 built-in tools (time, calculator, weather, Wikipedia, timer, system info)
- ✅ CLI and full tkinter GUI
- ✅ Context-aware multi-turn conversation
- ✅ Background threading for non-blocking UI
Key skills practiced:
- OpenAI API with function calling (tools)
- Speech recognition (
SpeechRecognition) - Text-to-speech (
pyttsx3) - Threading for concurrent operations
- tkinter advanced GUI with real-time updates
*Next Section: Resources →*