Code
import numpy as np
import plotly.graph_objects as go
import plotly.express as px
from sklearn import datasets
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
from sklearn.svm import SVC
import gradio as grSupport Vector Machines
1 Introduction
Support Vector Machines (SVM) are a class of supervised learning models used for classification and regression analysis. SVMs are particularly effective for high-dimensional spaces and are versatile due to their ability to use kernel functions for decision boundary clarity. This notebook explores SVMs using Scikit-learn’s SVC for binary classification tasks with different kernel functions.
1.1 SVM Decision Function
Given a set of training data \((x_i, y_i)\) with \(x_i \in \mathbb{R}^d\) and \(y_i \in \{-1,1\}\), the decision function is:
\[ f(x) = \operatorname{sgn}\left(\sum_{i=1}^{n} \alpha_i\, k(x,x_i) + b\right), \]
where \(k(x,x_i)\) is a kernel function (e.g., Gaussian, Polynomial, or Linear). SVM seeks the hyperplane that maximizes the margin between the two classes. We will visualize the decision boundary and highlight the support vectors in this notebook.
2 Python Implementation
Code
def load_dataset(name: str) -> tuple[np.ndarray, np.ndarray]:
"""
Load a dataset from scikit-learn.
Parameters:
name (str): Name of the dataset ('Breast Cancer', 'Digits', 'Iris', 'Wine').
Returns:
tuple[np.ndarray, np.ndarray]: Feature matrix X and label vector y.
"""
if name == "Breast Cancer":
data = datasets.load_breast_cancer()
elif name == "Digits":
data = datasets.load_digits()
elif name == "Iris":
data = datasets.load_iris()
elif name == "Wine":
data = datasets.load_wine()
else:
raise ValueError("Dataset not supported.")
X = data.data
y = data.target
# For multi-class datasets, restrict to the first two classes for binary classification.
if len(np.unique(y)) > 2:
mask = y < 2
X = X[mask]
y = y[mask]
# Map labels: 0 -> -1, 1 -> 1
y = np.where(y == 0, -1, 1)
return X, y
def reduce_dimensionality(X: np.ndarray, n_components: int = 2) -> np.ndarray:
"""
Reduce dimensionality of X using PCA.
Parameters:
X (np.ndarray): Feature matrix.
n_components (int): Number of principal components.
Returns:
np.ndarray: Data reduced to n_components dimensions.
"""
pca = PCA(n_components=n_components)
return pca.fit_transform(X)We define a function to create a Plotly contour plot representing the decision function of the classifier.
It overlays the decision boundary, the training points, and highlights the support vectors.
Code
def plot_decision_boundary(classifier: SVC, X: np.ndarray, y: np.ndarray) -> go.Figure:
"""
Plot the decision boundary for a trained SVC classifier on 2D data.
Parameters:
classifier (SVC): Trained SVC classifier.
X (np.ndarray): 2D feature matrix of shape (n_samples, 2).
y (np.ndarray): Labels vector.
Returns:
go.Figure: Plotly figure with the decision boundary and data points.
"""
x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
xx, yy = np.meshgrid(np.linspace(x_min, x_max, 300), np.linspace(y_min, y_max, 300))
grid = np.c_[xx.ravel(), yy.ravel()]
Z = classifier.decision_function(grid)
Z = Z.reshape(xx.shape)
fig = go.Figure()
# Contour for the decision function (decision boundary at level 0)
fig.add_trace(
go.Contour(
x=np.linspace(x_min, x_max, 300),
y=np.linspace(y_min, y_max, 300),
z=Z,
showscale=False,
colorscale="RdBu",
opacity=0.7,
contours=dict(start=0, end=0, size=0.1, coloring="lines"),
name="Decision Boundary",
)
)
# Scatter plot for training data
fig.add_trace(
go.Scatter(
x=X[:, 0],
y=X[:, 1],
mode="markers",
marker=dict(
color=y, colorscale="RdBu", line=dict(width=1, color="black"), size=8
),
name="Data Points",
)
)
# Highlight support vectors
sv = classifier.support_vectors_
fig.add_trace(
go.Scatter(
x=sv[:, 0],
y=sv[:, 1],
mode="markers",
marker=dict(symbol="x", color="black", size=12, line=dict(width=2)),
name="Support Vectors",
)
)
fig.update_layout(
title="SVM Decision Boundary",
xaxis_title="Feature 1",
yaxis_title="Feature 2",
height=500,
)
return figThe run_kernel_svm function loads the data, preprocesses it, trains the SVM with a selected kernel, and returns the visualization along with key SVM parameters.
Code
def run_kernel_svm(
dataset_name: str, kernel_type: str, gamma: float, degree: int
) -> tuple[go.Figure, dict]:
"""
Run the SVM classification and return the decision boundary plot and classifier parameters.
Parameters:
dataset_name (str): The selected dataset ('Breast Cancer', 'Digits', 'Iris', 'Wine').
kernel_type (str): The type of kernel ('Gaussian', 'Polynomial', 'Linear').
gamma (float): Gamma parameter for the Gaussian kernel.
degree (int): Degree for the Polynomial kernel.
Returns:
Tuple[go.Figure, dict]: A Plotly figure and a JSON-like dict of classifier parameters.
"""
# Load and standardize the dataset
X, y = load_dataset(dataset_name)
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)
# Reduce dimensionality to 2D for visualization if necessary
if X_scaled.shape[1] > 2:
X_reduced = reduce_dimensionality(X_scaled, 2)
else:
X_reduced = X_scaled
# Map kernel type to scikit-learn's SVC kernel string
if kernel_type == "Gaussian":
kernel = "rbf"
elif kernel_type == "Polynomial":
kernel = "poly"
elif kernel_type == "Linear":
kernel = "linear"
else:
raise ValueError("Unsupported kernel type.")
# Create and fit the SVC classifier
svc = SVC(
kernel=kernel,
gamma=gamma if kernel == "rbf" else "scale",
degree=degree if kernel == "poly" else 3,
)
svc.fit(X_reduced, y)
# Generate the decision boundary plot
fig = plot_decision_boundary(svc, X_reduced, y)
# Prepare SVM parameters for JSON output
result_info = {
"dataset": dataset_name,
"kernel": kernel_type,
"n_samples": int(len(y)),
"n_support_vectors": int(len(svc.support_)),
"support_vectors": svc.support_vectors_.tolist(),
"dual_coef": svc.dual_coef_.tolist(),
"intercept": svc.intercept_.tolist(),
}
return fig, result_info3 Interactive Dashboard
The dashboard allows you to adjust the dataset, kernel type, and kernel parameters below to interactively explore the decision boundaries and SVM parameters.
Code
with gr.Blocks(
css="""gradio-app {background: #222222 !important}""",
title="Interactive Support Vector Machine (SVM) Classifier",
) as demo:
dataset = gr.Dropdown(
choices=["Breast Cancer", "Digits", "Iris", "Wine"], label="Dataset"
)
kernel_type = gr.Radio(
choices=["Gaussian", "Polynomial", "Linear"],
label="Kernel Type",
value="Gaussian",
)
gamma = gr.Slider(
minimum=0.01,
maximum=5,
step=0.01,
value=1.0,
label="Gamma (for Gaussian Kernel)",
)
degree = gr.Slider(
minimum=1, maximum=10, step=1, value=3, label="Degree (for Polynomial Kernel)"
)
plot_output = gr.Plot(label="Decision Boundary")
json_output = gr.JSON(label="Classifier Parameters")
kwargs = dict(
fn=run_kernel_svm,
inputs=[dataset, kernel_type, gamma, degree],
outputs=[plot_output, json_output],
)
[component.change(**kwargs) for component in [dataset, kernel_type, gamma, degree]]
demo.load(**kwargs)Code
demo.launch(pwa=True, show_api=False, show_error=True)Code
# Output of this cell set dynamically in Quarto filter step