Displaying Numpy Images in Tooltips#
In this tutorial, you’ll learn how to display images stored as Numpy arrays in tooltips with any Altair chart.
First, we create some example image arrays with blobs (objects) of different sizes and shapes (circular and square). We measure the area of the blobs in order to have a quantitative measurement to compare them with in our charts.
import numpy as np
import pandas as pd
from scipy import ndimage as ndi
rng = np.random.default_rng([ord(c) for c in 'altair'])
n_rows = 200
def create_blobs(blob_shape, img_width=96, n_dim=2, sizes=[0.05, 0.1, 0.15]):
"""Helper function to create blobs in the images"""
shape = tuple([img_width] * n_dim)
mask = np.zeros(shape)
points = (img_width * rng.random(n_dim)).astype(int)
mask[tuple(indices for indices in points)] = 1
if blob_shape == 'circle':
im = ndi.gaussian_filter(mask, sigma=rng.choice(sizes) * img_width)
elif blob_shape == 'square':
im = ndi.uniform_filter(mask, size=rng.choice(sizes) * img_width, mode='constant') * rng.normal(4, size=(img_width, img_width))
return im / im.max()
df = pd.DataFrame({
'image1': [create_blobs('circle') for _ in range(n_rows)],
'image2': [create_blobs('square', sizes=[0.3, 0.4, 0.5]) for _ in range(n_rows)],
'group': rng.choice(['a', 'b', 'c'], size=n_rows)
})
# Compute the area as the proportion of pixels above a threshold
df[['image1_area', 'image2_area']] = df[['image1', 'image2']].map(lambda x: (x > 0.4).mean())
df
image1 image2 group image1_area image2_area
0 [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,... [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,... b 0.030599 0.070855
1 [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,... [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,... b 0.134006 0.167752
2 [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,... [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,... a 0.014865 0.195421
3 [[0.8408188559277084, 0.8434385804894683, 0.84... [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,... c 0.097222 0.208550
4 [[0.0, 0.0, 0.00030006935893488147, 0.00039595... [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,... b 0.072266 0.043294
.. ... ... ... ... ...
195 [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,... [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,... a 0.014865 0.100260
196 [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,... [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,... a 0.040148 0.070747
197 [[0.020642835524827916, 0.021450917851981367, ... [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,... b 0.122830 0.135091
198 [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,... [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,... a 0.064779 0.088542
199 [[0.0, 2.078157310309435e-06, 2.74224551900630... [[0.5236619475835835, 0.446206634299134, 0.398... b 0.130317 0.145942
[200 rows x 5 columns]
Next, we define the function that will convert the Numpy arrays to base64-encoded strings. This is a necessary step for the tooltip to recognize that the data is in the form of an image and render it appropriately.
from io import BytesIO
from PIL import Image, ImageDraw
import base64
def create_tooltip_image(df_row):
"""Concatenate, rescale, and convert images to base64 strings."""
# Concatenate images to show together in the tooltip
# This can be skipped if only one image is to be displayed
img_gap = np.ones([df_row['image1'].shape[0], 10]) # 10 px white gap between imgs
img_arr = np.concatenate(
[
df_row['image1'],
img_gap,
df_row['image2']
],
axis=1
)
# Create a PIL image from the array.
# Multiplying by 255 and recasting as uint8 for the images to occupy the entire supported instensity space from 0-255
img = Image.fromarray((255 * img_arr).astype('uint8'))
# Optional: Burn in labels as pixels in the images. Can be helpful to keep track of which image is which
ImageDraw.Draw(img).text((3, 0), 'im1', fill=255)
ImageDraw.Draw(img).text((3 + df_row['image1'].shape[1] + img_gap.shape[1], 0), 'im2', fill=255)
# Convert to base64 encoded image string that can be displayed in the tooltip
buffered = BytesIO()
img.save(buffered, format="PNG")
img_str = base64.b64encode(buffered.getvalue()).decode()
return f"data:image/png;base64,{img_str}"
# The column with the base64 image string must be called "image" in order for it to trigger the image rendering in the tooltip
df['image'] = df[['image1', 'image2']].apply(create_tooltip_image, axis=1)
# Dropping the image arrays since they are large an no longer needed
df_plot = df.drop(columns=['image1', 'image2'])
df_plot
group image1_area image2_area image
0 b 0.030599 0.070855 data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...
1 b 0.134006 0.167752 data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...
2 a 0.014865 0.195421 data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...
3 c 0.097222 0.208550 data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...
4 b 0.072266 0.043294 data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...
.. ... ... ... ...
195 a 0.014865 0.100260 data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...
196 a 0.040148 0.070747 data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...
197 b 0.122830 0.135091 data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...
198 a 0.064779 0.088542 data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...
199 b 0.130317 0.145942 data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...
[200 rows x 4 columns]
Now we are ready to create the charts that show the images as tooltips when the dots are hovered with the mouse. We can see that the large white blobs correspond to the higher area measurements as expected.
import altair as alt
# The random() function is used to jitter points in the x-direction
alt.Chart(df_plot, width=alt.Step(40)).mark_circle(xOffset=alt.expr('random() * 16 - 8')).encode(
x='group',
y=alt.Y(alt.repeat(), type='quantitative'),
tooltip=['image'],
color='group',
).repeat(
['image1_area', 'image2_area']
).resolve_scale(
y='shared'
).properties(
title='Comparison of blob areas'
)
Note that when including images as part of the chart data, the chart size often increases several-fold. The size of the chart above would have been 19 Kb without the images, but with the images added it is 760 Kb. While this is a 20x size increase, the base64 encoding is still quite storage efficient; if we would have included the images in their original Numpy array format the chart size would have been 35Mb!
If we want to have even more fun and get a bit more sophisticated, we could show one chart at a time and update what is shown on the y-axis as well as what is shown in the image tooltip based on a dropdown selector. We start by defining a tooltip that only contains a single image instead of both the images concatenated together.
def create_tooltip_image(img_arr):
"""Rescale and convert an image to a base64 string."""
# print(img_arr)
# Create a PIL image from the array.
# Multiplying by 255 and recasting as uint8 for the images to occupy the entire supported instensity space from 0-255
img = Image.fromarray((255 * img_arr).astype('uint8'))
# Convert to base64 encoded image string that can be displayed in the tooltip
buffered = BytesIO()
img.save(buffered, format="PNG")
img_str = base64.b64encode(buffered.getvalue()).decode()
return f"data:image/png;base64,{img_str}"
# The column with the base64 image string must be called "image" in order for it to trigger the image rendering in the tooltip
df[['image1_base64', 'image2_base64']] = df[['image1', 'image2']].map(create_tooltip_image)
# Dropping the image arrays since they are large an no longer needed
# Also drop the previous tooltip image for clarity
df_plot = df.drop(columns=['image1', 'image2', 'image'])
df_plot
group image1_area image2_area image1_base64 image2_base64
0 b 0.030599 0.070855 data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA... data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...
1 b 0.134006 0.167752 data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA... data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...
2 a 0.014865 0.195421 data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA... data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...
3 c 0.097222 0.208550 data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA... data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...
4 b 0.072266 0.043294 data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA... data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...
.. ... ... ... ... ...
195 a 0.014865 0.100260 data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA... data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...
196 a 0.040148 0.070747 data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA... data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...
197 b 0.122830 0.135091 data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA... data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...
198 a 0.064779 0.088542 data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA... data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...
199 b 0.130317 0.145942 data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA... data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...
[200 rows x 5 columns]
In our chart, we need to use a transform to update both the y-axis column as well as the tooltip column dynamically based on the selection in the dropdown. The comments in the code explain more in detail what each line in this chart specification does.
metric_dropdown = alt.binding_select(
options=['image1_area', 'image2_area'],
name='Image metric '
)
metric_param = alt.param(
value='image1_area',
bind=metric_dropdown
)
alt.hconcat(
# This first chart is the axis title and is only needed because
# Vega-Lite does not yet support passing an expression directly to the axis title
alt.Chart().mark_text(angle=270, dx=-150, fontWeight='bold').encode(
alt.TextValue(alt.expr(f'{metric_param.name}'))
),
alt.Chart(df_plot, width=alt.Step(40)).mark_circle(xOffset=alt.expr('random() * 16 - 8')).encode(
x='group',
y=alt.Y('image_area:Q').title(''),
tooltip=['image:N'],
color='group',
).properties(
title='Area of blobs'
).transform_calculate(
# This first line updates the image_area which is used for the y axis
# to correspond to the selected string in the dropdown
image_area=f'datum[{metric_param.name}]',
# Since altair needs the tooltip field to be called `image`, we need to dynamically
# change what's in the `image` field depending on the selection in the dropdown
# This is further complicated by the fact that the string in the dropdown is not
# an exact match for the column holding the image data so we need
# to replace part of the name to match to match the corresponding base 64 image field
image=f'datum[replace({metric_param.name}, "_area", "_base64")]',
)
).add_params(
metric_param
)