la unión derecha es el enfoque correcto, pero está utilizando columnas de clicks tabla en la sentencia where. En su lugar, coloque el filtro en on :
SELECT CONCAT(Hour, ':00-', Hour+1, ':00') AS Hours, IFNULL(COUNT(product_id), 0) AS `total_count`
FROM clicks
RIGHT JOIN (
SELECT 0 AS Hour
UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3
UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6
UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9
UNION ALL SELECT 10 UNION ALL SELECT 11 UNION ALL SELECT 12
UNION ALL SELECT 13 UNION ALL SELECT 14 UNION ALL SELECT 15
UNION ALL SELECT 16 UNION ALL SELECT 17 UNION ALL SELECT 18
UNION ALL SELECT 19 UNION ALL SELECT 20 UNION ALL SELECT 21
UNION ALL SELECT 22 UNION ALL SELECT 23
) AS AllHours ON HOUR(clicked_at) = Hour
and ( clicked_at BETWEEN CURRENT_DATE() AND NOW() OR clicked_at IS NULL ) AND clicks.site='awesome-site.com'
GROUP BY Hour
ORDER BY Hour
