Puedes probar de esta manera..
WHERE
Vendor_Registration.Category_ID = 5
AND Vendor_PrimaryInfo.City = 'City'
AND (value_text in ('sample value') or
(CASE WHEN (ISNUMERIC(value_text) = 1)
THEN CAST(value_text AS INT)
ELSE -1
END) BETWEEN 0 AND 100)