diff --git a/skorch/callbacks/scoring.py b/skorch/callbacks/scoring.py
index 23c1782c1..4700fa999 100644
--- a/skorch/callbacks/scoring.py
+++ b/skorch/callbacks/scoring.py
@@ -100,7 +100,7 @@ def convert_sklearn_metric_function(scoring):
 
         # those are scoring objects returned by make_scorer starting
         # from sklearn 0.22
-        scorer_names = ('_PredictScorer', '_ProbaScorer', '_ThresholdScorer')
+        scorer_names = ('_PredictScorer', '_ProbaScorer', '_ThresholdScorer', '_Scorer')
         if (
                 hasattr(module, 'startswith') and
                 module.startswith('sklearn.metrics.') and
diff --git a/skorch/classifier.py b/skorch/classifier.py
index b7f552eaf..82707367b 100644
--- a/skorch/classifier.py
+++ b/skorch/classifier.py
@@ -97,7 +97,7 @@ def classes_(self):
             if not len(self.classes):
                 raise AttributeError("{} has no attribute 'classes_'".format(
                     self.__class__.__name__))
-            return self.classes
+            return np.asarray(self.classes)
 
         try:
             return self.classes_inferred_
@@ -301,7 +301,7 @@ def _default_callbacks(self):
 
     @property
     def classes_(self):
-        return [0, 1]
+        return np.array([0, 1])
 
     # pylint: disable=signature-differs
     def check_data(self, X, y):
diff --git a/skorch/tests/test_classifier.py b/skorch/tests/test_classifier.py
index b8f6b0235..1d0c74164 100644
--- a/skorch/tests/test_classifier.py
+++ b/skorch/tests/test_classifier.py
@@ -134,7 +134,7 @@ def test_classes_with_gaps(self, net_cls, module_cls, data):
 
     def test_pass_classes_explicitly_overrides(self, net_cls, module_cls, data):
         net = net_cls(module_cls, max_epochs=0, classes=['foo', 'bar']).fit(*data)
-        assert net.classes_ == ['foo', 'bar']
+        assert (net.classes_ == np.array(['foo', 'bar'])).all()
 
     def test_classes_are_set_with_tensordataset_explicit_y(
             self, net_cls, module_cls, data
@@ -172,7 +172,12 @@ def test_pass_empty_classes_raises(
         expected = "NeuralNetClassifier has no attribute 'classes_'"
         assert msg == expected
 
+    @pytest.mark.xfail
     def test_with_calibrated_classifier_cv(self, net_fit, data):
+        # TODO: This fails with sklearn 1.4.0 because CCCV does not work when
+        # y_proba is float32. This will be fixed in
+        # https://github.com/scikit-learn/scikit-learn/pull/28247, at which
+        # point the test should pass again and the xfail can be removed.
         from sklearn.calibration import CalibratedClassifierCV
         cccv = CalibratedClassifierCV(net_fit, cv=2)
         cccv.fit(*data)
@@ -376,7 +381,12 @@ def test_default_loss_does_call_sigmoid(
         net.predict_proba(X)
         assert mock.call_count > 0
 
+    @pytest.mark.xfail
     def test_with_calibrated_classifier_cv(self, net_fit, data):
+        # TODO: This fails with sklearn 1.4.0 because CCCV does not work when
+        # y_proba is float32. This will be fixed in
+        # https://github.com/scikit-learn/scikit-learn/pull/28247, at which
+        # point the test should pass again and the xfail can be removed.
         from sklearn.calibration import CalibratedClassifierCV
         cccv = CalibratedClassifierCV(net_fit, cv=2)
         cccv.fit(*data)