Improve TypeVar instantiation logic (#24697)

This PR refines the `TypeVar` instantiation logic introduced by #24231.

We only check the `calleeType` when there is actually a `TypeVar` in
`pt` to instantiate now, prioritizing inference from the callee.

Close #24686 #24689 #24694 #24696

Maybe #24695, since the behaviour is the same as 3.3 and 3.7 now.

`calleeType` itself still has some issues, will fix in a separate PR:
#24732

Author: noti0na1

compiler/src/dotty/tools/dotc/typer/Typer.scala

@@ -1933,19 +1933,20 @@ class Typer(@constructorOnly nestingLevel: Int = 0) extends Namer
         NoType
     }
 
-    /** Try to instantiate one type variable bounded by function types that appear
+    /** Find one instantiatable type variable bounded by function types that appear
      *  deeply inside `tp`, including union or intersection types.
      */
-    def tryToInstantiateDeeply(tp: Type): Boolean = tp.dealias match
+    def instantiatableTypeVar(tp: Type): Type = tp.dealias match
       case tp: AndOrType =>
-        tryToInstantiateDeeply(tp.tp1)
-        || tryToInstantiateDeeply(tp.tp2)
+        val t1 = instantiatableTypeVar(tp.tp1)
+        if t1.exists then t1
+        else instantiatableTypeVar(tp.tp2)
       case tp: FlexibleType =>
-        tryToInstantiateDeeply(tp.hi)
+        instantiatableTypeVar(tp.hi)
       case tp: TypeVar if isConstrainedByFunctionType(tp) =>
         // Only instantiate if the type variable is constrained by function types
-        isFullyDefined(tp, ForceDegree.flipBottom)
-      case _ => false
+        tp
+      case _ => NoType
 
     def isConstrainedByFunctionType(tvar: TypeVar): Boolean =
       val origin = tvar.origin
@@ -1961,16 +1962,18 @@ class Typer(@constructorOnly nestingLevel: Int = 0) extends Namer
         case _ => false
       containsFunctionType(bounds.lo) || containsFunctionType(bounds.hi)
 
-    if untpd.isFunctionWithUnknownParamType(tree) && !calleeType.exists then
+    if untpd.isFunctionWithUnknownParamType(tree) then
       // Try to instantiate `pt` when possible.
       // * If `pt` is a type variable, we try to instantiate it directly.
       // * If `pt` is a more complex type, we try to instantiate it deeply by searching
       //   a nested type variable bounded by a function type to help infer parameter types.
       // If it does not work the error will be reported later in `inferredParam`,
       // when we try to infer the parameter type.
-      pt match
-        case pt: TypeVar => isFullyDefined(pt, ForceDegree.flipBottom)
-        case _ => tryToInstantiateDeeply(pt)
+      // Note: we only check the `calleeType` if there is a TypeVar to instantiate to
+      // prioritize inferring from the callee.
+      val tp = if pt.isInstanceOf[TypeVar] then pt else instantiatableTypeVar(pt)
+      if tp.exists && !calleeType.exists then
+        isFullyDefined(tp, ForceDegree.flipBottom)
 
     val (protoFormals, resultTpt) = decomposeProtoFunction(pt, params.length, tree.srcPos)
 

tests/pos/i12679.scala

@@ -7,3 +7,11 @@ object Example:
 
   def example[F[_]](maybeQux: Option[String], bool: Boolean) =
     maybeQux.fold(foo[F](bool))(foo[F](_))
+
+object Example2:
+  def foo(qux: String, quux: String = ""): Unit = ???
+
+  def foo(qux: Boolean): Unit = ???
+
+  def example(maybeQux: Option[String], bool: Boolean) =
+    maybeQux.fold(foo(bool))(s => foo(s))

tests/pos/i24686.scala

@@ -0,0 +1,20 @@
+def Test = Seq.empty[DenseMatrix[Double]].reduce(DenseMatrix.horzcat(_, _))
+
+trait Matrix[T]
+trait DenseMatrix[T] extends Matrix[T]
+
+object DenseMatrix:
+    def horzcat[M, V](matrices: M*)(using OpSet.InPlaceImpl2[DenseMatrix[V], M]): DenseMatrix[V] = ???
+
+object OpSet extends HasOps:
+  trait InPlaceImpl2[V1, V2]
+
+trait HasOps
+object HasOps extends DenseMatrixExpandedOps with DensMatrixLowPriority
+
+trait DenseMatrixExpandedOps:
+  given OpSet.InPlaceImpl2[DenseMatrix[Double], DenseMatrix[Double]] = ???
+
+trait DensMatrixLowPriority extends LowPriorityDenseMatrix1
+trait LowPriorityDenseMatrix1:
+  given [V]: OpSet.InPlaceImpl2[DenseMatrix[V], Matrix[V]] = ???

tests/pos/i24689a.scala

@@ -0,0 +1,13 @@
+import java.io.*
+import java.nio.file.*
+
+abstract class Using[Source, A]:
+  def apply[R](src: Source)(f: A => R): R = ???
+
+object Using:
+  val fileInputStream: Using[Path, InputStream] = ???
+
+def transfer(in: Path, out: OutputStream): Unit =
+  Using.fileInputStream(in)(in => transfer(in, out))
+
+def transfer(in: InputStream, out: OutputStream): Unit = ???

tests/pos/i24689b.scala

@@ -0,0 +1,19 @@
+trait A
+trait B
+
+object Test1:
+  def foo(f: B => Unit) = ???
+  def transfer(in: A): Unit =
+    foo(in => transfer(in))
+    foo(transfer)
+    foo(transfer(_))
+  def transfer(in: B): Unit = ???
+
+// TODO: need to fix callee type in typedFunctionValue
+// object Test2:
+//   def foo[T <: (B => Unit)](f: T) = ???
+//   def transfer(in: A): Unit =
+//     foo(in => transfer(in))
+//     foo(transfer)
+//     foo(transfer(_))
+//   def transfer(in: B): Unit = ???

tests/pos/i24694/A_1.scala

@@ -0,0 +1,17 @@
+trait Show[T]
+object Show:
+  given [A: Show]: Show[List[A]] = ???
+
+trait Eq[A] extends Any, Serializable
+object Eq:
+  def fromUniversalEquals[A]: Eq[A] = ???
+
+object expect:
+  def same[A](expected: A, found: A)(using
+      eqA: Eq[A] = Eq.fromUniversalEquals[A],
+      showA: Show[A]
+  ): Unit = ???
+
+sealed trait XmlEvent
+object XmlEvent:
+  given Show[XmlEvent] = ???

tests/pos/i24694/B_2.scala

@@ -0,0 +1,5 @@
+def test(input: Option[List[XmlEvent]]) =
+  val works =
+    expect.same(List.empty[XmlEvent], input.get)
+  val fails = input.map: tokens =>
+    expect.same(List.empty[XmlEvent], tokens)

tests/pos/i24696.scala

@@ -0,0 +1,18 @@
+trait Schema[A]
+object Schema:
+    final case class Either[A, B](left: Schema[A], right: Schema[B]) extends Schema[scala.util.Either[A, B]]
+trait DecodeError
+
+object DynamicValue:
+  final case class LeftValue(value: DynamicValue) extends DynamicValue
+  final case class RightValue(value: DynamicValue) extends DynamicValue
+
+sealed trait DynamicValue:
+  self =>
+    def toTypedValueLazyError[A](using schema: Schema[A]): Either[DecodeError, A] =
+      (self, schema) match
+        case (DynamicValue.LeftValue(value), Schema.Either(schema1, _)) =>
+          value.toTypedValueLazyError(using schema1).map(Left(_))
+        case (DynamicValue.RightValue(value), Schema.Either(_, schema1)) =>
+          value.toTypedValueLazyError(using schema1).map(Right(_))
+        case _ => ???