scaladoc: support expectations in snippets

docs/_docs/reference/experimental/capture-checking/advanced.md

@@ -94,4 +94,4 @@ By leveraging capability polymorphism, capability members, and path-dependent ca
 * `Label`s store the free capabilities `C` of the `block` passed to `boundary` in their capability member `Fv`.
 * When suspending on a given label, the suspension handler can capture at most the capabilities that occur freely at the `boundary` that introduced the label. That prevents mentioning nested bound labels.
 
-[Back to Capability Polymorphism](polymorphism.md)
+[Back to Capability Polymorphism](polymorphism.md)

docs/_docs/reference/experimental/capture-checking/basics.md

@@ -85,7 +85,7 @@ trait LzyList[+A]:
 object LzyList:
   def apply[T](xs: T*): LzyList[T] = ???
 //}
-val xs = usingLogFile { f =>
+val xs = usingLogFile { f => // error // error
   LzyList(1, 2, 3).map { x => f.write(x); x * x }
 }
 ```
@@ -372,7 +372,7 @@ like this:
 ```scala sc:fail sc-compile-with:logfile-checked
 var loophole: () => Unit = () => ()
 usingLogFile { f =>
-  loophole = () => f.write(0)
+  loophole = () => f.write(0) // error
 }
 loophole()
 ```

docs/_docs/reference/experimental/capture-checking/cc.md

@@ -4,4 +4,4 @@ title: "Capture Checking"
 nightlyOf: https://docs.scala-lang.org/scala3/reference/experimental/capture-checking/index.html
 ---
 
-Capture checking is a research project that modifies the Scala type system to track references to capabilities in values.
+Capture checking is a research project that modifies the Scala type system to track references to capabilities in values.

docs/_docs/reference/experimental/capture-checking/checked-exceptions.md

@@ -73,9 +73,9 @@ As with other capability based schemes, one needs to guard against capabilities
 that are captured in results. For instance, here is a problematic use case:
 ```scala sc:fail sc-compile-with:checked-exceptions-base
 def escaped(xs: Double*): (() => Double) throws LimitExceeded =
-  try () => xs.map(f).sum // error: CanThrow escapes into returned closure
+  try () => xs.map(f).sum
   catch case ex: LimitExceeded => () => -1
-val crasher = escaped(1, 2, 10e+11)
+val crasher = escaped(1, 2, 10e+11) // error: CanThrow escapes into returned closure
 crasher()
 ```
 This code needs to be rejected since otherwise the call to `crasher()` would cause

docs/_docs/reference/experimental/capture-checking/internals.md

@@ -80,5 +80,3 @@ This section lists all variables that appeared in previous diagnostics and their
  - variable `31` has a constant fixed superset `{xs, f}`
  - variable `32` has no dependencies.
 
-
-

docs/_docs/reference/experimental/capture-checking/scoped-capabilities.md

@@ -142,7 +142,7 @@ If any scope refuses to absorb the capability, capture checking fails:
 
 ```scala sc:fail sc-compile-with:scoped-fs-context
 def process(fs: FileSystem^): Unit =
-  val f: () -> Unit = () => fs.read()  // Error: fs cannot flow into {}
+  val f: () -> Unit = () => fs.read()  // error: fs cannot flow into {}
 ```
 
 The closure is declared pure (`() -> Unit`), meaning its local `any` is the empty set. The
@@ -312,14 +312,14 @@ determines the binding structure automatically from where `fresh` appears in the
 
 The rules above establish a key practical distinction when writing function types. Consider:
 
-```scala sc:fail
+```scala sc:fail sc-compile-with:scoped-cc-context
 import caps.fresh
 class A
 class B
 
 def test(): Unit =
-  val f: (x: A^) -> B^{fresh} = ???   // B^{fresh}: existentially bound
-  val g: A^ -> B^             = ???   // B^{any}: enclosing scope's local any
+  val f: (x: A^) -> B^{fresh} = ???  // B^{fresh}: existentially bound
+  val g: A^ -> B^             = ???  // B^{any}: enclosing scope's local any
 
   val _: A^ -> B^        = f   // error: fresh is not in {any}
   val _: A^ -> B^{fresh} = f   // ok
@@ -390,7 +390,7 @@ directly returning a closure that captures it:
 ```scala sc:fail sc-compile-with:scoped-withfile-context
 withFile[() => File^]("test.txt"): f =>
 //       ^^^^^^^^^^^ T = () => File^, i.e., () ->{any} File^{any} for some outer any
-  () => f  // error: We want to return this as () => File^
+  () => f  // error // error // error: We want to return this as () => File^
 ```
 
 The lambda `(f: File^) => () => f` has inferred type:
@@ -414,7 +414,7 @@ into this outer `any`, so the assignment fails.
 Otherwise, allowing widening `∃fresh. () ->{fresh} File^{fresh}` to `() => File^` would let the scoped file escape:
 
 ```scala sc:fail sc-compile-with:scoped-withfile-context
-val escaped: () => File^ = withFile[() => File^]("test.txt")(f => () => f)
+val escaped: () => File^ = withFile[() => File^]("test.txt")(f => () => f) // error // error
 //           ^^^^^^^^^^^ any here is in the outer scope
 escaped().read()  // Use-after-close!
 ```

project/Build.scala

@@ -2137,9 +2137,7 @@ object Build {
               .add(NoLinkWarnings(true))
               .add(NoLinkAssetWarnings(true))
               .add(GenerateAPI(false))
-              .add(SnippetCompiler(List(
-                s"${tempDocsRoot.getAbsolutePath}=compile"
-              )))
+              .add(SnippetCompiler(referenceSnippetCompilerTargets(tempRoot.getAbsolutePath)))
           }
 
           generateDocumentation(config)
@@ -3071,8 +3069,14 @@ object ScaladocConfigs {
     "enums",
     "experimental/capture-checking",
   )
+  def captureCheckingSnippetTestTargets(docsRoot: String) = List(
+    s"$docsRoot/_docs/reference/experimental/capture-checking/basics.md=compile+test",
+    s"$docsRoot/_docs/reference/experimental/capture-checking/checked-exceptions.md=compile+test",
+    s"$docsRoot/_docs/reference/experimental/capture-checking/scoped-capabilities.md=compile+test"
+  )
   def referenceSnippetCompilerTargets(docsRoot: String) =
-    referenceSnippetRelativeRoots.map(path => s"$docsRoot/_docs/reference/$path=compile")
+    referenceSnippetRelativeRoots.map(path => s"$docsRoot/_docs/reference/$path=compile") ++
+      captureCheckingSnippetTestTargets(docsRoot)
 
   lazy val Scala3 = Def.task {
     val stdlib = { // relative path to the stdlib directory ('library/')

scaladoc/src/dotty/tools/scaladoc/site/templates.scala

@@ -76,24 +76,24 @@ case class TemplateFile(
   def isIndexPage() = file.isFile && (file.getName == "index.md" || file.getName == "index.html")
 
   private[site] def resolveInner(ctx: RenderingContext)(using ssctx: StaticSiteContext): ResolvedPage =
-
     lazy val snippetCheckingFunc: SnippetChecker.SnippetCheckingFunc =
       val path = Some(Paths.get(file.getAbsolutePath))
       val pathBasedArg = ssctx.snippetCompilerArgs.get(path)
       val sourceFile = dotty.tools.dotc.util.SourceFile(dotty.tools.io.AbstractFile.getFile(path.get), scala.io.Codec.UTF8)
-      (snippet: SnippetSource, argOverride: Option[SnippetCompilerArg]) => {
-          val arg = argOverride.fold(pathBasedArg)(pathBasedArg.merge(_))
-          val compilerData = SnippetCompilerData(
-            "staticsitesnippet",
-            SnippetCompilerData.Position(configOffset - 1, 0)
-          )
-          ssctx.snippetChecker.checkSnippet(snippet, Some(compilerData), arg, sourceFile, 0).collect {
-              case r: SnippetCompilationResult if !r.isSuccessful =>
-                ssctx.bufferSnippetMessages(r.messages)
-                r
-              case r => r
-          }
-      }
+      (snippet: SnippetSource, argOverride: Option[SnippetCompilerArg]) =>
+        val arg = argOverride.fold(pathBasedArg)(pathBasedArg.merge(_))
+        val compilerData = SnippetCompilerData("staticsitesnippet", SnippetCompilerData.Position(configOffset - 1, 0))
+        val result = ssctx.snippetChecker.checkSnippet(
+          snippet,
+          Some(compilerData),
+          arg,
+          sourceFile,
+          0
+        )
+        result.foreach: r =>
+          if !r.isSuccessful then
+            ssctx.bufferSnippetMessages(r.messages)
+        result
 
     if (ctx.resolving.contains(file.getAbsolutePath))
       throw new RuntimeException(s"Cycle in templates involving $file: ${ctx.resolving}")

scaladoc/src/dotty/tools/scaladoc/snippets/FlexmarkSnippetProcessor.scala

@@ -3,7 +3,6 @@ package snippets
 
 import com.vladsch.flexmark.util.{ast => mdu, sequence}
 import com.vladsch.flexmark.{ast => mda}
-import com.vladsch.flexmark.formatter.Formatter
 import scala.jdk.CollectionConverters._
 
 import dotty.tools.scaladoc.tasty.comments.markdown.ExtendedFencedCodeBlock
@@ -20,7 +19,7 @@ object FlexmarkSnippetProcessor:
     nodes.foldLeft[Map[String, SnippetSource]](Map()) { (snippetMap, node) =>
       val lineOffset = node.getStartLineNumber + preparsed.fold(0)(_.strippedLinesBeforeNo)
       val codeStartLine = lineOffset + SnippetChecker.codeFenceContentLineOffset
-      val info = node.getInfo.toString.split(" ")
+      val info = node.getInfo.toString.split(" ").filter(_.nonEmpty)
       if info.contains("scala") then {
         val flagOverride = info
           .find(_.startsWith("sc:"))

scaladoc/src/dotty/tools/scaladoc/snippets/SnippetChecker.scala

@@ -1,15 +1,10 @@
 package dotty.tools.scaladoc
 package snippets
 
-import dotty.tools.scaladoc.DocContext
-import java.nio.file.Paths
-import java.io.File
-
+import dotty.tools.dotc.config.Settings._
+import dotty.tools.dotc.fromtasty.TastyFileUtil
 import dotty.tools.dotc.util.SourceFile
 import dotty.tools.io.AbstractFile
-import dotty.tools.dotc.fromtasty.TastyFileUtil
-import dotty.tools.dotc.config.Settings._
-import dotty.tools.dotc.config.ScalaSettings
 
 class SnippetChecker(val args: Scaladoc.Args)(using cctx: CompilerContext):
   private val sep = System.getProperty("path.separator")
@@ -18,15 +13,19 @@ class SnippetChecker(val args: Scaladoc.Args)(using cctx: CompilerContext):
     args.tastyFiles
       .map(_.getAbsolutePath())
       .map(AbstractFile.getFile(_))
-      .flatMap(t => try { TastyFileUtil.getClassPath(t) } catch { case e: AssertionError => Seq() })
-      .distinct.mkString(sep),
+      .flatMap(t => try TastyFileUtil.getClassPath(t) catch case _: AssertionError => Seq.empty)
+      .distinct
+      .mkString(sep),
     args.classpath
   ).mkString(sep)
 
-  private val snippetCompilerSettings: Seq[SnippetCompilerSetting[?]] = cctx.settings.userSetSettings(cctx.settingsState).filter(_ != cctx.settings.classpath)
-  .map[SnippetCompilerSetting[?]]( s =>
-    SnippetCompilerSetting(s, s.valueIn(cctx.settingsState))
-  ) :+ SnippetCompilerSetting(cctx.settings.classpath, fullClasspath)
+  private val snippetCompilerSettings: Seq[SnippetCompilerSetting[?]] =
+    val userSetSettings =
+      cctx.settings.userSetSettings(cctx.settingsState)
+        .filter(_ != cctx.settings.classpath)
+        .map[SnippetCompilerSetting[?]]: setting =>
+          SnippetCompilerSetting(setting, setting.valueIn(cctx.settingsState))
+    userSetSettings :+ SnippetCompilerSetting(cctx.settings.classpath, fullClasspath)
 
   private val compiler: SnippetCompiler = SnippetCompiler(snippetCompilerSettings = snippetCompilerSettings)
 
@@ -36,23 +35,31 @@ class SnippetChecker(val args: Scaladoc.Args)(using cctx: CompilerContext):
     arg: SnippetCompilerArg,
     sourceFile: SourceFile,
     sourceColumnOffset: Int
-  ): Option[SnippetCompilationResult] = {
+  ): Option[SnippetCompilationResult] =
     if arg.flag != SCFlags.NoCompile then
       val baseLineOffset = data.fold(0)(_.position.line)
       val baseColumnOffset = data.fold(0)(_.position.column) + sourceColumnOffset
+      val sourceLines = snippet.sourceLines.map(_.map(_ + baseLineOffset))
+      val adjustedSnippet = snippet.copy(
+        sourceLines = sourceLines,
+        outerLineOffset = snippet.outerLineOffset + baseLineOffset
+      )
       val wrapped = WrappedSnippet(
         snippet.snippet,
         data.map(_.packageName),
         snippet.outerLineOffset + baseLineOffset,
         baseColumnOffset,
-        snippet.sourceLines.map(_.map(_ + baseLineOffset))
+        sourceLines
       )
-      Some(compiler.compile(wrapped, arg, sourceFile))
+      Some(compiler.compile(
+        adjustedSnippet,
+        wrapped,
+        arg,
+        sourceFile
+      ))
     else
       None
 
-  }
-
 object SnippetChecker:
   // The first line of snippet content is two lines below the opening code fence.
   val codeFenceContentLineOffset = 2

scaladoc/src/dotty/tools/scaladoc/snippets/SnippetCompilationResult.scala

@@ -8,12 +8,13 @@ case class Position(srcPos: SourcePosition, relativeLine: Int)
 
 case class SnippetCompilerMessage(position: Option[Position], message: String, level: MessageLevel):
   def emit()(using CompilerContext): Unit =
+    val ctx = position.fold(summon[CompilerContext])(pos => summon[CompilerContext].withSource(pos.srcPos.source))
     val pos: SrcPos = position.fold(dotty.tools.dotc.util.NoSourcePosition)(_.srcPos)
     level match
-      case MessageLevel.Info => report.log(message, pos)
-      case MessageLevel.Warning => report.warning(message, pos)
-      case MessageLevel.Error => report.error(message, pos)
-      case MessageLevel.Debug => report.log(message, pos)
+      case MessageLevel.Info => report.log(message, pos)(using ctx)
+      case MessageLevel.Warning => report.warning(message, pos)(using ctx)
+      case MessageLevel.Error => report.error(message, pos)(using ctx)
+      case MessageLevel.Debug => report.log(message, pos)(using ctx)
 
 case class SnippetCompilationResult(
   wrappedSnippet: WrappedSnippet,

scaladoc/src/dotty/tools/scaladoc/snippets/SnippetCompiler.scala

@@ -7,19 +7,10 @@ import dotty.tools.dotc.core.Contexts.Context
 import dotty.tools.dotc.core.Mode
 import dotty.tools.dotc.core.MacroClassLoader
 import dotty.tools.dotc.config.Settings.Setting._
-import dotty.tools.dotc.interfaces.{ SourcePosition => ISourcePosition }
-import dotty.tools.dotc.ast.Trees.Tree
-import dotty.tools.dotc.interfaces.{SourceFile => ISourceFile}
-import dotty.tools.dotc.reporting.{ Diagnostic, StoreReporter }
-import dotty.tools.dotc.parsing.Parsers.Parser
+import dotty.tools.dotc.reporting.StoreReporter
 import dotty.tools.dotc.{ Compiler, Run }
-import dotty.tools.io.{AbstractFile, VirtualDirectory}
-import dotty.tools.io.AbstractFileClassLoader
-import dotty.tools.dotc.util.Spans._
-import dotty.tools.dotc.interfaces.Diagnostic._
-import dotty.tools.dotc.util.{ SourcePosition, NoSourcePosition, SourceFile }
-
-import scala.util.{ Try, Success, Failure }
+import dotty.tools.dotc.util.{SourceFile, SourcePosition}
+import dotty.tools.dotc.util.Spans.NoSpan
 
 class SnippetCompiler(
   val snippetCompilerSettings: Seq[SnippetCompilerSetting[?]],
@@ -49,32 +40,12 @@ class SnippetCompiler(
 
   private def newRun(using ctx: Context): Run = scala3Compiler.newRun
 
-  private def nullableMessage(msgOrNull: String | Null): String =
-    if (msgOrNull == null) "" else msgOrNull
-
-  private def createReportMessage(wrappedSnippet: WrappedSnippet, diagnostics: Seq[Diagnostic], sourceFile: SourceFile): Seq[SnippetCompilerMessage] = {
-    val infos = diagnostics.toSeq.sortBy(_.pos.source.path)
-    val errorMessages = infos.map {
-      case diagnostic if diagnostic.position.isPresent =>
-        val diagPos = diagnostic.position.get match
-          case s: SourcePosition => s
-          case _ => NoSourcePosition
-        val pos = wrappedSnippet.sourcePosition(diagPos, sourceFile)
-        val dmsg = Try(diagnostic.message) match {
-          case Success(msg) => msg
-          case Failure(ex) => ex.getMessage
-        }
-        val msg = nullableMessage(dmsg)
-        val level = MessageLevel.fromOrdinal(diagnostic.level)
-        SnippetCompilerMessage(pos, msg, level)
-      case d =>
-        val level = MessageLevel.fromOrdinal(d.level)
-        SnippetCompilerMessage(None, nullableMessage(d.message), level)
-    }
-    errorMessages
-  }
-
-  private def additionalMessages(wrappedSnippet: WrappedSnippet, arg: SnippetCompilerArg, sourceFile: SourceFile, context: Context): Seq[SnippetCompilerMessage] = {
+  private def additionalMessages(
+    wrappedSnippet: WrappedSnippet,
+    arg: SnippetCompilerArg,
+    sourceFile: SourceFile,
+    context: Context
+  ): Seq[SnippetCompilerMessage] = {
       Option.when(arg.flag == SCFlags.Fail && !context.reporter.hasErrors)(
         SnippetCompilerMessage(
           Some(Position(SourcePosition(sourceFile, NoSpan), wrappedSnippet.outerLineOffset)),
@@ -84,10 +55,16 @@ class SnippetCompiler(
 
   private def isSuccessful(arg: SnippetCompilerArg, context: Context): Boolean = {
     if arg.flag == SCFlags.Fail then context.reporter.hasErrors
-    else !context.reporter.hasErrors
+      else !context.reporter.hasErrors
   }
 
+  private def missingExpectedErrorsMessage(arg: SnippetCompilerArg): Seq[SnippetCompilerMessage] =
+    Option.when(arg.flag == SCFlags.Fail)(
+      SnippetCompilerMessage(None, "No errors found when compiling snippet", MessageLevel.Error)
+    ).toList
+
   def compile(
+    snippet: SnippetSource,
     wrappedSnippet: WrappedSnippet,
     arg: SnippetCompilerArg,
     sourceFile: SourceFile
@@ -108,10 +85,35 @@ class SnippetCompiler(
     val run = newRun(using context)
     run.compileFromStrings(List(wrappedSnippet.snippet))
 
+    val diagnostics = context.reporter.pendingMessages(using context)
+    val observed = SnippetExpectations.observe(diagnostics, wrappedSnippet, sourceFile)
+    val shouldVerifyDiagnostics = arg.verifyDiagnostics
+    val expected =
+      if shouldVerifyDiagnostics then SnippetExpectations.parse(snippet, sourceFile)
+      else SnippetExpectations.Parsed(Nil, Nil)
+    val diagnosticMessages =
+      if shouldVerifyDiagnostics then SnippetExpectations.validate(expected, observed, sourceFile)
+      else Nil
+    val failMessages =
+      if shouldVerifyDiagnostics && expected.expectedErrors == 0 && !context.reporter.hasErrors then
+        missingExpectedErrorsMessage(arg)
+      else Nil
+    val compatibilityMessages =
+      if !shouldVerifyDiagnostics then
+        additionalMessages(wrappedSnippet, arg, sourceFile, context)
+      else Nil
+    val validationMessages = diagnosticMessages ++ failMessages ++ compatibilityMessages
+    val expectationDriven = shouldVerifyDiagnostics
+    val hasMismatches = validationMessages.exists(_.level == MessageLevel.Error)
     val messages =
-      createReportMessage(wrappedSnippet, context.reporter.pendingMessages(using context), sourceFile) ++
-      additionalMessages(wrappedSnippet, arg, sourceFile, context)
+      if expectationDriven && hasMismatches then validationMessages
+      else observed.map(_.message) ++ validationMessages
+    val succeeded =
+      if expectationDriven then
+        !hasMismatches
+          && (arg.flag != SCFlags.Fail || context.reporter.hasErrors || expected.expectedErrors > 0)
+      else isSuccessful(arg, context)
 
     val t = Option.when(!context.reporter.hasErrors)(target)
-    SnippetCompilationResult(wrappedSnippet, isSuccessful(arg, context), t, messages)
+    SnippetCompilationResult(wrappedSnippet, succeeded, t, messages)
   }