Update Jelly-JVM, add jelly as in/out format

README.md

@@ -78,6 +78,12 @@ The `rdf transcode` command turns one or more input Jelly streams into a single
 jelly-cli rdf transcode input.jelly > output.jelly
 ```
 
+If you need other options than that, use `rdf to-jelly` with Jelly input instead. It fully decodes and re-encodes the stream, so it's slower, but you can set any `--opt.*` options. The same works for Jelly-SPARQL with `sparql to-jelly --in-format jelly-sparql`.
+
+```shell
+jelly-cli rdf to-jelly input.jelly --opt.max-name-table-size=256 > output.jelly
+```
+
 ### Inspect Jelly files
 
 To inspect a Jelly file and get basic information describing its contents, such as stream options or number of triples in the file, run:

build.sbt

@@ -11,7 +11,7 @@ resolvers +=
   "Sonatype OSS Snapshots" at "https://central.sonatype.com/repository/maven-snapshots"
 
 lazy val jenaV = "6.2.0"
-lazy val jellyV = "3.7.3+76-b127ba1c-SNAPSHOT"
+lazy val jellyV = "3.7.3+80-0f51136f-SNAPSHOT"
 lazy val graalvmV = "25.4.4.1.1"
 
 addCommandAlias("fixAll", "scalafixAll; scalafmtAll")

src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfFromJelly.scala

@@ -6,6 +6,7 @@ import eu.neverblink.jelly.cli.command.rdf.util.*
 import eu.neverblink.jelly.cli.command.rdf.util.RdfFormat.*
 import eu.neverblink.jelly.cli.util.args.IndexRange
 import eu.neverblink.jelly.cli.util.io.ProtoText
+import eu.neverblink.jelly.cli.util.jena.riot.JellyWriterUtil
 import eu.neverblink.jelly.cli.util.jena.{
   JenaSystemOptions,
   RdfSparqlConverter,
@@ -107,6 +108,7 @@ object RdfFromJelly extends RdfSerDesCommand[RdfFromJellyOptions, RdfFormat.Writ
       format: RdfFormat.Writeable,
   ): Option[WriteAction] =
     (format, getOptions.combine) match
+      case (RdfFormat.JellyBinary, _) => Some(jellyToJelly)
       case (j: RdfFormat.Jena.StreamWriteable, _) =>
         Some((in, out, opt) =>
           jellyToLang(in, StreamRDFWriter.getWriterStream(out, j.jenaLang), j, opt),
@@ -122,6 +124,24 @@ object RdfFromJelly extends RdfSerDesCommand[RdfFromJellyOptions, RdfFormat.Writ
       case (RdfFormat.JellyText, _) => Some(jellyBinaryToText)
       case (RdfFormat.JellySparql, _) => Some(jellyToSparql)
 
+  /** Re-encodes the Jelly-RDF stream with the default options of rdf to-jelly. Each input frame is
+    * written as one or more output frames.
+    */
+  private def jellyToJelly(
+      inputStream: InputStream,
+      outputStream: OutputStream,
+      opt: RdfFromJellyOptions,
+  ): Unit =
+    val writer = JellyWriterUtil.createWriter(
+      RdfJellySerializationOptions.defaultFor(RdfFormat.JellyBinary),
+      RdfJellySerializationOptions.defaultRowsPerFrame,
+      enableNamespaceDeclarations = false,
+      delimited = true,
+      outputStream,
+    )
+    writer.start()
+    jellyToLang(inputStream, writer, RdfFormat.JellyBinary, opt)
+
   /** Converts the Jelly-RDF stream to a Jelly-SPARQL result set of ?s ?p ?o (?g), with the default
     * options. To set them, use sparql to-jelly --in-format jelly-rdf.
     */

src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfToJelly.scala

@@ -52,9 +52,10 @@ case class RdfToJellyOptions(
     )
     optionsFrom: Option[String] = None,
     @HelpMessage(
-      "Target number of rows per frame – the writer may slightly exceed that. Default: 256",
+      "Target number of rows per frame – the writer may slightly exceed that. Default: " +
+        RdfJellySerializationOptions.defaultRowsPerFrame,
     )
-    rowsPerFrame: Int = 256,
+    rowsPerFrame: Int = RdfJellySerializationOptions.defaultRowsPerFrame,
     @HelpMessage(
       "Whether to preserve explicit namespace declarations in the output (PREFIX: in Turtle). " +
         "Default: false",

src/main/scala/eu/neverblink/jelly/cli/command/rdf/util/RdfFormat.scala

@@ -94,9 +94,13 @@ object RdfFormat:
     override val supportsBaseIri: Boolean = true
     override val supportsQuads: Boolean = true
 
-  // We do not ever want to write or read from Jelly to Jelly
-  // So better not have it as Writeable or Readable, just mark that it's integrated into Jena
-  case object JellyBinary extends RdfFormat.Jena, RdfFormat.SupportsGeneralizedRdf:
+  /** Jelly-RDF. Reading it in rdf to-jelly and writing it in rdf from-jelly re-encodes the stream
+    * with new options.
+    */
+  case object JellyBinary
+      extends RdfFormat.Jena.Readable,
+        RdfFormat.Writeable,
+        RdfFormat.SupportsGeneralizedRdf:
     override val fullName: String = "Jelly binary"
     override val cliOptions: List[String] = List("jelly")
     override val jenaLang: Lang = JellyLanguage.JELLY

src/main/scala/eu/neverblink/jelly/cli/command/rdf/util/RdfJellySerializationOptions.scala

@@ -13,6 +13,15 @@ private val `default.opt.maxPrefixTableSize`: Int = JellyOptions.BIG_STRICT.getM
 private val `default.opt.maxDatatypeTableSize`: Int =
   JellyOptions.BIG_STRICT.getMaxDatatypeTableSize
 
+object RdfJellySerializationOptions:
+  val defaultRowsPerFrame: Int = 256
+
+  /** The stream options rdf to-jelly would use for input in this format, with no options set. */
+  def defaultFor(inputFormat: RdfFormat): RdfStreamOptions =
+    val options = RdfJellySerializationOptions()
+    options.inferGeneralized(Some(inputFormat.cliOptions.head), None)
+    options.asRdfStreamOptions
+
 /** Options for serializing in Jelly-RDF */
 case class RdfJellySerializationOptions(
     @HelpMessage("Name of the output stream (in metadata). Default: (empty)")

src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlFromJelly.scala

@@ -2,7 +2,7 @@ package eu.neverblink.jelly.cli.command.sparql
 
 import caseapp.*
 import eu.neverblink.jelly.cli.*
-import eu.neverblink.jelly.cli.command.rdf.util.RdfJellySerializationOptions
+import eu.neverblink.jelly.cli.command.rdf.util.{RdfFormat, RdfJellySerializationOptions}
 import eu.neverblink.jelly.cli.command.sparql.util.SparqlFormat
 import eu.neverblink.jelly.cli.util.jena.RdfSparqlConverter
 
@@ -67,7 +67,7 @@ object SparqlFromJelly extends SparqlSerDesCommand[SparqlFromJellyOptions]:
     RdfSparqlConverter.sparqlToRdf(
       inputStream,
       outputStream,
-      RdfJellySerializationOptions().asRdfStreamOptions,
-      rowsPerFrame = 256,
+      RdfJellySerializationOptions.defaultFor(RdfFormat.JellySparql),
+      RdfJellySerializationOptions.defaultRowsPerFrame,
       delimited = true,
     )

src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlSerDesCommand.scala

@@ -2,10 +2,23 @@ package eu.neverblink.jelly.cli.command.sparql
 
 import caseapp.*
 import eu.neverblink.jelly.cli.*
-import eu.neverblink.jelly.cli.command.sparql.util.{SparqlFormat, SparqlJellyUtil}
+import eu.neverblink.jelly.cli.command.sparql.util.{
+  SparqlFormat,
+  SparqlJellyUtil,
+  SparqlResultSetReader,
+}
 import eu.neverblink.jelly.cli.util.io.ProtoText
-import eu.neverblink.jelly.convert.jena.sparql.{JenaSparqlConverterFactory, RowSetReaderJelly}
-import eu.neverblink.jelly.core.proto.v1.sparql.SparqlResultsFrame
+import eu.neverblink.jelly.convert.jena.sparql.{
+  JellySparqlLanguage,
+  JenaSparqlConverterFactory,
+  RowSetReaderJelly,
+  RowSetWriterJelly,
+}
+import eu.neverblink.jelly.core.proto.v1.sparql.{
+  SparqlResultsFrame,
+  SparqlResultsOptions,
+  SparqlStreamType,
+}
 import org.apache.jena.riot.RIOT
 import org.apache.jena.riot.rowset.{RowSetReaderRegistry, RowSetWriterRegistry}
 import org.apache.jena.sparql.exec.QueryExecResult
@@ -50,9 +63,15 @@ abstract class SparqlSerDesCommand[T <: HasJellyCommandOptions: {Parser, Help}]
     */
   protected def jenaContext: Context = RIOT.getContext.copy()
 
-  /** Whether the Jelly-SPARQL text format should be converted to delimited Jelly-SPARQL. */
+  /** Whether the Jelly-SPARQL output should be delimited. */
   protected def delimitedOutput: Boolean = true
 
+  /** Stream options of the Jelly-SPARQL output, made from the options of the Jelly-SPARQL input.
+    * Commands that let the user set the options override this.
+    */
+  protected def jellySparqlOutputOptions(inputOptions: SparqlResultsOptions): SparqlResultsOptions =
+    SparqlJellyUtil.defaultOptions(inputOptions)
+
   /** Converts Jelly-RDF to Jelly-SPARQL. Commands that read Jelly-RDF override this. */
   protected def jellyRdfToSparql(inputStream: InputStream, outputStream: OutputStream): Unit =
     throw CriticalException("This command cannot read Jelly-RDF.")
@@ -82,6 +101,8 @@ abstract class SparqlSerDesCommand[T <: HasJellyCommandOptions: {Parser, Help}]
           jellyRdfToSparql(inputStream, outputStream)
         case (SparqlFormat.JellySparql, SparqlFormat.JellyRdf) =>
           jellySparqlToRdf(inputStream, outputStream)
+        case (SparqlFormat.JellySparql, SparqlFormat.JellySparql) =>
+          jellySparqlToSparql(inputStream, outputStream)
         case (f: SparqlFormat.Jena, t: SparqlFormat.Jena) =>
           jenaConvert(f, t, inputStream, outputStream)
         case _ =>
@@ -96,26 +117,79 @@ abstract class SparqlSerDesCommand[T <: HasJellyCommandOptions: {Parser, Help}]
       outputStream: OutputStream,
   ): Unit =
     val context = jenaContext
-    val writer = RowSetWriterRegistry.getFactory(to.jenaLang).create(to.jenaLang)
-    def write(result: QueryExecResult): Unit =
-      if result.isBoolean then
-        writer.write(outputStream, result.booleanResult.booleanValue, context)
-      else writer.write(outputStream, result.rowSet, context)
+    // The result sets of a PUNCTUATED stream are written one after another
+    readResults(from, inputStream, context)(write(to, outputStream, context, _))
+
+  /** Reads the result sets of the input, in order. A Jelly-SPARQL stream may have many of them,
+    * other formats have one.
+    */
+  protected final def readResults(
+      from: SparqlFormat.Jena,
+      inputStream: InputStream,
+      context: Context,
+  )(consumer: QueryExecResult => Unit): Unit =
     from match
       case SparqlFormat.JellySparql =>
-        // A PUNCTUATED stream has many result sets, which are written one after another
         RowSetReaderJelly(RowSetReaderJelly.Options(), JenaSparqlConverterFactory.getInstance())
-          .readAll(inputStream, context, write)
-      case _ => write(readResult(from, inputStream, context))
+          .readAll(inputStream, context, consumer(_))
+      case _ =>
+        consumer(
+          RowSetReaderRegistry.getFactory(from.jenaLang).create(from.jenaLang)
+            .readAny(inputStream, context),
+        )
 
-  /** Reads a result set in a format other than Jelly-SPARQL. */
-  protected final def readResult(
-      from: SparqlFormat.Jena,
-      inputStream: InputStream,
+  /** Writes one result set, as a whole stream in the given format. */
+  private def write(
+      to: SparqlFormat.Jena,
+      outputStream: OutputStream,
       context: Context,
-  ): QueryExecResult =
-    RowSetReaderRegistry.getFactory(from.jenaLang).create(from.jenaLang)
-      .readAny(inputStream, context)
+      result: QueryExecResult,
+  ): Unit =
+    val writer = RowSetWriterRegistry.getFactory(to.jenaLang).create(to.jenaLang)
+    if result.isBoolean then writer.write(outputStream, result.booleanResult.booleanValue, context)
+    else writer.write(outputStream, result.rowSet, context)
+
+  /** Starts a PUNCTUATED Jelly-SPARQL stream, with the options in the context. */
+  protected final def resultSetsWriter(
+      outputStream: OutputStream,
+      context: Context,
+  ): RowSetWriterJelly.ResultSetsWriter =
+    RowSetWriterJelly(RowSetWriterJelly.Options(), JenaSparqlConverterFactory.getInstance())
+      .resultSetsWriter(outputStream, context)
+
+  /** Writes the result set as the next one of a PUNCTUATED stream. */
+  protected final def writeNext(
+      writer: RowSetWriterJelly.ResultSetsWriter,
+      result: QueryExecResult,
+  ): Unit =
+    if result.isBoolean then writer.write(result.booleanResult.booleanValue)
+    else writer.write(result.rowSet)
+
+  /** Re-encodes a Jelly-SPARQL stream, with the options from [[jellySparqlOutputOptions]].
+    *
+    * All result sets of a PUNCTUATED output go into one stream. A FLAT output can only have one.
+    */
+  private def jellySparqlToSparql(inputStream: InputStream, outputStream: OutputStream): Unit =
+    val resultSets = SparqlResultSetReader(SparqlJellyUtil.iterateSparqlStream(inputStream))
+    if !resultSets.hasNext then throw CriticalException("Empty input stream")
+    val options = jellySparqlOutputOptions(resultSets.options)
+    val context = jenaContext.set(JellySparqlLanguage.SYMBOL_STREAM_OPTIONS, options)
+    if options.getStreamType == SparqlStreamType.PUNCTUATED then
+      if !delimitedOutput then
+        throw InvalidArgument(
+          "--delimited",
+          "false",
+          Some("PUNCTUATED streams are always written delimited"),
+        )
+      val writer = resultSetsWriter(outputStream, context)
+      resultSets.foreach(r => writeNext(writer, r.toQueryExecResult))
+    else
+      write(SparqlFormat.JellySparql, outputStream, context, resultSets.next().toQueryExecResult)
+      if resultSets.hasNext then
+        throw CriticalException(
+          "The input has more than one result set, so it can only be written as a PUNCTUATED " +
+            "stream.",
+        )
 
   private val frameCommentPrefix = "# Frame"
 

src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlToJelly.scala

@@ -5,11 +5,7 @@ import eu.neverblink.jelly.cli.*
 import eu.neverblink.jelly.cli.command.sparql.util.*
 import eu.neverblink.jelly.cli.util.jena.RdfSparqlConverter
 import eu.neverblink.jelly.cli.util.io.IoUtil
-import eu.neverblink.jelly.convert.jena.sparql.{
-  JellySparqlLanguage,
-  JenaSparqlConverterFactory,
-  RowSetWriterJelly,
-}
+import eu.neverblink.jelly.convert.jena.sparql.JellySparqlLanguage
 import eu.neverblink.jelly.core.RdfProtoDeserializationError
 import eu.neverblink.jelly.core.proto.v1.sparql.{SparqlResultsOptions, SparqlStreamType}
 import eu.neverblink.jelly.core.sparql.{JellySparqlConstants, JellySparqlOptions}
@@ -130,28 +126,29 @@ object SparqlToJelly extends SparqlSerDesCommand[SparqlToJellyOptions]:
       getIoStreamsFromOptions(inputFiles.headOption, options.outputFile)
     inputFormats.head match
       case format: SparqlFormat.Jena if punctuated =>
-        val writer = resultSetsWriter(outputStream)
-        writeResultSet(writer, format, inputStream)
+        val context = jenaContext
+        val writer = resultSetsWriter(outputStream, context)
+        def writeAll(format: SparqlFormat.Jena, inputStream: InputStream): Unit =
+          SparqlJellyUtil.translateErrors {
+            readResults(format, inputStream, context)(writeNext(writer, _))
+          }
+        writeAll(format, inputStream)
         // The formats of the other files were checked above
         for case (file, format: SparqlFormat.Jena) <- inputFiles.zip(inputFormats).drop(1) do
-          Using.resource(IoUtil.inputStream(file))(writeResultSet(writer, format, _))
+          Using.resource(IoUtil.inputStream(file))(writeAll(format, _))
         outputStream.flush()
       case format => convert(format, SparqlFormat.JellySparql, inputStream, outputStream)
 
-  private def resultSetsWriter(outputStream: OutputStream): RowSetWriterJelly.ResultSetsWriter =
-    RowSetWriterJelly(RowSetWriterJelly.Options(), JenaSparqlConverterFactory.getInstance())
-      .resultSetsWriter(outputStream, jenaContext)
+  /** The options set by the user, on top of `options-from` or the given default. */
+  private def outputOptions(default: SparqlResultsOptions): SparqlResultsOptions =
+    getOptions.jellySerializationOptions.toSparqlResultsOptions(
+      Some(baseOptions.getOrElse(default)),
+    )
 
-  private def writeResultSet(
-      writer: RowSetWriterJelly.ResultSetsWriter,
-      format: SparqlFormat.Jena,
-      inputStream: InputStream,
-  ): Unit =
-    SparqlJellyUtil.translateErrors {
-      val result = readResult(format, inputStream, jenaContext)
-      if result.isBoolean then writer.write(result.booleanResult.booleanValue)
-      else writer.write(result.rowSet)
-    }
+  override protected def jellySparqlOutputOptions(
+      inputOptions: SparqlResultsOptions,
+  ): SparqlResultsOptions =
+    outputOptions(SparqlJellyUtil.defaultOptions(inputOptions))
 
   override protected def jellyRdfToSparql(
       inputStream: InputStream,
@@ -160,10 +157,7 @@ object SparqlToJelly extends SparqlSerDesCommand[SparqlToJellyOptions]:
     RdfSparqlConverter.rdfToSparql(
       inputStream,
       outputStream,
-      inputOptions =>
-        getOptions.jellySerializationOptions.toSparqlResultsOptions(
-          Some(baseOptions.getOrElse(RdfSparqlConverter.defaultSparqlOptions(inputOptions))),
-        ),
+      inputOptions => outputOptions(RdfSparqlConverter.defaultSparqlOptions(inputOptions)),
       getOptions.valuesPerFrame,
       getOptions.delimited,
     )

src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlValidate.scala

@@ -388,16 +388,9 @@ object SparqlValidate extends JellyCommand[SparqlValidateOptions]:
     if lang == SparqlFormat.JellySparql.jenaLang then
       SparqlResultSetReader(SparqlJellyUtil.iterateSparqlStream(inputStream))
     else
-      val result = RowSetReaderRegistry.getFactory(lang).create(lang)
-        .readAny(inputStream, RIOT.getContext.copy())
-      if result.isBoolean then Iterator(SparqlResultSet.Ask(result.booleanResult.booleanValue))
-      else
-        val rowSet = result.rowSet
-        val vars = rowSet.getResultVars.asScala.toIndexedSeq
-        Iterator(
-          SparqlResultSet.Select(
-            vars,
-            Iterator.continually(rowSet).takeWhile(_.hasNext).map(_.next())
-              .map(b => vars.map(b.get).toArray),
-          ),
-        )
+      Iterator(
+        SparqlResultSet(
+          RowSetReaderRegistry.getFactory(lang).create(lang)
+            .readAny(inputStream, RIOT.getContext.copy()),
+        ),
+      )

src/main/scala/eu/neverblink/jelly/cli/command/sparql/util/SparqlFormat.scala

@@ -53,10 +53,10 @@ object SparqlFormat:
     override val cliOptions: List[String] = List("text")
     override val jenaLang: Lang = ResultSetLang.RS_Text
 
-  /** We never convert Jelly to Jelly, so this is neither Readable nor Writeable – it is only here
-    * so that the other side of the conversion has a name.
+  /** Converting Jelly-SPARQL to Jelly-SPARQL re-encodes the stream with new options. This does not
+    * go through Jena, see [[eu.neverblink.jelly.cli.command.sparql.SparqlSerDesCommand]].
     */
-  case object JellySparql extends SparqlFormat.Jena:
+  case object JellySparql extends SparqlFormat.Jena, SparqlFormat.Readable, SparqlFormat.Writeable:
     override val fullName: String = "Jelly-SPARQL"
     override val cliOptions: List[String] = List("jelly-sparql")
     override val jenaLang: Lang = JellySparqlLanguage.JELLY_SPARQL